trafficserver-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sor...@apache.org
Subject git commit: TS-2624: Make thread affinity more robust
Date Sun, 09 Mar 2014 05:27:30 GMT
Repository: trafficserver
Updated Branches:
  refs/heads/master 061695895 -> d94f8bc46


TS-2624: Make thread affinity more robust


Project: http://git-wip-us.apache.org/repos/asf/trafficserver/repo
Commit: http://git-wip-us.apache.org/repos/asf/trafficserver/commit/d94f8bc4
Tree: http://git-wip-us.apache.org/repos/asf/trafficserver/tree/d94f8bc4
Diff: http://git-wip-us.apache.org/repos/asf/trafficserver/diff/d94f8bc4

Branch: refs/heads/master
Commit: d94f8bc46aa3d4b218529d7b1861376fd4d819b6
Parents: 0616958
Author: Phil Sorber <sorber@apache.org>
Authored: Sat Mar 8 22:27:05 2014 -0700
Committer: Phil Sorber <sorber@apache.org>
Committed: Sat Mar 8 22:27:05 2014 -0700

----------------------------------------------------------------------
 CHANGES                                         |   2 +
 .../configuration/records.config.en.rst         |  11 +-
 iocore/eventsystem/UnixEventProcessor.cc        | 140 ++++++-------------
 lib/ts/ink_defs.cc                              |  18 +--
 4 files changed, 52 insertions(+), 119 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/trafficserver/blob/d94f8bc4/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index de548dd..ee3ef6e 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,6 +1,8 @@
                                                          -*- coding: utf-8 -*-
 Changes with Apache Traffic Server 5.0.0
 
+  *) [TS-2624] Make thread affinity more robust
+
   *) [TS-2620] Make the stats_over_http plugin publish node and plugin stats.
 
   *) [TS-2614] Response to invalid Content-Length for POST should be a 400 error

http://git-wip-us.apache.org/repos/asf/trafficserver/blob/d94f8bc4/doc/reference/configuration/records.config.en.rst
----------------------------------------------------------------------
diff --git a/doc/reference/configuration/records.config.en.rst b/doc/reference/configuration/records.config.en.rst
index 21b70d3..944502d 100644
--- a/doc/reference/configuration/records.config.en.rst
+++ b/doc/reference/configuration/records.config.en.rst
@@ -241,15 +241,16 @@ A value of ``0`` means no signal will be sent.
 
 .. ts:cv:: CONFIG proxy.config.exec_thread.affinity INT 0
 
-   Bind threads to specific CPUs or CPU cores.
+   Bind threads to specific processing units.
 
 ===== ====================
 Value Effect
 ===== ====================
-1     assign threads to sockets
-2     assign threads to real cores
-3     assign threads to logical cores
-0     don't assign threads to any cores
+0     assign threads to machine
+1     assign threads to NUMA nodes
+2     assign threads to sockets
+3     assign threads to cores
+4     assign threads to processing units
 ===== ====================
 
 .. note::

http://git-wip-us.apache.org/repos/asf/trafficserver/blob/d94f8bc4/iocore/eventsystem/UnixEventProcessor.cc
----------------------------------------------------------------------
diff --git a/iocore/eventsystem/UnixEventProcessor.cc b/iocore/eventsystem/UnixEventProcessor.cc
index 29f730a..40d2779 100644
--- a/iocore/eventsystem/UnixEventProcessor.cc
+++ b/iocore/eventsystem/UnixEventProcessor.cc
@@ -24,72 +24,11 @@
 #include "P_EventSystem.h"      /* MAGIC_EDITING_TAG */
 #include <sched.h>
 #if TS_USE_HWLOC
-
-#if HAVE_HWLOC_H
+#include <alloca.h>
 #include <hwloc.h>
 #endif
-
-#if HAVE_SCHED_H
-#include <sched.h>
-#if !defined(solaris) && !defined(freebsd)
-typedef cpu_set_t ink_cpuset_t;
-#define PTR_FMT PRIuPTR
-#endif
-#endif
-
-#if HAVE_SYS_PARAM_H
-#include <sys/param.h>
-#endif
-
-#if HAVE_SYS_CPUSET_H
-#include <sys/cpuset.h>
-typedef cpuset_t ink_cpuset_t;
-#define PTR_FMT "p"
-#endif
-
-#if HAVE_SYS_PSET_H
-#include <sys/pset.h>
-typedef psetid_t ink_cpuset_t;
-#define PTR_FMT "u"
-#endif
-
-#if HAVE_SYS_TYPES_H
-#include <sys/types.h>
-#endif
-
-#endif
 #include "ink_defs.h"
 
-#if TS_USE_HWLOC
-static void
-set_cpu(ink_cpuset_t *cpuset, int cpu)
-{
-#if !defined(solaris)
-  CPU_ZERO(cpuset);
-  CPU_SET(cpu, cpuset);
-#else
-  pset_create(cpuset);
-  pset_assign(*cpuset, cpu, NULL);
-#endif
-}
-
-
-static bool
-bind_cpu(ink_cpuset_t *cpuset, ink_thread tid)
-{
-  if ( 0 != 
-#if !defined(solaris)
-    pthread_setaffinity_np(tid, sizeof(ink_cpuset_t), cpuset)
-#else
-    pset_bind(*cpuset, P_LWPID, P_MYID, NULL)
-#endif
-    ){
-    return false;
-  }
-  return true;
-}
-#endif
-
 EventType
 EventProcessor::spawn_event_threads(int n_threads, const char* et_name, size_t stacksize)
 {
@@ -118,7 +57,7 @@ EventProcessor::spawn_event_threads(int n_threads, const char* et_name,
size_t s
 
   n_thread_groups++;
   n_ethreads += n_threads;
-  Debug("iocore_thread", "Created thread group '%s' id %d with %d threads", et_name, new_thread_group_id,
n_threads); 
+  Debug("iocore_thread", "Created thread group '%s' id %d with %d threads", et_name, new_thread_group_id,
n_threads);
 
   return new_thread_group_id;
 }
@@ -160,53 +99,56 @@ EventProcessor::start(int n_event_threads, size_t stacksize)
 #if TS_USE_HWLOC
   int affinity = 0;
   REC_ReadConfigInteger(affinity, "proxy.config.exec_thread.affinity");
-  ink_cpuset_t cpuset;
-  int socket = hwloc_get_nbobjs_by_type(ink_get_topology(), HWLOC_OBJ_SOCKET);
-  int cu = hwloc_get_nbobjs_by_type(ink_get_topology(), HWLOC_OBJ_CORE);
-  int pu = cu;
-
-  // Older versions of libhwloc (eg. Ubuntu 10.04) don't have pHWLOC_OBJ_PU.
+  hwloc_obj_t obj;
+  hwloc_obj_type_t obj_type;
+  int obj_count = 0, cpu_mask_len = 0;
+  char *obj_name, *cpu_mask;
+
+  switch(affinity) {
+    case 4:           // assign threads to logical processing units
+// Older versions of libhwloc (eg. Ubuntu 10.04) don't have HWLOC_OBJ_PU.
 #if HAVE_HWLOC_OBJ_PU
-  pu = hwloc_get_nbobjs_by_type(ink_get_topology(), HWLOC_OBJ_PU);
+      obj_type = HWLOC_OBJ_PU;
+      obj_name = (char *) "Logical Processor";
+      break;
 #endif
+    case 3:           // assign threads to real cores
+      obj_type = HWLOC_OBJ_CORE;
+      obj_name = (char *) "Core";
+      break;
+    case 1:           // assign threads to NUMA nodes (often 1:1 with sockets)
+      obj_type = HWLOC_OBJ_NODE;
+      obj_name = (char *) "NUMA Node";
+      if (hwloc_get_nbobjs_by_type(ink_get_topology(), obj_type) > 0) {
+        break;
+      }
+    case 2:           // assign threads to sockets
+      obj_type = HWLOC_OBJ_SOCKET;
+      obj_name = (char *) "Socket";
+      break;
+    default:         // assign threads to the machine as a whole (a level below SYSTEM)
+      obj_type = HWLOC_OBJ_MACHINE;
+      obj_name = (char *) "Machine";
+  }
 
-  Debug("iocore_thread", "socket: %d core: %d logical processor: %d affinity: %d", socket,
cu, pu, affinity);
-#endif
+    obj_count = hwloc_get_nbobjs_by_type(ink_get_topology(), obj_type);
+    Debug("iocore_thread", "Affinity: %d %ss: %d PU: %d", affinity, obj_name, obj_count,
ink_number_of_processors());
 
+#endif
   for (i = first_thread; i < n_ethreads; i++) {
     snprintf(thr_name, MAX_THREAD_NAME_LENGTH, "[ET_NET %d]", i);
     ink_thread tid = all_ethreads[i]->start(thr_name, stacksize);
     (void)tid;
-
 #if TS_USE_HWLOC
-    if (affinity != 0) {
-      int logical_ratio;
-      switch(affinity) {
-      case 3:           // assign threads to logical cores
-        logical_ratio = 1;
-        break;
-      case 2:           // assign threads to real cores
-        logical_ratio = pu / cu;
-        break;
-      case 1:           // assign threads to sockets
-      default:
-        logical_ratio = pu / socket;
-      }
-
-      char debug_message[256];
-      int len = snprintf(debug_message, sizeof(debug_message), "setaffinity tid: %" PTR_FMT
", net thread: %u cpu:", tid, i);
-      for (int cpu_count = 0; cpu_count < logical_ratio; cpu_count++) {
-        int cpu = ((i - 1) * logical_ratio + cpu_count) % pu;
-        set_cpu(&cpuset, cpu);
-        len += snprintf(debug_message + len, sizeof(debug_message) - len, " %d", cpu);
-      }
-      Debug("iocore_thread", "%s", debug_message);
-      if (!bind_cpu(&cpuset, tid)){
-        Error("%s, failed with errno: %d", debug_message, errno);
-      }
-    }
+    obj = hwloc_get_obj_by_type(ink_get_topology(), obj_type, i % obj_count);
+    cpu_mask_len = hwloc_bitmap_list_snprintf(NULL, 0, obj->cpuset) + 1;
+    cpu_mask = (char *) alloca(cpu_mask_len);
+    hwloc_bitmap_list_snprintf(cpu_mask, cpu_mask_len, obj->cpuset);
+    Debug("iocore_thread","EThread: %d %s: %d CPU List: %s\n", i, obj_name, obj->logical_index,
cpu_mask);
+    hwloc_set_thread_cpubind(ink_get_topology(), tid, obj->cpuset, HWLOC_CPUBIND_STRICT);
 #endif
   }
+
   Debug("iocore_thread", "Created event thread group id %d with %d threads", ET_CALL, n_event_threads);
   return 0;
 }

http://git-wip-us.apache.org/repos/asf/trafficserver/blob/d94f8bc4/lib/ts/ink_defs.cc
----------------------------------------------------------------------
diff --git a/lib/ts/ink_defs.cc b/lib/ts/ink_defs.cc
index dc68183..beffbd9 100644
--- a/lib/ts/ink_defs.cc
+++ b/lib/ts/ink_defs.cc
@@ -119,21 +119,12 @@ int
 ink_number_of_processors()
 {
 #if TS_USE_HWLOC
-
-  int cu = hwloc_get_nbobjs_by_type(ink_get_topology(), HWLOC_OBJ_CORE);
-
 #if HAVE_HWLOC_OBJ_PU
-  int pu = hwloc_get_nbobjs_by_type(ink_get_topology(), HWLOC_OBJ_PU);
-
-  if (pu > cu) {
-    return cu + (pu - cu)/4;
-  }
+  return hwloc_get_nbobjs_by_type(ink_get_topology(), HWLOC_OBJ_PU);
+#else
+  return hwloc_get_nbobjs_by_type(ink_get_topology(), HWLOC_OBJ_CORE);
 #endif
-
-  return cu;
-
 #elif defined(freebsd)
-
   int mib[2], n;
   mib[0] = CTL_HW;
   mib[1] = HW_NCPU;
@@ -141,10 +132,7 @@ ink_number_of_processors()
   if (sysctl(mib, 2, &n, &len, NULL, 0) == -1)
     return 1;
   return n;
-
 #else
-
   return sysconf(_SC_NPROCESSORS_ONLN); // number of processing units (includes Hyper Threading)
-
 #endif
 }


Mime
View raw message