trafficserver-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a..@apache.org
Subject [trafficserver] branch master updated: use schedule_spawn to start accept threads when set to 0
Date Tue, 13 Feb 2018 22:23:26 GMT
This is an automated email from the ASF dual-hosted git repository.

amc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/trafficserver.git


The following commit(s) were added to refs/heads/master by this push:
     new ea85a7d  use schedule_spawn to start accept threads when set to 0
ea85a7d is described below

commit ea85a7d59a3cbfa666091594e01db4ba20ea79f8
Author: Fei Deng <duke8253@gmail.com>
AuthorDate: Tue Nov 14 15:51:25 2017 -0600

    use schedule_spawn to start accept threads when set to 0
---
 iocore/net/P_NetAccept.h          |  4 ++++
 iocore/net/SSLSNIConfig.cc        |  2 +-
 iocore/net/UnixNetAccept.cc       |  4 ++++
 iocore/net/UnixNetProcessor.cc    |  8 ++++++--
 proxy/InkAPI.cc                   |  8 +++++---
 proxy/Main.cc                     | 32 ++++++++++++++++++++++----------
 proxy/http/HttpProxyServerMain.cc | 39 ++++++++++++++++++++++++++++++++++++++-
 proxy/http/HttpProxyServerMain.h  | 17 +++++++++++++++--
 8 files changed, 95 insertions(+), 19 deletions(-)

diff --git a/iocore/net/P_NetAccept.h b/iocore/net/P_NetAccept.h
index a9c98ca..d14c5a5 100644
--- a/iocore/net/P_NetAccept.h
+++ b/iocore/net/P_NetAccept.h
@@ -39,6 +39,7 @@
 #ifndef __P_NETACCEPT_H__
 #define __P_NETACCEPT_H__
 
+#include <vector>
 #include "ts/ink_platform.h"
 #include "P_Connection.h"
 
@@ -114,4 +115,7 @@ struct NetAccept : public Continuation {
   virtual ~NetAccept() { action_ = nullptr; }
 };
 
+extern Ptr<ProxyMutex> naVecMutex;
+extern std::vector<NetAccept *> naVec;
+
 #endif
diff --git a/iocore/net/SSLSNIConfig.cc b/iocore/net/SSLSNIConfig.cc
index fa51bf9..8f14d5b 100644
--- a/iocore/net/SSLSNIConfig.cc
+++ b/iocore/net/SSLSNIConfig.cc
@@ -42,7 +42,6 @@
 
 static ConfigUpdateHandler<SNIConfig> *sniConfigUpdate;
 struct NetAccept;
-extern std::vector<NetAccept *> naVec;
 Map<int, SSLNextProtocolSet *> snpsMap;
 extern TunnelHashMap TunnelMap;
 NextHopProperty::NextHopProperty()
@@ -226,6 +225,7 @@ SNIConfig::startup()
 void
 SNIConfig::cloneProtoSet()
 {
+  SCOPED_MUTEX_LOCK(lock, naVecMutex, this_ethread());
   for (auto na : naVec) {
     if (na->snpa) {
       auto snps = na->snpa->cloneProtoSet();
diff --git a/iocore/net/UnixNetAccept.cc b/iocore/net/UnixNetAccept.cc
index aac3bbd..29775f1 100644
--- a/iocore/net/UnixNetAccept.cc
+++ b/iocore/net/UnixNetAccept.cc
@@ -31,6 +31,9 @@
 using NetAcceptHandler = int (NetAccept::*)(int, void *);
 int accept_till_done   = 1;
 
+// we need to protect naVec since it might be accessed
+// in different threads at the same time
+Ptr<ProxyMutex> naVecMutex;
 std::vector<NetAccept *> naVec;
 static void
 safe_delay(int msec)
@@ -139,6 +142,7 @@ Ldone:
 NetAccept *
 getNetAccept(int ID)
 {
+  SCOPED_MUTEX_LOCK(lock, naVecMutex, this_ethread());
   return naVec.at(ID);
 }
 
diff --git a/iocore/net/UnixNetProcessor.cc b/iocore/net/UnixNetProcessor.cc
index 13d37a9..f63eaa0 100644
--- a/iocore/net/UnixNetProcessor.cc
+++ b/iocore/net/UnixNetProcessor.cc
@@ -30,7 +30,6 @@
 #include "StatPages.h"
 
 int net_accept_number = 0;
-extern std::vector<NetAccept *> naVec;
 NetProcessor::AcceptOptions const NetProcessor::DEFAULT_ACCEPT_OPTIONS;
 
 NetProcessor::AcceptOptions &
@@ -174,7 +173,12 @@ UnixNetProcessor::accept_internal(Continuation *cont, int fd, AcceptOptions
cons
   } else {
     na->init_accept(nullptr);
   }
-  naVec.push_back(na);
+
+  {
+    SCOPED_MUTEX_LOCK(lock, naVecMutex, this_ethread());
+    naVec.push_back(na);
+  }
+
 #ifdef TCP_DEFER_ACCEPT
   // set tcp defer accept timeout if it is configured, this will not trigger an accept until
there is
   // data on the socket ready to be read
diff --git a/proxy/InkAPI.cc b/proxy/InkAPI.cc
index a077b21..eb59342 100644
--- a/proxy/InkAPI.cc
+++ b/proxy/InkAPI.cc
@@ -9289,12 +9289,13 @@ TSAcceptorGet(TSVConn sslp)
   return ssl_vc ? reinterpret_cast<TSAcceptor>(ssl_vc->accept_object) : nullptr;
 }
 
-extern std::vector<NetAccept *> naVec;
 TSAcceptor
 TSAcceptorGetbyID(int ID)
 {
-  Debug("ssl", "getNetAccept in INK API.cc %p", naVec.at(ID));
-  return reinterpret_cast<TSAcceptor>(naVec.at(ID));
+  SCOPED_MUTEX_LOCK(lock, naVecMutex, this_ethread());
+  auto ret = naVec.at(ID);
+  Debug("ssl", "getNetAccept in INK API.cc %p", ret);
+  return reinterpret_cast<TSAcceptor>(ret);
 }
 
 int
@@ -9307,6 +9308,7 @@ TSAcceptorIDGet(TSAcceptor acceptor)
 int
 TSAcceptorCount()
 {
+  SCOPED_MUTEX_LOCK(lock, naVecMutex, this_ethread());
   return naVec.size();
 }
 
diff --git a/proxy/Main.cc b/proxy/Main.cc
index 5117954..53a122a 100644
--- a/proxy/Main.cc
+++ b/proxy/Main.cc
@@ -128,9 +128,11 @@ static void *mgmt_lifecycle_msg_callback(void *x, char *data, int len);
 static void init_ssl_ctx_callback(void *ctx, bool server);
 static void load_ssl_file_callback(const char *ssl_file, unsigned int options);
 
-static int num_of_net_threads = ink_number_of_processors();
+// We need these two to be accessible somewhere else now
+int num_of_net_threads = ink_number_of_processors();
+int num_accept_threads = 0;
+
 static int num_of_udp_threads = 0;
-static int num_accept_threads = 0;
 static int num_task_threads   = 0;
 
 static char *http_accept_port_descriptor;
@@ -1718,7 +1720,7 @@ main(int /* argc ATS_UNUSED */, const char **argv)
   // We need to do this early so we can initialize the Machine
   // singleton, which depends on configuration values loaded in this.
   // We want to initialize Machine as early as possible because it
-  // has other dependencies. Hopefully not in init_HttpProxyServer().
+  // has other dependencies. Hopefully not in prep_HttpProxyServer().
   HttpConfig::startup();
   /* Set up the machine with the outbound address if that's set,
      or the inbound address if set, otherwise let it default.
@@ -1793,9 +1795,21 @@ main(int /* argc ATS_UNUSED */, const char **argv)
   ink_dns_init(makeModuleVersion(HOSTDB_MODULE_MAJOR_VERSION, HOSTDB_MODULE_MINOR_VERSION,
PRIVATE_MODULE_HEADER));
   ink_split_dns_init(makeModuleVersion(1, 0, PRIVATE_MODULE_HEADER));
 
+  naVecMutex             = new_ProxyMutex();
+  started_et_net_threads = 0;
+
   // Do the inits for NetProcessors that use ET_NET threads. MUST be before starting those
threads.
   netProcessor.init();
-  init_HttpProxyServer();
+  prep_HttpProxyServer();
+
+  if (num_accept_threads == 0) {
+    eventProcessor.schedule_spawn(&init_HttpProxyServer, ET_NET);
+  } else {
+    std::unique_lock<std::mutex> lock(proxyServerMutex);
+    et_net_threads_ready = true;
+    lock.unlock();
+    proxyServerCheck.notify_one();
+  }
 
   // !! ET_NET threads start here !!
   // This means any spawn scheduling must be done before this point.
@@ -1934,6 +1948,10 @@ main(int /* argc ATS_UNUSED */, const char **argv)
       if (delay_p && ink_atomic_cas(&delay_listen_for_cache_p, 0, 1)) {
         Debug("http_listen", "Delaying listen, waiting for cache initialization");
       } else {
+        // Use a condition variable to check if we are ready to call
+        // start_HttpProxyServer() when num_accept_threads is set to 0.
+        std::unique_lock<std::mutex> lock(proxyServerMutex);
+        proxyServerCheck.wait(lock, [] { return et_net_threads_ready; });
         start_HttpProxyServer(); // PORTS_READY_HOOK called from in here
       }
     }
@@ -1946,12 +1964,6 @@ main(int /* argc ATS_UNUSED */, const char **argv)
     // "Task" processor, possibly with its own set of task threads
     tasksProcessor.start(num_task_threads, stacksize);
 
-    int back_door_port = NO_FD;
-    REC_ReadConfigInteger(back_door_port, "proxy.config.process_manager.mgmt_port");
-    if (back_door_port != NO_FD) {
-      start_HttpProxyServerBackDoor(back_door_port, num_accept_threads > 0 ? 1 : 0); //
One accept thread is enough
-    }
-
     if (netProcessor.socks_conf_stuff->accept_enabled) {
       start_SocksProxy(netProcessor.socks_conf_stuff->accept_port);
     }
diff --git a/proxy/http/HttpProxyServerMain.cc b/proxy/http/HttpProxyServerMain.cc
index 21142ca..e4dce2d 100644
--- a/proxy/http/HttpProxyServerMain.cc
+++ b/proxy/http/HttpProxyServerMain.cc
@@ -39,6 +39,7 @@
 #include "ProtocolProbeSessionAccept.h"
 #include "http2/Http2SessionAccept.h"
 #include "HttpConnectionCount.h"
+#include "HttpProxyServerMain.h"
 
 #include <vector>
 
@@ -48,6 +49,15 @@ HttpSessionAccept *plugin_http_transparent_accept = nullptr;
 static SLL<SSLNextProtocolAccept> ssl_plugin_acceptors;
 static Ptr<ProxyMutex> ssl_plugin_mutex;
 
+// used to keep count of how many et_net threads we have started
+std::atomic<int> started_et_net_threads;
+std::mutex proxyServerMutex;
+std::condition_variable proxyServerCheck;
+bool et_net_threads_ready = false;
+
+extern int num_of_net_threads;
+extern int num_accept_threads;
+
 bool
 ssl_register_protocol(const char *protocol, Continuation *contp)
 {
@@ -227,7 +237,7 @@ MakeHttpProxyAcceptor(HttpProxyAcceptor &acceptor, HttpProxyPort &port,
unsigned
 
 /// Do all pre-thread initialization / setup.
 void
-init_HttpProxyServer()
+prep_HttpProxyServer()
 {
   httpSessionManager.init();
 }
@@ -274,6 +284,25 @@ init_accept_HttpProxyServer(int n_accept_threads)
   }
 }
 
+/** Increment the counter to keep track of how many et_net threads
+ *  we have started. This function is scheduled at the start of each
+ *  et_net thread using schedule_spawn(). We also check immediately
+ *  after incrementing the counter to see whether all of the et_net
+ *  threads have started such that we can notify main() to call
+ *  start_HttpProxyServer().
+ */
+void
+init_HttpProxyServer(EThread *)
+{
+  auto check_et_net_num = ++started_et_net_threads;
+  if (check_et_net_num == num_of_net_threads) {
+    std::unique_lock<std::mutex> lock(proxyServerMutex);
+    et_net_threads_ready = true;
+    lock.unlock();
+    proxyServerCheck.notify_one();
+  }
+}
+
 void
 start_HttpProxyServer()
 {
@@ -318,6 +347,14 @@ start_HttpProxyServer()
     hook->invoke(TS_EVENT_LIFECYCLE_PORTS_READY, nullptr);
     hook = hook->next();
   }
+
+  // Start the back door, since it's just a special HttpProxyServer,
+  // the requirements to start it has been met if we got here.
+  int back_door_port = NO_FD;
+  REC_ReadConfigInteger(back_door_port, "proxy.config.process_manager.mgmt_port");
+  if (back_door_port != NO_FD) {
+    start_HttpProxyServerBackDoor(back_door_port, !!num_accept_threads); // One accept thread
is enough
+  }
 }
 
 void
diff --git a/proxy/http/HttpProxyServerMain.h b/proxy/http/HttpProxyServerMain.h
index c640069..039c334 100644
--- a/proxy/http/HttpProxyServerMain.h
+++ b/proxy/http/HttpProxyServerMain.h
@@ -21,17 +21,25 @@
   limitations under the License.
  */
 
+#include <atomic>
+#include <mutex>
+#include <condition_variable>
+
 struct HttpProxyPort;
 
 /// Perform any pre-thread start initialization.
-void init_HttpProxyServer();
+void prep_HttpProxyServer();
 
 /** Initialize all HTTP proxy port data structures needed to accept connections.
  */
 void init_accept_HttpProxyServer(int n_accept_threads = 0);
 
+/** Checkes whether we can call start_HttpProxyServer().
+ */
+void init_HttpProxyServer(EThread *);
+
 /** Start the proxy server.
-    The port data should have been created by @c init_HttpProxyServer().
+    The port data should have been created by @c prep_HttpProxyServer().
 */
 void start_HttpProxyServer();
 
@@ -40,3 +48,8 @@ void stop_HttpProxyServer();
 void start_HttpProxyServerBackDoor(int port, int accept_threads = 0);
 
 NetProcessor::AcceptOptions make_net_accept_options(const HttpProxyPort *port, unsigned nthreads);
+
+extern std::atomic<int> started_et_net_threads;
+extern std::mutex proxyServerMutex;
+extern std::condition_variable proxyServerCheck;
+extern bool et_net_threads_ready;

-- 
To stop receiving notification emails like this one, please contact
amc@apache.org.

Mime
View raw message