trafficserver-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zw...@apache.org
Subject [trafficserver] branch master updated: Allows for resizing librecords via command line option
Date Mon, 24 Jun 2019 21:21:57 GMT
This is an automated email from the ASF dual-hosted git repository.

zwoop pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/trafficserver.git


The following commit(s) were added to refs/heads/master by this push:
     new cba8b22  Allows for resizing librecords via command line option
cba8b22 is described below

commit cba8b22ad53e62a9a08443330c74f5ccbd3f3f6d
Author: Leif Hedstrom <zwoop@apache.org>
AuthorDate: Wed Jun 12 20:41:23 2019 -0600

    Allows for resizing librecords via command line option
    
    This also eliminates the old configure option --with-max-api-stats. Rather,
    you should set the full size of the entire librecords with this setting, with
    ~1100 needed for the core, and (recommended) 500 for metrics as a minimum.
    That is the default (1600) unless specified.
    
    The reason this has to be a command line is because librecords can not be
    initialized without it, hence, it can not be a records.config setting.
---
 doc/appendices/command-line/traffic_manager.en.rst |  1 +
 doc/appendices/command-line/traffic_server.en.rst  |  2 ++
 include/tscore/ink_config.h.in                     |  2 --
 lib/records/P_RecDefs.h                            |  9 +++++----
 lib/records/RecCore.cc                             |  7 ++++++-
 lib/records/RecUtils.cc                            |  5 +++--
 mgmt/RecordsConfig.h                               |  3 +++
 mgmt/api/CoreAPI.cc                                | 10 +++++++---
 plugins/experimental/remap_stats/remap_stats.c     |  5 ++++-
 src/traffic_layout/info.cc                         |  1 -
 src/traffic_manager/traffic_manager.cc             |  5 +++--
 src/traffic_server/InkAPI.cc                       | 10 ++++++----
 src/traffic_server/traffic_server.cc               |  3 ++-
 13 files changed, 42 insertions(+), 21 deletions(-)

diff --git a/doc/appendices/command-line/traffic_manager.en.rst b/doc/appendices/command-line/traffic_manager.en.rst
index 61a7fbd..d7a0f60 100644
--- a/doc/appendices/command-line/traffic_manager.en.rst
+++ b/doc/appendices/command-line/traffic_manager.en.rst
@@ -40,6 +40,7 @@ Description
 .. option:: --proxyPort PORT
 .. option:: --recordsConf FILE
 .. option:: --tsArgs ARGUMENTS
+.. option:: --maxRecords RECORDS
 .. option:: --version
 
 Signals
diff --git a/doc/appendices/command-line/traffic_server.en.rst b/doc/appendices/command-line/traffic_server.en.rst
index 75e17e8..7a2dae2 100644
--- a/doc/appendices/command-line/traffic_server.en.rst
+++ b/doc/appendices/command-line/traffic_server.en.rst
@@ -83,6 +83,8 @@ the available tests.
 
 .. option:: -t MSECS, --poll_timeout MSECS
 
+.. option:: -m RECORDS, --maxRecords RECORDS
+
 .. option:: -h, --help
 
    Print usage information and exit.
diff --git a/include/tscore/ink_config.h.in b/include/tscore/ink_config.h.in
index 36f9871..8c920f9 100644
--- a/include/tscore/ink_config.h.in
+++ b/include/tscore/ink_config.h.in
@@ -92,8 +92,6 @@
 
 #define TS_MAX_HOST_NAME_LEN @max_host_name_len@
 
-#define TS_MAX_API_STATS @max_api_stats@
-
 #define SPLIT_DNS 1
 
 /* Defaults for user / group */
diff --git a/lib/records/P_RecDefs.h b/lib/records/P_RecDefs.h
index fb89c2a..0aa4c09 100644
--- a/lib/records/P_RecDefs.h
+++ b/lib/records/P_RecDefs.h
@@ -31,10 +31,11 @@
 
 #define REC_MESSAGE_ELE_MAGIC 0xF00DF00D
 
-// This is for the internal stats and configs, as well as API stats. We currently use
-// about 1600 stats + configs for the core, but we're allocating 2000 for some growth.
-// TODO: if/when we switch to a new config system, we should make this run-time dynamic.
-#define REC_MAX_RECORDS (2000 + TS_MAX_API_STATS)
+// We need at least this many internal record entries for our configurations and metrics.
Any
+// additional slots in librecords will be allocated to the plugin metrics. These should be
+// updated if we change the internal librecords size significantly.
+#define REC_INTERNAL_RECORDS 1100
+#define REC_MIN_API_RECORDS 500
 
 #define REC_CONFIG_UPDATE_INTERVAL_MS 3000
 #define REC_REMOTE_SYNC_INTERVAL_MS 5000
diff --git a/lib/records/RecCore.cc b/lib/records/RecCore.cc
index 7c1f8ab..ba2e833 100644
--- a/lib/records/RecCore.cc
+++ b/lib/records/RecCore.cc
@@ -25,11 +25,16 @@
 #include "tscore/ink_memory.h"
 #include "tscore/ink_string.h"
 
+#include "RecordsConfig.h"
 #include "P_RecFile.h"
 #include "P_RecCore.h"
 #include "P_RecUtils.h"
 #include "tscore/I_Layout.h"
 
+// This is needed to manage the size of the librecords record. It can't be static, because
it needs to be modified
+// and used (read) from several binaries / modules.
+int max_records_entries = REC_INTERNAL_RECORDS + REC_MIN_API_RECORDS;
+
 static bool g_initialized = false;
 
 RecRecord *g_records = nullptr;
@@ -197,7 +202,7 @@ RecCoreInit(RecModeT mode_type, Diags *_diags)
   g_num_records = 0;
 
   // initialize record array for our internal stats (this can be reallocated later)
-  g_records = (RecRecord *)ats_malloc(REC_MAX_RECORDS * sizeof(RecRecord));
+  g_records = (RecRecord *)ats_malloc(max_records_entries * sizeof(RecRecord));
 
   // initialize record rwlock
   ink_rwlock_init(&g_records_rwlock);
diff --git a/lib/records/RecUtils.cc b/lib/records/RecUtils.cc
index 4a380b9..66d09c9 100644
--- a/lib/records/RecUtils.cc
+++ b/lib/records/RecUtils.cc
@@ -24,6 +24,7 @@
 #include "tscore/ink_platform.h"
 #include "tscore/ink_memory.h"
 #include "tscore/ParseRules.h"
+#include "RecordsConfig.h"
 #include "P_RecUtils.h"
 #include "P_RecCore.h"
 
@@ -49,8 +50,8 @@ RecRecordFree(RecRecord *r)
 RecRecord *
 RecAlloc(RecT rec_type, const char *name, RecDataT data_type)
 {
-  if (g_num_records >= REC_MAX_RECORDS) {
-    Warning("too many stats/configs, please increase REC_MAX_RECORDS or rebuild with --with_max_api_stats=<n>");
+  if (g_num_records >= max_records_entries) {
+    Warning("too many stats/configs, please increase max_records_entries using the --maxRecords
command line option");
     return nullptr;
   }
 
diff --git a/mgmt/RecordsConfig.h b/mgmt/RecordsConfig.h
index 3cb6b4a..c45cd3d 100644
--- a/mgmt/RecordsConfig.h
+++ b/mgmt/RecordsConfig.h
@@ -25,6 +25,9 @@
 
 #include "records/P_RecCore.h"
 
+// This is to manage the librecords table sizes. Not awesome, but better than the earlier
recompiling of ATS requirement...
+extern int max_records_entries;
+
 enum RecordRequiredType {
   RR_NULL,    // config is _not_ required to be defined in records.config
   RR_REQUIRED // config _is_ required to be defined in record.config
diff --git a/mgmt/api/CoreAPI.cc b/mgmt/api/CoreAPI.cc
index 1a0ff56..9b6555b 100644
--- a/mgmt/api/CoreAPI.cc
+++ b/mgmt/api/CoreAPI.cc
@@ -28,10 +28,12 @@
  *
  *
  ***************************************************************************/
+#include <vector>
 
 #include "tscore/ink_platform.h"
 #include "tscore/ink_file.h"
 #include "tscore/ParseRules.h"
+#include "RecordsConfig.h"
 #include "Alarms.h"
 #include "MgmtUtils.h"
 #include "LocalManager.h"
@@ -47,8 +49,6 @@
 #include "tscore/I_Layout.h"
 #include "tscore/ink_cap.h"
 
-#include <vector>
-
 // global variable
 static CallbackTable *local_event_callbacks;
 
@@ -170,7 +170,11 @@ ProxyStateSet(TSProxyStateT state, TSCacheClearT clear)
 
     // Start with the default options from records.config.
     if (RecGetRecordString_Xmalloc("proxy.config.proxy_binary_opts", &proxy_options)
== REC_ERR_OKAY) {
-      snprintf(tsArgs, sizeof(tsArgs), "%s", proxy_options);
+      if (max_records_entries == (REC_INTERNAL_RECORDS + REC_MIN_API_RECORDS)) { // Default
size, don't need to pass down to _server
+        snprintf(tsArgs, sizeof(tsArgs), "%s", proxy_options);
+      } else {
+        snprintf(tsArgs, sizeof(tsArgs), "%s --maxRecords %d", proxy_options, max_records_entries);
+      }
       ats_free(proxy_options);
     }
 
diff --git a/plugins/experimental/remap_stats/remap_stats.c b/plugins/experimental/remap_stats/remap_stats.c
index b5a9892..79c9315 100644
--- a/plugins/experimental/remap_stats/remap_stats.c
+++ b/plugins/experimental/remap_stats/remap_stats.c
@@ -42,6 +42,9 @@ typedef struct {
   TSMutex stat_creation_mutex;
 } config_t;
 
+// From "core".... sigh, but we need it for now at least.
+extern int max_records_entries;
+
 static void
 stat_add(char *name, TSMgmtInt amount, TSStatPersistence persist_type, TSMutex create_mutex)
 {
@@ -52,7 +55,7 @@ stat_add(char *name, TSMgmtInt amount, TSStatPersistence persist_type, TSMutex
c
 
   if (unlikely(!hash_init)) {
     // NOLINTNEXTLINE
-    hcreate_r(TS_MAX_API_STATS << 1, &stat_cache);
+    hcreate_r(max_records_entries << 1, &stat_cache); // This is weird, but oh
well.
     hash_init = true;
     TSDebug(DEBUG_TAG, "stat cache hash init");
   }
diff --git a/src/traffic_layout/info.cc b/src/traffic_layout/info.cc
index 184049d..2294524 100644
--- a/src/traffic_layout/info.cc
+++ b/src/traffic_layout/info.cc
@@ -116,7 +116,6 @@ produce_features(bool json)
   print_feature("TS_MAX_THREADS_IN_EACH_THREAD_TYPE", TS_MAX_THREADS_IN_EACH_THREAD_TYPE,
json);
   print_feature("TS_MAX_NUMBER_EVENT_THREADS", TS_MAX_NUMBER_EVENT_THREADS, json);
   print_feature("TS_MAX_HOST_NAME_LEN", TS_MAX_HOST_NAME_LEN, json);
-  print_feature("TS_MAX_API_STATS", TS_MAX_API_STATS, json);
   print_feature("SPLIT_DNS", SPLIT_DNS, json);
   print_feature("TS_PKGSYSUSER", TS_PKGSYSUSER, json);
   print_feature("TS_PKGSYSGROUP", TS_PKGSYSGROUP, json, true);
diff --git a/src/traffic_manager/traffic_manager.cc b/src/traffic_manager/traffic_manager.cc
index 99b7466..3612507 100644
--- a/src/traffic_manager/traffic_manager.cc
+++ b/src/traffic_manager/traffic_manager.cc
@@ -87,8 +87,7 @@ static int proxy_off          = false;
 static int listen_off         = false;
 static char bind_stdout[512]  = "";
 static char bind_stderr[512]  = "";
-
-static const char *mgmt_path = nullptr;
+static const char *mgmt_path  = nullptr;
 
 // By default, set the current directory as base
 static const char *recs_conf = "records.config";
@@ -497,6 +496,8 @@ main(int argc, const char **argv)
     {"recordsConf", '-', "Path to records.config", "S*", &recs_conf, nullptr, nullptr},
     {"tsArgs", '-', "Additional arguments for traffic_server", "S*", &tsArgs, nullptr,
nullptr},
     {"proxyPort", '-', "HTTP port descriptor", "S*", &proxy_port, nullptr, nullptr},
+    {"maxRecords", 'm', "Max number of librecords metrics and configurations (default &
minimum: 1600)", "I", &max_records_entries,
+     "PROXY_MAX_RECORDS", nullptr},
     {TM_OPT_BIND_STDOUT, '-', "Regular file to bind stdout to", "S512", &bind_stdout,
"PROXY_BIND_STDOUT", nullptr},
     {TM_OPT_BIND_STDERR, '-', "Regular file to bind stderr to", "S512", &bind_stderr,
"PROXY_BIND_STDERR", nullptr},
 #if TS_USE_DIAGS
diff --git a/src/traffic_server/InkAPI.cc b/src/traffic_server/InkAPI.cc
index 316282b..8e1e1ba 100644
--- a/src/traffic_server/InkAPI.cc
+++ b/src/traffic_server/InkAPI.cc
@@ -26,6 +26,7 @@
 #include <string_view>
 #include <tuple>
 #include <unordered_map>
+#include <string_view>
 
 #include "tscore/ink_platform.h"
 #include "tscore/ink_base64.h"
@@ -66,11 +67,11 @@
 #include "I_Tasks.h"
 
 #include "P_OCSPStapling.h"
+#include "RecordsConfig.h"
 #include "records/I_RecDefs.h"
 #include "records/I_RecCore.h"
 #include "I_Machine.h"
 #include "HttpProxyServerMain.h"
-#include <string_view>
 
 #include "ts/ts.h"
 
@@ -1678,12 +1679,13 @@ api_init()
     lifecycle_hooks   = new LifecycleAPIHooks;
     global_config_cbs = new ConfigUpdateCbTable;
 
-    if (TS_MAX_API_STATS > 0) {
-      api_rsb = RecAllocateRawStatBlock(TS_MAX_API_STATS);
+    int api_metrics = max_records_entries - REC_INTERNAL_RECORDS;
+    if (api_metrics > 0) {
+      api_rsb = RecAllocateRawStatBlock(api_metrics);
       if (nullptr == api_rsb) {
         Warning("Can't allocate API stats block");
       } else {
-        Debug("sdk", "initialized SDK stats APIs with %d slots", TS_MAX_API_STATS);
+        Debug("sdk", "initialized SDK stats APIs with %d slots", api_metrics);
       }
     } else {
       api_rsb = nullptr;
diff --git a/src/traffic_server/traffic_server.cc b/src/traffic_server/traffic_server.cc
index 66bd411..e11b1ed 100644
--- a/src/traffic_server/traffic_server.cc
+++ b/src/traffic_server/traffic_server.cc
@@ -163,7 +163,6 @@ HttpBodyFactory *body_factory              = nullptr;
 static int accept_mss           = 0;
 static int poll_timeout         = -1; // No value set.
 static int cmd_disable_freelist = 0;
-
 static bool signal_received[NSIG];
 
 // 1: delay listen, wait for cache.
@@ -182,6 +181,8 @@ static ArgumentDescription argument_descriptions[] = {
   {"disable_freelist", 'f', "Disable the freelist memory allocator", "T", &cmd_disable_freelist,
"PROXY_DPRINTF_LEVEL", nullptr},
   {"disable_pfreelist", 'F', "Disable the freelist memory allocator in ProxyAllocator", "T",
&cmd_disable_pfreelist,
    "PROXY_DPRINTF_LEVEL", nullptr},
+  {"maxRecords", 'm', "Max number of librecords metrics and configurations (default &
minimum: 1600)", "I", &max_records_entries,
+   "PROXY_MAX_RECORDS", nullptr},
 
 #if TS_HAS_TESTS
   {"regression", 'R', "Regression Level (quick:1..long:3)", "I", &regression_level, "PROXY_REGRESSION",
nullptr},


Mime
View raw message