trafficserver-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sor...@apache.org
Subject [trafficserver] branch master updated: Remove Clustering
Date Mon, 08 May 2017 18:59:41 GMT
This is an automated email from the ASF dual-hosted git repository.

sorber pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/trafficserver.git

The following commit(s) were added to refs/heads/master by this push:
       new  e846f31   Remove Clustering
e846f31 is described below

commit e846f31235e3ed18e945bfbd8b4b148c552e0754
Author: Phil Sorber <sorber@apache.org>
AuthorDate: Mon May 8 11:25:56 2017 -0600

    Remove Clustering
---
 cmd/traffic_cop/traffic_cop.cc                   |  144 +-
 cmd/traffic_ctl/config.cc                        |    2 -
 cmd/traffic_ctl/metric.cc                        |   24 +-
 cmd/traffic_manager/Makefile.am                  |    2 -
 cmd/traffic_manager/WebOverview.cc               |  638 -----
 cmd/traffic_manager/WebOverview.h                |  140 -
 cmd/traffic_manager/metrics.cc                   |   50 -
 cmd/traffic_manager/traffic_manager.cc           |   38 -
 configure.ac                                     |    2 -
 iocore/Makefile.am                               |    2 +-
 iocore/cache/Cache.cc                            |  121 -
 iocore/cache/CacheDir.cc                         |    1 +
 iocore/cache/CacheHosting.cc                     |    1 +
 iocore/cache/P_CacheHosting.h                    |    2 +
 iocore/cache/P_CacheInternal.h                   |   10 -
 iocore/cluster/ClusterAPI.cc                     |  560 ----
 iocore/cluster/ClusterCache.cc                   | 3059 ---------------------
 iocore/cluster/ClusterConfig.cc                  |  450 ----
 iocore/cluster/ClusterHandler.cc                 | 3068 ----------------------
 iocore/cluster/ClusterHandlerBase.cc             | 1438 ----------
 iocore/cluster/ClusterHash.cc                    |  171 --
 iocore/cluster/ClusterLib.cc                     |  346 ---
 iocore/cluster/ClusterLoadMonitor.cc             |  307 ---
 iocore/cluster/ClusterMachine.cc                 |  292 --
 iocore/cluster/ClusterProcessor.cc               |  764 ------
 iocore/cluster/ClusterRPC.cc                     |  375 ---
 iocore/cluster/ClusterVConnection.cc             |  655 -----
 iocore/cluster/Inline.cc                         |   30 -
 iocore/cluster/Makefile.am                       |   63 -
 iocore/cluster/P_Cluster.h                       |  133 -
 iocore/cluster/P_ClusterCache.h                  | 1175 ---------
 iocore/cluster/P_ClusterCacheInternal.h          |  857 ------
 iocore/cluster/P_ClusterHandler.h                |  678 -----
 iocore/cluster/P_ClusterInline.h                 |  375 ---
 iocore/cluster/P_ClusterInternal.h               |  534 ----
 iocore/cluster/P_ClusterLib.h                    |   70 -
 iocore/cluster/P_ClusterLoadMonitor.h            |  116 -
 iocore/cluster/P_ClusterMachine.h                |  131 -
 iocore/cluster/P_TimeTrace.h                     |   83 -
 iocore/cluster/test_I_Cluster.cc                 |   50 -
 iocore/cluster/test_P_Cluster.cc                 |   50 -
 iocore/hostdb/HostDB.cc                          |  368 +--
 iocore/hostdb/P_HostDB.h                         |    1 -
 iocore/hostdb/P_HostDBProcessor.h                |    3 -
 lib/bindings/metrics.cc                          |    2 +-
 lib/records/I_RecDefs.h                          |    1 -
 lib/records/P_RecCore.cc                         |   32 +-
 lib/records/P_RecUtils.h                         |    3 +-
 lib/records/RecConfigParse.cc                    |    2 -
 lib/records/RecLocal.cc                          |    1 -
 lib/records/RecProcess.cc                        |    2 -
 lib/ts/HostLookup.h                              |    3 +
 lib/ts/apidefs.h.in                              |    1 -
 mgmt/Alarms.cc                                   |   67 -
 mgmt/LocalManager.cc                             |  105 -
 mgmt/LocalManager.h                              |    6 -
 mgmt/Makefile.am                                 |    4 +-
 mgmt/RecordsConfigUtils.cc                       |    1 -
 mgmt/api/APITestCliRemote.cc                     |    2 +-
 mgmt/api/CoreAPI.cc                              |   23 +-
 mgmt/api/CoreAPI.h                               |    2 +-
 mgmt/api/CoreAPIRemote.cc                        |    4 +-
 mgmt/api/INKMgmtAPI.cc                           |    4 +-
 mgmt/api/TSControlMain.cc                        |    2 +-
 mgmt/api/include/mgmtapi.h                       |    3 +-
 mgmt/cluster/ClusterCom.cc                       | 2414 -----------------
 mgmt/cluster/ClusterCom.h                        |  194 --
 mgmt/cluster/Makefile.am                         |   38 -
 mgmt/cluster/VMap.cc                             |  863 ------
 mgmt/cluster/VMap.h                              |  121 -
 plugins/experimental/ts_lua/ts_lua_http_config.c |    2 -
 proxy/EventName.cc                               |   13 -
 proxy/IPAllow.cc                                 |    1 +
 proxy/InkAPI.cc                                  |   11 -
 proxy/InkAPITest.cc                              |    1 -
 proxy/Main.cc                                    |   27 -
 proxy/Makefile.am                                |    2 -
 77 files changed, 47 insertions(+), 21289 deletions(-)

diff --git a/cmd/traffic_cop/traffic_cop.cc b/cmd/traffic_cop/traffic_cop.cc
index 871c7f6..193b6c9 100644
--- a/cmd/traffic_cop/traffic_cop.cc
+++ b/cmd/traffic_cop/traffic_cop.cc
@@ -33,7 +33,6 @@
 #include "I_RecCore.h"
 #include "mgmtapi.h"
 #include "RecordsConfig.h"
-#include "ClusterCom.h"
 #include "ts/ink_cap.h"
 #include "Cop.h"
 
@@ -98,10 +97,8 @@ static char server_binary[PATH_NAME_MAX]  = "traffic_server";
 
 static char log_file[PATH_NAME_MAX] = "traffic.out";
 
-static int synthetic_port           = 8083;
-static int rs_port                  = 8088;
-static MgmtClusterType cluster_type = NO_CLUSTER;
-static int http_backdoor_port       = 8084;
+static int synthetic_port     = 8083;
+static int http_backdoor_port = 8084;
 
 #if defined(linux)
 // TS-1075 : auto-port ::connect DoS on high traffic linux systems
@@ -658,7 +655,6 @@ config_reload_records()
 
   config_read_int("proxy.config.process_manager.mgmt_port", &http_backdoor_port, true);
   config_read_int("proxy.config.admin.synthetic_port", &synthetic_port, true);
-  config_read_int("proxy.config.cluster.rsport", &rs_port, true);
   config_read_int("proxy.config.cop.init_sleep_time", &init_sleep_time, true);
 
   config_read_int("proxy.config.cop.active_health_checks", &tmp_int, true);
@@ -1027,112 +1023,6 @@ error:
 }
 
 static int
-read_manager_string(const char *variable, char *value, size_t val_len)
-{
-  char buffer[4096];
-  char request[1024];
-  char *p, *e;
-  int err;
-
-  snprintf(request, sizeof(request), "read %s\n", variable);
-
-  err = test_port(rs_port, request, buffer, 4095, cop_manager_timeout * 1000);
-  if (err < 0) {
-    return err;
-  }
-
-  p = strstr(buffer, variable);
-  if (!p) {
-    cop_log(COP_WARNING, "(manager test) could not find record name in response\n");
-    return -1;
-  }
-  p += strlen(variable);
-
-  p = strstr(p, "Val:");
-  if (!p) {
-    cop_log(COP_WARNING, "(manager test) could not find record value in response\n");
-    return -1;
-  }
-  p += sizeof("Val:") - 1;
-
-  while (*p && (*p != '\'')) {
-    p += 1;
-  }
-
-  if (*p == '\0') {
-    cop_log(COP_WARNING, "(manager test) could not find properly delimited value in response\n");
-    return -1;
-  }
-  p += 1;
-
-  e = p;
-  while (*e && (*e != '\'')) {
-    e += 1;
-  }
-
-  if (*e != '\'') {
-    cop_log(COP_WARNING, "(manager test) could not find properly delimited value in response\n");
-    return -1;
-  }
-
-  ink_strlcpy(value, p, MIN((size_t)(e - p + 1), val_len));
-
-  return 0;
-}
-
-static int
-read_manager_int(const char *variable, int *value)
-{
-  char buffer[4096];
-  char request[1024];
-  char *p;
-  int err;
-
-  snprintf(request, sizeof(request), "read %s\n", variable);
-
-  err = test_port(rs_port, request, buffer, 4095, cop_manager_timeout * 1000);
-  if (err < 0) {
-    return err;
-  }
-
-  p = strstr(buffer, variable);
-  if (!p) {
-    cop_log(COP_WARNING, "(manager test) could not find record name in response\n");
-    return -1;
-  }
-  p += strlen(variable);
-
-  p = strstr(p, "Val:");
-  if (!p) {
-    cop_log(COP_WARNING, "(manager test) could not find record value in response\n");
-    return -1;
-  }
-  p += sizeof("Val:") - 1;
-
-  while (*p && (*p != '\'')) {
-    p += 1;
-  }
-
-  if (*p == '\0') {
-    cop_log(COP_WARNING, "(manager test) could not find properly delimited value in response\n");
-    return -1;
-  }
-  p += 1;
-
-  *value = 0;
-  while (isdigit(*p)) {
-    *value = *value * 10 + (*p - '0');
-    p += 1;
-  }
-
-  if (*p != '\'') {
-    cop_log(COP_WARNING, "(manager test) could not find properly delimited value in response\n");
-    return -1;
-  }
-  return 0;
-}
-
-static int
 read_mgmt_cli_int(const char *variable, int *value)
 {
   TSInt val;
@@ -1146,25 +1036,6 @@ read_mgmt_cli_int(const char *variable, int *value)
 }
 
 static int
-test_rs_port()
-{
-  char buffer[4096];
-  int err;
-
-  err = read_manager_string("proxy.config.manager_binary", buffer, sizeof(buffer));
-  if (err < 0) {
-    return err;
-  }
-
-  if (strcmp(buffer, manager_binary) != 0) {
-    cop_log(COP_WARNING, "(manager test) bad response value\n");
-    return -1;
-  }
-
-  return 0;
-}
-
-static int
 test_mgmt_cli_port()
 {
   TSString val = nullptr;
@@ -1262,11 +1133,8 @@ heartbeat_manager()
   int err;
 
   cop_log_trace("Entering heartbeat_manager()\n");
-  // the CLI, and the rsport if cluster is enabled.
+  // the CLI.
   err = test_mgmt_cli_port();
-  if ((0 == err) && (cluster_type != NO_CLUSTER)) {
-    err = test_rs_port();
-  }
 
   if (err < 0) {
     // See heartbeat_server()'s comments for how we determine a server/manager failure.
@@ -1350,11 +1218,7 @@ server_up()
   int err;
 
   cop_log_trace("Entering server_up()\n");
-  if (cluster_type != NO_CLUSTER) {
-    err = read_manager_int("proxy.node.proxy_running", &val);
-  } else {
-    err = read_mgmt_cli_int("proxy.node.proxy_running", &val);
-  }
+  err = read_mgmt_cli_int("proxy.node.proxy_running", &val);
 
   if (err < 0) {
     cop_log(COP_WARNING, "could not contact manager, assuming server is down\n");
diff --git a/cmd/traffic_ctl/config.cc b/cmd/traffic_ctl/config.cc
index 5d4c3b0..660174b 100644
--- a/cmd/traffic_ctl/config.cc
+++ b/cmd/traffic_ctl/config.cc
@@ -82,8 +82,6 @@ rec_classof(int rec_class)
     return "process metric";
   case RECT_NODE:
     return "node metric";
-  case RECT_CLUSTER:
-    return "cluster metric";
   case RECT_PLUGIN:
     return "plugin metric";
   default:
diff --git a/cmd/traffic_ctl/metric.cc b/cmd/traffic_ctl/metric.cc
index bee3356..d207c3f 100644
--- a/cmd/traffic_ctl/metric.cc
+++ b/cmd/traffic_ctl/metric.cc
@@ -80,20 +80,11 @@ metric_match(unsigned argc, const char **argv)
 static int
 metric_clear(unsigned argc, const char **argv)
 {
-  int cluster = 0;
   TSMgmtError error;
 
-  const ArgumentDescription opts[] = {
-    {"cluster", '-', "Clear cluster metrics", "F", &cluster, nullptr, nullptr},
-  };
-
-  if (!CtrlProcessArguments(argc, argv, opts, countof(opts)) || n_file_arguments != 0) {
-    return CtrlCommandUsage("metric clear [OPTIONS]", opts, countof(opts));
-  }
-
-  error = TSStatsReset(cluster, nullptr);
+  error = TSStatsReset(nullptr);
   if (error != TS_ERR_OKAY) {
-    CtrlMgmtError(error, "failed to clear %smetrics", cluster ? "cluster " : "");
+    CtrlMgmtError(error, "failed to clear metrics");
     return CTRL_EX_ERROR;
   }
 
@@ -103,19 +94,10 @@ metric_clear(unsigned argc, const char **argv)
 static int
 metric_zero(unsigned argc, const char **argv)
 {
-  int cluster = 0;
   TSMgmtError error;
 
-  const ArgumentDescription opts[] = {
-    {"cluster", '-', "Zero cluster metrics", "F", &cluster, nullptr, nullptr},
-  };
-
-  if (!CtrlProcessArguments(argc, argv, opts, countof(opts)) || n_file_arguments < 1) {
-    return CtrlCommandUsage("metric zero [OPTIONS] METRIC [METRIC ...]", opts, countof(opts));
-  }
-
   for (unsigned i = 0; i < n_file_arguments; ++i) {
-    error = TSStatsReset(cluster, file_arguments[i]);
+    error = TSStatsReset(file_arguments[i]);
     if (error != TS_ERR_OKAY) {
       CtrlMgmtError(error, "failed to clear %s", file_arguments[i]);
       return CTRL_EX_ERROR;
diff --git a/cmd/traffic_manager/Makefile.am b/cmd/traffic_manager/Makefile.am
index 084486e..d9c11ea 100644
--- a/cmd/traffic_manager/Makefile.am
+++ b/cmd/traffic_manager/Makefile.am
@@ -42,8 +42,6 @@ traffic_manager_SOURCES = \
   AddConfigFilesHere.cc \
   MgmtHandlers.cc \
   MgmtHandlers.h \
-  WebOverview.cc \
-  WebOverview.h \
   metrics.h \
   traffic_manager.cc
 
diff --git a/cmd/traffic_manager/WebOverview.cc b/cmd/traffic_manager/WebOverview.cc
deleted file mode 100644
index 21433f5..0000000
--- a/cmd/traffic_manager/WebOverview.cc
+++ /dev/null
@@ -1,638 +0,0 @@
-/** @file
-
-  A brief file description
-
-  @section license License
-
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
- */
-
-/****************************************************************************
- *
- *  WebOverview.cc - code to overview page
- *
- *
- ****************************************************************************/
-
-#include "ts/ink_platform.h"
-#include "ts/ink_string.h"
-#include "MgmtDefs.h"
-#include "WebOverview.h"
-#include "WebMgmtUtils.h"
-
-#include "LocalManager.h"
-#include "ClusterCom.h"
-#include "MgmtUtils.h"
-#include "MgmtDefs.h"
-#include "ts/Diags.h"
-
-// Make this pointer to avoid nasty destruction
-//   problems do to alarm
-//   fork, execl, exit squences
-overviewPage *overviewGenerator;
-
-overviewRecord::overviewRecord(unsigned long inet_addr, bool local, ClusterPeerInfo *cpi)
-{
-  char *name_l; // hostname looked up from node record
-  bool name_found;
-  struct in_addr nameFailed;
-
-  inetAddr = inet_addr;
-
-  this->up        = false;
-  this->localNode = local;
-
-  // If this is the local node, there is no cluster peer info
-  //   record.  Remote nodes require a cluster peer info record
-  ink_assert((local == false && cpi != nullptr) || (local == true && cpi == nullptr));
-
-  // Set up the copy of the records array and initialize it
-  if (local == true) {
-    node_rec_data.num_recs = 0;
-    node_rec_data.recs     = nullptr;
-    recordArraySize        = 0;
-    node_rec_first_ix      = 0;
-  } else {
-    node_rec_data.num_recs = cpi->node_rec_data.num_recs;
-    recordArraySize        = node_rec_data.num_recs * sizeof(RecRecord);
-    node_rec_data.recs     = new RecRecord[recordArraySize];
-    memcpy(node_rec_data.recs, cpi->node_rec_data.recs, recordArraySize);
-
-    // Recaculate the old relative index
-    RecGetRecordOrderAndId(node_rec_data.recs[0].name, &node_rec_first_ix, nullptr);
-  }
-
-  // Query for the name of the node.  If it is not there, some
-  //   their cluster ip address
-  name_l = this->readString("proxy.node.hostname_FQ", &name_found);
-  if (name_found == false || name_l == nullptr) {
-    nameFailed.s_addr = inetAddr;
-    mgmt_log("[overviewRecord::overviewRecord] Unable to find hostname for %s\n", inet_ntoa(nameFailed));
-    ats_free(name_l); // about to overwrite name_l, so we need to free it first
-    name_l = ats_strdup(inet_ntoa(nameFailed));
-  }
-
-  const size_t hostNameLen = strlen(name_l) + 1;
-  this->hostname           = new char[hostNameLen];
-  ink_strlcpy(this->hostname, name_l, hostNameLen);
-  ats_free(name_l);
-}
-
-overviewRecord::~overviewRecord()
-{
-  delete[] hostname;
-
-  if (localNode == false) {
-    delete[] node_rec_data.recs;
-  }
-}
-
-// void overviewRecord::updateStatus(time_t, ClusterPeerInfo*)
-// updates up/down status based on the cluster peer info record
-//
-//   currentTime is the value of localtime(time()) - sent in as
-//     a parameter so we do not have to make repetitive system calls.
-//     overviewPage::checkForUpdates can just make one call
-//
-//   cpi - is a pointer to a structure we got from ClusterCom that represnets
-//         information about this node
-//
-//   a machine is up if we have heard from it in the last 15 seconds
-//
-void
-overviewRecord::updateStatus(time_t currentTime, ClusterPeerInfo *cpi)
-{
-  // Update if the node is up or down
-  if (currentTime - cpi->idle_ticks > 15) {
-    up = false;
-  } else {
-    up = true;
-  }
-
-  // Update the node records by copying them from cpi
-  //  (remote nodes only)
-  if (localNode == false) {
-    memcpy(node_rec_data.recs, cpi->node_rec_data.recs, recordArraySize);
-    RecGetRecordOrderAndId(node_rec_data.recs[0].name, &node_rec_first_ix, nullptr);
-  }
-}
-
-// overview::readInteger
-//
-//  Accessor functions for node records.  For remote node,
-//    we get the value in the node_data array we maintain
-//    in this object.  For the node, we do not maintain any data
-//    and rely on lmgmt->record_data for both the retrieval
-//    code and the records array
-//
-//  Locking should be done by overviewPage::accessLock.
-//  CALLEE is responsible for obtaining and releasing the lock
-//
-RecInt
-overviewRecord::readInteger(const char *name, bool *found)
-{
-  RecInt rec     = 0;
-  int rec_status = REC_ERR_OKAY;
-  int order      = -1;
-  if (localNode == false) {
-    rec_status = RecGetRecordOrderAndId(name, &order, nullptr);
-    if (rec_status == REC_ERR_OKAY) {
-      order -= node_rec_first_ix; // Offset
-      ink_release_assert(order < node_rec_data.num_recs);
-      ink_assert(order < node_rec_data.num_recs);
-      rec = node_rec_data.recs[order].data.rec_int;
-    }
-  } else {
-    rec_status = RecGetRecordInt(name, &rec);
-  }
-
-  if (found) {
-    *found = (rec_status == REC_ERR_OKAY);
-  } else {
-    mgmt_log("node variables '%s' not found!\n");
-  }
-  return rec;
-}
-
-RecFloat
-overviewRecord::readFloat(const char *name, bool *found)
-{
-  RecFloat rec   = 0.0;
-  int rec_status = REC_ERR_OKAY;
-  int order      = -1;
-  if (localNode == false) {
-    rec_status = RecGetRecordOrderAndId(name, &order, nullptr);
-    if (rec_status == REC_ERR_OKAY) {
-      order -= node_rec_first_ix; // Offset
-      ink_release_assert(order < node_rec_data.num_recs);
-      ink_assert(order < node_rec_data.num_recs);
-      rec = node_rec_data.recs[order].data.rec_float;
-    }
-  } else {
-    rec_status = RecGetRecordFloat(name, &rec);
-  }
-
-  if (found) {
-    *found = (rec_status == REC_ERR_OKAY);
-  } else {
-    mgmt_log("node variables '%s' not found!\n");
-  }
-  return rec;
-}
-
-RecString
-overviewRecord::readString(const char *name, bool *found)
-{
-  RecString rec  = nullptr;
-  int rec_status = REC_ERR_OKAY;
-  int order      = -1;
-  if (localNode == false) {
-    rec_status = RecGetRecordOrderAndId(name, &order, nullptr);
-    if (rec_status == REC_ERR_OKAY) {
-      order -= node_rec_first_ix; // Offset
-      ink_release_assert(order < node_rec_data.num_recs);
-      ink_assert(order < node_rec_data.num_recs);
-      rec = ats_strdup(node_rec_data.recs[order].data.rec_string);
-    }
-  } else {
-    rec_status = RecGetRecordString_Xmalloc(name, &rec);
-  }
-
-  if (found) {
-    *found = (rec_status == REC_ERR_OKAY);
-  } else {
-    mgmt_log("node variables '%s' not found!\n");
-  }
-  return rec;
-}
-
-//  overview::readData, read RecData according varType
-//
-//  Accessor functions for node records.  For remote node,
-//    we get the value in the node_data array we maintain
-//    in this object.  For the node, we do not maintain any data
-//    and rely on lmgmt->record_data for both the retrieval
-//    code and the records array
-//
-//  Locking should be done by overviewPage::accessLock.
-//  CALLEE is responsible for obtaining and releasing the lock
-//
-RecData
-overviewRecord::readData(RecDataT varType, const char *name, bool *found)
-{
-  int rec_status = REC_ERR_OKAY;
-  int order      = -1;
-  RecData rec;
-  RecDataZero(RECD_NULL, &rec);
-
-  if (localNode == false) {
-    rec_status = RecGetRecordOrderAndId(name, &order, nullptr);
-    if (rec_status == REC_ERR_OKAY) {
-      order -= node_rec_first_ix; // Offset
-      ink_release_assert(order < node_rec_data.num_recs);
-      ink_assert(order < node_rec_data.num_recs);
-      RecDataSet(varType, &rec, &node_rec_data.recs[order].data);
-    } else {
-      Fatal("node variables '%s' not found!\n", name);
-    }
-  } else {
-    rec_status = RecGetRecord_Xmalloc(name, varType, &rec, true);
-  }
-
-  if (found) {
-    *found = (rec_status == REC_ERR_OKAY);
-  } else {
-    mgmt_log("node variables '%s' not found!\n");
-  }
-  return rec;
-}
-
-bool
-overviewRecord::varFloatFromName(const char *name, MgmtFloat *value)
-{
-  bool found = false;
-
-  if (value) {
-    *value = readFloat((char *)name, &found);
-  }
-
-  return found;
-}
-
-overviewPage::overviewPage() : sortRecords(10, false)
-{
-  ink_mutex_init(&accessLock, "overviewRecord");
-  nodeRecords = ink_hash_table_create(InkHashTableKeyType_Word);
-  numHosts    = 0;
-  ourAddr     = 0; // We will update this when we add the record for
-  //  this machine
-}
-
-overviewPage::~overviewPage()
-{
-  // Since we only have one global object and we never destruct it
-  //  do not actually free memeory since it causes problems the
-  //  process is vforked, and the child execs something
-  // The below code is DELIBERTLY commented out
-  //
-  // ink_mutex_destroy(&accessLock);
-  // ink_hash_table_destroy(nodeRecords);
-}
-
-// overviewPage::checkForUpdates - updates node records as to whether peers
-//    are up or down
-void
-overviewPage::checkForUpdates()
-{
-  ClusterPeerInfo *tmp;
-  InkHashTableEntry *entry;
-  InkHashTableIteratorState iterator_state;
-  overviewRecord *current;
-  time_t currentTime;
-  bool newHostAdded = false;
-
-  // grok through the cluster communication stuff and update information
-  //  about hosts in the cluster
-  //
-  ink_mutex_acquire(&accessLock);
-  ink_mutex_acquire(&(lmgmt->ccom->mutex));
-  currentTime = time(nullptr);
-  for (entry = ink_hash_table_iterator_first(lmgmt->ccom->peers, &iterator_state); entry != nullptr;
-       entry = ink_hash_table_iterator_next(lmgmt->ccom->peers, &iterator_state)) {
-    tmp = (ClusterPeerInfo *)ink_hash_table_entry_value(lmgmt->ccom->peers, entry);
-
-    if (ink_hash_table_lookup(nodeRecords, (InkHashTableKey)tmp->inet_address, (InkHashTableValue *)&current) == 0) {
-      this->addRecord(tmp);
-      newHostAdded = true;
-    } else {
-      current->updateStatus(currentTime, tmp);
-    }
-  }
-  ink_mutex_release(&lmgmt->ccom->mutex);
-
-  // If we added a new host we must resort sortRecords
-  if (newHostAdded) {
-    this->sortHosts();
-  }
-
-  ink_mutex_release(&accessLock);
-}
-
-// overrviewPage::sortHosts()
-//
-// resorts sortRecords, but always leaves the local node
-//   as the first record
-//
-// accessLock must be held by callee
-void
-overviewPage::sortHosts()
-{
-  void **array = sortRecords.getArray();
-
-  qsort(array + 1, numHosts - 1, sizeof(void *), hostSortFunc);
-}
-
-// overviewPage::addRecord(ClusterPerrInfo* cpi)
-//   Adds a new node record
-//   Assuems that this->accessLock is already held
-//
-void
-overviewPage::addRecord(ClusterPeerInfo *cpi)
-{
-  overviewRecord *newRec;
-
-  ink_assert(cpi != nullptr);
-
-  newRec = new overviewRecord(cpi->inet_address, false, cpi);
-  newRec->updateStatus(time(nullptr), cpi);
-
-  ink_hash_table_insert(nodeRecords, (InkHashTableKey)cpi->inet_address, (InkHashTableEntry *)newRec);
-
-  sortRecords.addEntry(newRec);
-  numHosts++;
-}
-
-// adds a record to nodeRecords for the local machine.
-//   gets IP addr from lmgmt->ccom so cluster communtication
-//   must be intialized before calling this function
-//
-//
-void
-overviewPage::addSelfRecord()
-{
-  overviewRecord *newRec;
-
-  ink_mutex_acquire(&accessLock);
-
-  // We should not have been called before
-  ink_assert(ourAddr == 0);
-
-  // Find out what our cluster addr is from
-  //   from cluster com
-  this->ourAddr = lmgmt->ccom->getIP();
-
-  newRec     = new overviewRecord(ourAddr, true);
-  newRec->up = true;
-
-  ink_hash_table_insert(nodeRecords, (InkHashTableKey)this->ourAddr, (InkHashTableEntry *)newRec);
-
-  sortRecords.addEntry(newRec);
-  numHosts++;
-  ink_mutex_release(&accessLock);
-}
-
-// overviewRecord* overviewPage::findNodeByName(const char* nodeName)
-//
-//   Returns a pointer to node name nodeName
-//     If node name is not found, returns nullptr
-//
-//   CALLEE MUST BE HOLDING this->accessLock
-//
-overviewRecord *
-overviewPage::findNodeByName(const char *nodeName)
-{
-  overviewRecord *current = nullptr;
-  bool nodeFound          = false;
-
-  // Do a linear search of the nodes for this nodeName.
-  //   Yes, I know this is slow but the current word is ten
-  //   nodes would be a huge cluster so this should not
-  //   be a problem
-  //
-  for (int i = 0; i < numHosts; i++) {
-    current = (overviewRecord *)sortRecords[i];
-    if (strcmp(nodeName, current->hostname) == 0) {
-      nodeFound = true;
-      break;
-    }
-  }
-
-  if (nodeFound == true) {
-    return current;
-  } else {
-    return nullptr;
-  }
-}
-
-// MgmtString overviewPage::readString(const char* nodeName, char* *name, bool *found = nullptr)
-//
-//   Looks up a node record for a specific by nodeName
-//    CALLEE deallocates the string with free()
-//
-MgmtString
-overviewPage::readString(const char *nodeName, const char *name, bool *found)
-{
-  MgmtString r = nullptr;
-  //  bool nodeFound = false;
-  bool valueFound = false;
-  overviewRecord *node;
-
-  ink_mutex_acquire(&accessLock);
-
-  node = this->findNodeByName(nodeName);
-
-  if (node != nullptr) {
-    r = node->readString(name, &valueFound);
-  }
-  ink_mutex_release(&accessLock);
-
-  if (found != nullptr) {
-    *found = valueFound;
-  }
-
-  return r;
-}
-
-// MgmtInt overviewPage::readInteger(const char* nodeName, char* *name, bool *found = nullptr)
-//
-//   Looks up a node record for a specific by nodeName
-//
-MgmtInt
-overviewPage::readInteger(const char *nodeName, const char *name, bool *found)
-{
-  MgmtInt r = -1;
-  //  bool nodeFound = false;
-  bool valueFound = false;
-  overviewRecord *node;
-
-  ink_mutex_acquire(&accessLock);
-
-  node = this->findNodeByName(nodeName);
-
-  if (node != nullptr) {
-    r = node->readInteger(name, &valueFound);
-  }
-  ink_mutex_release(&accessLock);
-
-  if (found != nullptr) {
-    *found = valueFound;
-  }
-
-  return r;
-}
-
-// MgmtFloat overviewPage::readFloat(const char* nodeName, char* *name, bool *found = nullptr)
-//
-//   Looks up a node record for a specific by nodeName
-//
-RecFloat
-overviewPage::readFloat(const char *nodeName, const char *name, bool *found)
-{
-  RecFloat r = -1.0;
-  //  bool nodeFound = false;
-  bool valueFound = false;
-  overviewRecord *node;
-
-  ink_mutex_acquire(&accessLock);
-
-  node = this->findNodeByName(nodeName);
-
-  if (node != nullptr) {
-    r = node->readFloat(name, &valueFound);
-  }
-  ink_mutex_release(&accessLock);
-
-  if (found != nullptr) {
-    *found = valueFound;
-  }
-
-  return r;
-}
-
-// int overviewPage::clusterSumData(RecDataT varType, const char* nodeVar,
-//                                  RecData* sum)
-//
-//   Sums nodeVar for every up node in the cluster and stores the
-//     sum in *sum.  Returns the number of nodes summed over
-//
-//   CALLEE MUST HOLD this->accessLock
-//
-int
-overviewPage::clusterSumData(RecDataT varType, const char *nodeVar, RecData *sum)
-{
-  int numUsed        = 0;
-  int numHosts_local = sortRecords.getNumEntries();
-  overviewRecord *current;
-  bool found;
-  RecData recTmp;
-
-  ink_assert(sum != nullptr);
-  RecDataZero(varType, sum);
-
-  for (int i = 0; i < numHosts_local; i++) {
-    current = (overviewRecord *)sortRecords[i];
-    if (current->up == true) {
-      numUsed++;
-      recTmp = current->readData(varType, nodeVar, &found);
-      *sum   = RecDataAdd(varType, *sum, recTmp);
-      if (found == false) {
-      }
-    }
-  }
-  return numUsed;
-}
-
-int
-overviewPage::varClusterDataFromName(RecDataT varType, const char *nodeVar, RecData *sum)
-{
-  int status = 0;
-
-  ink_mutex_acquire(&accessLock);
-
-  status = clusterSumData(varType, nodeVar, sum);
-
-  ink_mutex_release(&accessLock);
-
-  return (status);
-}
-
-// Moved from the now removed StatAggregation.cc
-void
-AgFloat_generic_scale_to_int(const char *processVar, const char *nodeVar, double factor)
-{
-  MgmtFloat tmp;
-
-  if (varFloatFromName(processVar, &tmp)) {
-    tmp = tmp * factor;
-    tmp = tmp + 0.5; // round up.
-    varSetInt(nodeVar, (int)tmp);
-  } else {
-    varSetInt(nodeVar, -20);
-  }
-}
-
-// char* overviewPage::resolvePeerHostname(char* peerIP)
-//
-//   A locking interface to overviewPage::resolvePeerHostname_ml
-//
-char *
-overviewPage::resolvePeerHostname(const char *peerIP)
-{
-  char *r;
-
-  ink_mutex_acquire(&accessLock);
-  r = this->resolvePeerHostname_ml(peerIP);
-  ink_mutex_release(&accessLock);
-
-  return r;
-}
-
-// char* overviewPage::resolvePeerHostname_ml(char* peerIP)
-//
-// Resolves the peer the hostname from its IP address
-//   The hostname is resolved by finding the overviewRecord
-//   Associated with the IP address and copying its hostname
-//
-// CALLEE frees storage
-// CALLEE is responsible for locking
-//
-char *
-overviewPage::resolvePeerHostname_ml(const char *peerIP)
-{
-  unsigned long int ipAddr;
-  InkHashTableValue lookup;
-  overviewRecord *peerRecord;
-  char *returnName = nullptr;
-
-  ipAddr = inet_addr(peerIP);
-
-  // Check to see if our address is malformed
-  if ((long int)ipAddr == -1) {
-    return nullptr;
-  }
-
-  if (ink_hash_table_lookup(nodeRecords, (InkHashTableKey)ipAddr, &lookup)) {
-    peerRecord = (overviewRecord *)lookup;
-    returnName = ats_strdup(peerRecord->hostname);
-  }
-
-  return returnName;
-}
-
-// int hostSortFunc(const void* arg1, const void* arg2)
-//
-//   A compare function that we can to qsort that sorts
-//    overviewRecord*
-//
-int
-hostSortFunc(const void *arg1, const void *arg2)
-{
-  overviewRecord *rec1 = (overviewRecord *)*(void **)arg1;
-  overviewRecord *rec2 = (overviewRecord *)*(void **)arg2;
-
-  return strcmp(rec1->hostname, rec2->hostname);
-}
diff --git a/cmd/traffic_manager/WebOverview.h b/cmd/traffic_manager/WebOverview.h
deleted file mode 100644
index 2cbce7a..0000000
--- a/cmd/traffic_manager/WebOverview.h
+++ /dev/null
@@ -1,140 +0,0 @@
-/** @file
-
-  A brief file description
-
-  @section license License
-
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
- */
-
-#ifndef _WEB_OVERVIEW_H_
-#define _WEB_OVERVIEW_H_
-
-#include "ts/ink_hash_table.h"
-#include "ts/ink_mutex.h"
-#include "ts/TextBuffer.h"
-#include "ts/List.h"
-
-#include "ExpandingArray.h"
-#include "ClusterCom.h"
-
-#include "P_RecCore.h"
-
-/****************************************************************************
- *
- *  WebOverview.h - code to overview page
- *
- *
- ****************************************************************************/
-
-//
-//  There is one instance of the class overviewPage in the LocalManger
-//    process.  The overviewPage instance stores a record of type
-//    overviewRecord for each node that has been seen in the cluster.
-//    The node records contain a list of active alarms on that node.
-//
-//  overviewPage is responsible for the synchronization issues for both
-//    it self and all of its overviewRecords.  Whenever updates are made
-//    to instances of either class, overviewPage's accessLock must be held.
-//
-//  Pointers to overviewRecords are stored in overviewPage::nodeRecords
-//    hash table which is indexed on the nodes ip address (as an
-//    unsigned long).  A second mapping is also maintained in sortRecords.
-//
-//
-//  Additional Notes
-//
-//  These classes have expanded over time.  overviewPage and
-//    overviewRecord are now clearing houses of cluster information
-//    for the UI.
-//  To simplify the locking issues, pointer to overviewRecords should
-//    NOT be returned by overviewPage.  overviewRecords are internal
-//    to overviewPage and any data needed from an overviewRecord
-//    should be returned as a copy (or a const ptr)
-//    through an accessor function
-
-// information about a specific node in the cluster
-class overviewRecord
-{
-public:
-  overviewRecord(unsigned long inet_addr, bool local, ClusterPeerInfo *cpi = nullptr);
-
-  ~overviewRecord();
-
-  void updateStatus(time_t currentTime, ClusterPeerInfo *cpi);
-
-  bool up;
-  bool localNode;
-  char *hostname;         // FQ hostname of the node
-  unsigned long inetAddr; // IP address of the node
-  RecInt readInteger(const char *name, bool *found);
-  RecFloat readFloat(const char *name, bool *found);
-  RecString readString(const char *name, bool *found);
-  RecData readData(RecDataT varType, const char *name, bool *found);
-  bool varFloatFromName(const char *varName, RecFloat *value);
-
-private:
-  RecRecords node_rec_data; // a copy from ClusterPeerInfo
-  int recordArraySize;      // the size of node_data.recs
-  int node_rec_first_ix;    // Kludge, but store the first order ix for later use
-  overviewRecord(const overviewRecord &);
-};
-
-// information about the entire cluster
-class overviewPage
-{
-public:
-  overviewPage();
-  ~overviewPage();
-
-  void checkForUpdates();
-  char *resolvePeerHostname(const char *peerIP);
-  char *resolvePeerHostname_ml(const char *peerIP);
-  MgmtInt readInteger(const char *nodeName, const char *name, bool *found = nullptr);
-  MgmtFloat readFloat(const char *nodeName, const char *name, bool *found = nullptr);
-  MgmtString readString(const char *nodeName, const char *name, bool *found = nullptr);
-  void addSelfRecord();
-
-  int varClusterDataFromName(RecDataT varType, const char *nodeVar, RecData *sum);
-
-private:
-  ink_mutex accessLock;
-
-  // Private fcns
-  overviewPage(const overviewPage &);
-  void addRecord(ClusterPeerInfo *cpi);
-  overviewRecord *findNodeByName(const char *nodeName);
-  void addReading(MgmtInt reading, TextBuffer *output, int nDigits, const char **gifs, const char **alts);
-  void addLoadBar(TextBuffer *output, MgmtInt load);
-  void sortHosts();
-  bool moreInfoButton(const char *submission, TextBuffer *output);
-
-  // Private variables
-  InkHashTable *nodeRecords;  // container for overviewRecords
-  unsigned long ourAddr;      // the IP address of this node
-  ExpandingArray sortRecords; // A second, sorted container for nodeRecords
-  int numHosts;               // number of peers we know about including ourself
-
-  int clusterSumData(RecDataT varType, const char *nodeVar, RecData *sum);
-};
-
-extern overviewPage *overviewGenerator; // global handle to overiewPage?
-                                        // defn found in WebOverview.cc
-
-int hostSortFunc(const void *arg1, const void *arg2);
-
-#endif
diff --git a/cmd/traffic_manager/metrics.cc b/cmd/traffic_manager/metrics.cc
index 456f163..988cce7 100644
--- a/cmd/traffic_manager/metrics.cc
+++ b/cmd/traffic_manager/metrics.cc
@@ -33,7 +33,6 @@
 #include "I_RecCore.h"
 #include "MgmtDefs.h"
 #include "MgmtUtils.h"
-#include "WebOverview.h"
 #include "metrics.h"
 
 struct Evaluator {
@@ -222,8 +221,6 @@ metrics_create_record(lua_State *L, RecDataT data_type)
     rec_type = RECT_PROCESS;
   } else if (strncmp(name, "proxy.node.", sizeof("proxy.node.") - 1) == 0) {
     rec_type = RECT_NODE;
-  } else if (strncmp(name, "proxy.cluster.", sizeof("proxy.cluster.") - 1) == 0) {
-    rec_type = RECT_CLUSTER;
   }
 
   // You have to follow the naming convention.
@@ -276,52 +273,6 @@ metrics_create_float(lua_State *L)
   return metrics_create_record(L, RECD_FLOAT);
 }
 
-static int
-metrics_cluster_sum(lua_State *L)
-{
-  const char *rec_name;
-  RecDataT data_type;
-  RecData rec_data;
-
-  // Get the name of the record to sum.
-  rec_name = lua_tostring(L, -1);
-
-  // XXX Check whether we have a cached value for this somewhere ...
-
-  // If not, get the record data type.
-  if (RecGetRecordDataType(rec_name, &data_type) == REC_ERR_FAIL) {
-    luaL_error(L, "unknown metric name '%s'", rec_name);
-  }
-
-  // Sum the cluster value.
-  if (!overviewGenerator->varClusterDataFromName(data_type, rec_name, &rec_data)) {
-    RecDataZero(data_type, &rec_data);
-
-    // If we can't get any cluster data, return nil. This will generally cause the
-    // evaluator to fail, which is handled by logging and ignoring the failure.
-    lua_pushnil(L);
-    return 1;
-  }
-
-  switch (data_type) {
-  case RECD_INT: /* fallthru */
-  case RECD_COUNTER:
-    lua_pushinteger(L, rec_data.rec_int);
-    break;
-  case RECD_FLOAT:
-    lua_pushnumber(L, rec_data.rec_float);
-    break;
-  case RECD_STRING:
-    lua_pushlstring(L, rec_data.rec_string, strlen(rec_data.rec_string));
-    break;
-  default:
-    lua_pushnil(L);
-  }
-
-  // Return 1 value on the stack.
-  return 1;
-}
-
 bool
 metrics_binding_initialize(BindingInstance &binding)
 {
@@ -340,7 +291,6 @@ metrics_binding_initialize(BindingInstance &binding)
   binding.bind_function("integer", metrics_create_integer);
   binding.bind_function("counter", metrics_create_counter);
   binding.bind_function("float", metrics_create_float);
-  binding.bind_function("metrics.cluster.sum", metrics_cluster_sum);
 
   binding.bind_constant("metrics.now.msec", timestamp_now_msec());
   binding.bind_constant("metrics.update.pass", lua_Integer(0));
diff --git a/cmd/traffic_manager/traffic_manager.cc b/cmd/traffic_manager/traffic_manager.cc
index 14fd48b..6e9d982 100644
--- a/cmd/traffic_manager/traffic_manager.cc
+++ b/cmd/traffic_manager/traffic_manager.cc
@@ -29,11 +29,8 @@
 #include "ts/ink_syslog.h"
 
 #include "WebMgmtUtils.h"
-#include "WebOverview.h"
 #include "MgmtUtils.h"
 #include "NetworkUtilsRemote.h"
-#include "ClusterCom.h"
-#include "VMap.h"
 #include "FileManager.h"
 #include "ts/I_Layout.h"
 #include "ts/I_Version.h"
@@ -440,7 +437,6 @@ main(int argc, const char **argv)
   int disable_syslog = false;
   char userToRunAs[MAX_LOGIN + 1];
   RecInt fds_throttle = -1;
-  time_t ticker;
   ink_thread synthThrId;
 
   int binding_version      = 0;
@@ -593,12 +589,6 @@ main(int argc, const char **argv)
   // Find out our hostname so we can use it as part of the initialization
   setHostnameVar();
 
-  // Create the data structure for overview page
-  //   Do this before the rest of the set up since it needs
-  //   to created to handle any alarms thrown by later
-  //   initialization
-  overviewGenerator = new overviewPage();
-
   // Initialize the Config Object bindings before
   //   starting any other threads
   lmgmt->configFiles = configFiles = new FileManager();
@@ -664,14 +654,8 @@ main(int argc, const char **argv)
                "224.0.1.0 - 239.255.255.255");
   }
 
-  /* TODO: Do we really need to init cluster communication? */
-  lmgmt->initCCom(appVersionInfo, configFiles, cluster_mcport, group_addr, cluster_rsport); /* Setup cluster communication */
-
   lmgmt->initMgmtProcessServer(); /* Setup p-to-p process server */
 
-  // Now that we know our cluster ip address, add the
-  //   UI record for this machine
-  overviewGenerator->addSelfRecord();
   lmgmt->listenForProxy();
 
   //
@@ -712,7 +696,6 @@ main(int argc, const char **argv)
   ink_thread_create(ts_ctrl_main, &mgmtapiFD, 0, 0, nullptr);
   ink_thread_create(event_callback_main, &eventapiFD, 0, 0, nullptr);
 
-  ticker = time(nullptr);
   mgmt_log("[TrafficManager] Setup complete\n");
 
   RecRegisterStatInt(RECT_NODE, "proxy.node.config.reconfigure_time", time(nullptr), RECP_NON_PERSISTENT);
@@ -751,24 +734,6 @@ main(int argc, const char **argv)
       mgmt_log("[main] Reading Configuration Files Reread\n");
     }
 
-    lmgmt->ccom->generateClusterDelta();
-
-    if (lmgmt->run_proxy && lmgmt->processRunning()) {
-      lmgmt->ccom->sendSharedData();
-      lmgmt->virt_map->lt_runGambit();
-    } else {
-      if (!lmgmt->run_proxy) { /* Down if we are not going to start another immed. */
-        /* Proxy is not up, so no addrs should be */
-        lmgmt->virt_map->downOurAddrs();
-      }
-
-      /* Proxy is not up, but we should still exchange config and alarm info */
-      lmgmt->ccom->sendSharedData(false);
-    }
-
-    lmgmt->ccom->checkPeers(&ticker);
-    overviewGenerator->checkForUpdates();
-
     metrics_binding_evaluate(*binding);
 
     if (lmgmt->mgmt_shutdown_outstanding != MGMT_PENDING_NONE) {
@@ -982,9 +947,6 @@ fileUpdated(char *fname, bool incVersion)
 
   } else if (strcmp(fname, "ip_allow.config") == 0) {
     lmgmt->signalFileChange("proxy.config.cache.ip_allow.filename");
-  } else if (strcmp(fname, "vaddrs.config") == 0) {
-    mgmt_log("[fileUpdated] vaddrs.config updated\n");
-    lmgmt->virt_map->lt_readAListFile(fname);
 
   } else if (strcmp(fname, "storage.config") == 0) {
     mgmt_log("[fileUpdated] storage.config changed, need restart auto-rebuild mode\n");
diff --git a/configure.ac b/configure.ac
index 86518a4..3c27c3b 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1916,7 +1916,6 @@ AC_CONFIG_FILES([
   iocore/Makefile
   iocore/aio/Makefile
   iocore/cache/Makefile
-  iocore/cluster/Makefile
   iocore/dns/Makefile
   iocore/eventsystem/Makefile
   iocore/hostdb/Makefile
@@ -1936,7 +1935,6 @@ AC_CONFIG_FILES([
   mgmt/Makefile
   mgmt/api/Makefile
   mgmt/api/include/Makefile
-  mgmt/cluster/Makefile
   mgmt/utils/Makefile
   plugins/Makefile
   proxy/Makefile
diff --git a/iocore/Makefile.am b/iocore/Makefile.am
index 3b52276..a459dc9 100644
--- a/iocore/Makefile.am
+++ b/iocore/Makefile.am
@@ -16,7 +16,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-SUBDIRS = eventsystem net aio dns hostdb utils cache cluster
+SUBDIRS = eventsystem net aio dns hostdb utils cache
 
 include $(top_srcdir)/build/tidy.mk
 
diff --git a/iocore/cache/Cache.cc b/iocore/cache/Cache.cc
index 64512b2..e5a9d58 100644
--- a/iocore/cache/Cache.cc
+++ b/iocore/cache/Cache.cc
@@ -248,25 +248,6 @@ cache_stats_bytes_used_cb(const char *name, RecDataT data_type, RecData *data, R
   return 1;
 }
 
-#ifdef CLUSTER_CACHE
-static Action *
-open_read_internal(int opcode, Continuation *cont, MIOBuffer *buf, const HttpCacheKey *key, CacheHTTPHdr *request,
-                   OverridableHttpConfigParams *params, time_t pin_in_cache, CacheFragType frag_type)
-{
-  ClusterMachine *m = cluster_machine_at_depth(cache_hash(key->hash));
-
-  if (m) {
-    return Cluster_read(m, opcode, cont, buf, request, &key->hash, pin_in_cache, frag_type, key->hostname, key->hostlen);
-  } else {
-    if ((opcode == CACHE_OPEN_READ_LONG) || (opcode == CACHE_OPEN_READ_BUFFER_LONG)) {
-      return caches[frag_type]->open_read(cont, &key->hash, request, params, frag_type, key->hostname, key->hostlen);
-    } else {
-      return caches[frag_type]->open_read(cont, &key->hash, frag_type, key->hostname, key->hostlen);
-    }
-  }
-}
-#endif
-
 static int
 validate_rww(int new_value)
 {
@@ -1057,12 +1038,6 @@ CacheProcessor::cacheInitialized()
     CacheProcessor::initialized = CACHE_INITIALIZED;
     CacheProcessor::cache_ready = caches_ready;
     Note("cache enabled");
-#ifdef CLUSTER_CACHE
-    if (!(start_internal_flags & (PROCESSOR_RECONFIGURE | PROCESSOR_CHECK))) {
-      CacheContinuation::init();
-      clusterProcessor.start();
-    }
-#endif
   } else {
     CacheProcessor::initialized = CACHE_INIT_FAILED;
     Note("cache disabled");
@@ -1103,15 +1078,6 @@ Action *
 CacheProcessor::lookup(Continuation *cont, const CacheKey *key, bool cluster_cache_local ATS_UNUSED, bool local_only ATS_UNUSED,
                        CacheFragType frag_type, const char *hostname, int host_len)
 {
-#ifdef CLUSTER_CACHE
-  // Try to send remote, if not possible, handle locally
-  if ((cache_clustering_enabled > 0) && !cluster_cache_local && !local_only) {
-    Action *a = Cluster_lookup(cont, key, frag_type, hostname, host_len);
-    if (a) {
-      return a;
-    }
-  }
-#endif
   return caches[frag_type]->lookup(cont, key, frag_type, hostname, host_len);
 }
 
@@ -1119,16 +1085,6 @@ inkcoreapi Action *
 CacheProcessor::open_read(Continuation *cont, const CacheKey *key, bool cluster_cache_local ATS_UNUSED, CacheFragType frag_type,
                           const char *hostname, int hostlen)
 {
-#ifdef CLUSTER_CACHE
-  if (cache_clustering_enabled > 0 && !cluster_cache_local) {
-    HttpCacheKey hkey;
-    hkey.hash     = *key;
-    hkey.hostname = hostname;
-    hkey.hostlen  = hostlen;
-    return open_read_internal(CACHE_OPEN_READ, cont, (MIOBuffer *)nullptr, &hkey, (CacheHTTPHdr *)nullptr,
-                              (OverridableHttpConfigParams *)nullptr, 0, frag_type);
-  }
-#endif
   return caches[frag_type]->open_read(cont, key, frag_type, hostname, hostlen);
 }
 
@@ -1136,14 +1092,6 @@ inkcoreapi Action *
 CacheProcessor::open_write(Continuation *cont, CacheKey *key, bool cluster_cache_local ATS_UNUSED, CacheFragType frag_type,
                            int expected_size ATS_UNUSED, int options, time_t pin_in_cache, char *hostname, int host_len)
 {
-#ifdef CLUSTER_CACHE
-  if (cache_clustering_enabled > 0 && !cluster_cache_local) {
-    ClusterMachine *m = cluster_machine_at_depth(cache_hash(*key));
-    if (m)
-      return Cluster_write(cont, expected_size, (MIOBuffer *)nullptr, m, key, frag_type, options, pin_in_cache, CACHE_OPEN_WRITE,
-                           (CacheHTTPHdr *)nullptr, (CacheHTTPInfo *)nullptr, hostname, host_len);
-  }
-#endif
   return caches[frag_type]->open_write(cont, key, frag_type, options, pin_in_cache, hostname, host_len);
 }
 
@@ -1152,26 +1100,9 @@ CacheProcessor::remove(Continuation *cont, const CacheKey *key, bool cluster_cac
                        const char *hostname, int host_len)
 {
   Debug("cache_remove", "[CacheProcessor::remove] Issuing cache delete for %u", cache_hash(*key));
-#ifdef CLUSTER_CACHE
-  if (cache_clustering_enabled > 0 && !cluster_cache_local) {
-    ClusterMachine *m = cluster_machine_at_depth(cache_hash(*key));
-
-    if (m) {
-      return Cluster_remove(m, cont, key, frag_type, hostname, host_len);
-    }
-  }
-#endif
   return caches[frag_type]->remove(cont, key, frag_type, hostname, host_len);
 }
 
-#if 0
-Action *
-scan(Continuation *cont, char *hostname = 0, int host_len = 0, int KB_per_second = 2500)
-{
-  return caches[CACHE_FRAG_TYPE_HTTP]->scan(cont, hostname, host_len, KB_per_second);
-}
-#endif
-
 Action *
 CacheProcessor::lookup(Continuation *cont, const HttpCacheKey *key, bool cluster_cache_local, bool local_only,
                        CacheFragType frag_type)
@@ -1179,34 +1110,6 @@ CacheProcessor::lookup(Continuation *cont, const HttpCacheKey *key, bool cluster
   return lookup(cont, &key->hash, cluster_cache_local, local_only, frag_type, key->hostname, key->hostlen);
 }
 
-#ifdef CLUSTER_CACHE
-Action *
-CacheProcessor::link(Continuation *cont, CacheKey *from, CacheKey *to, bool cluster_cache_local, CacheFragType type, char *hostname,
-                     int host_len)
-{
-  if (cache_clustering_enabled > 0 && !cluster_cache_local) {
-    // Use INK_MD5 in "from" to determine target machine
-    ClusterMachine *m = cluster_machine_at_depth(cache_hash(*from));
-    if (m) {
-      return Cluster_link(m, cont, from, to, type, hostname, host_len);
-    }
-  }
-  return caches[type]->link(cont, from, to, type, hostname, host_len);
-}
-
-Action *
-CacheProcessor::deref(Continuation *cont, CacheKey *key, bool cluster_cache_local, CacheFragType type, char *hostname, int host_len)
-{
-  if (cache_clustering_enabled > 0 && !cluster_cache_local) {
-    ClusterMachine *m = cluster_machine_at_depth(cache_hash(*key));
-    if (m) {
-      return Cluster_deref(m, cont, key, type, hostname, host_len);
-    }
-  }
-  return caches[type]->deref(cont, key, type, hostname, host_len);
-}
-#endif
-
 Action *
 CacheProcessor::scan(Continuation *cont, char *hostname, int host_len, int KB_per_second)
 {
@@ -3275,12 +3178,6 @@ Action *
 CacheProcessor::open_read(Continuation *cont, const HttpCacheKey *key, bool cluster_cache_local, CacheHTTPHdr *request,
                           OverridableHttpConfigParams *params, time_t pin_in_cache, CacheFragType type)
 {
-#ifdef CLUSTER_CACHE
-  if (cache_clustering_enabled > 0 && !cluster_cache_local) {
-    return open_read_internal(CACHE_OPEN_READ_LONG, cont, (MIOBuffer *)nullptr, key, request, params, pin_in_cache, type);
-  }
-#endif
-
   return caches[type]->open_read(cont, &key->hash, request, params, type, key->hostname, key->hostlen);
 }
 
@@ -3289,17 +3186,6 @@ Action *
 CacheProcessor::open_write(Continuation *cont, int expected_size, const HttpCacheKey *key, bool cluster_cache_local,
                            CacheHTTPHdr *request, CacheHTTPInfo *old_info, time_t pin_in_cache, CacheFragType type)
 {
-#ifdef CLUSTER_CACHE
-  if (cache_clustering_enabled > 0 && !cluster_cache_local) {
-    ClusterMachine *m = cluster_machine_at_depth(cache_hash(key->hash));
-
-    if (m) {
-      // Do remote open_write()
-      return Cluster_write(cont, expected_size, (MIOBuffer *)nullptr, m, &key->hash, type, false, pin_in_cache,
-                           CACHE_OPEN_WRITE_LONG, request, old_info, key->hostname, key->hostlen);
-    }
-  }
-#endif
   return caches[type]->open_write(cont, &key->hash, old_info, pin_in_cache, nullptr /* key1 */, type, key->hostname, key->hostlen);
 }
 
@@ -3309,13 +3195,6 @@ CacheProcessor::open_write(Continuation *cont, int expected_size, const HttpCach
 Action *
 CacheProcessor::remove(Continuation *cont, const HttpCacheKey *key, bool cluster_cache_local, CacheFragType frag_type)
 {
-#ifdef CLUSTER_CACHE
-  if (cache_clustering_enabled > 0 && !cluster_cache_local) {
-    // Remove from cluster
-    return remove(cont, &key->hash, cluster_cache_local, frag_type, key->hostname, key->hostlen);
-  }
-#endif
-
   // Remove from local cache only.
   return caches[frag_type]->remove(cont, &key->hash, frag_type, key->hostname, key->hostlen);
 }
diff --git a/iocore/cache/CacheDir.cc b/iocore/cache/CacheDir.cc
index 7e058fd..5abeeee 100644
--- a/iocore/cache/CacheDir.cc
+++ b/iocore/cache/CacheDir.cc
@@ -24,6 +24,7 @@
 #include "P_Cache.h"
 
 #include "ts/hugepages.h"
+#include "ts/Regression.h"
 
 // #define LOOP_CHECK_MODE 1
 #ifdef LOOP_CHECK_MODE
diff --git a/iocore/cache/CacheHosting.cc b/iocore/cache/CacheHosting.cc
index b18b287..5eb45b0 100644
--- a/iocore/cache/CacheHosting.cc
+++ b/iocore/cache/CacheHosting.cc
@@ -25,6 +25,7 @@
 #include "ts/I_Layout.h"
 #include "ts/HostLookup.h"
 #include "ts/Tokenizer.h"
+#include "ts/Regression.h"
 
 extern int gndisks;
 
diff --git a/iocore/cache/P_CacheHosting.h b/iocore/cache/P_CacheHosting.h
index 075c50d..7902682 100644
--- a/iocore/cache/P_CacheHosting.h
+++ b/iocore/cache/P_CacheHosting.h
@@ -24,6 +24,8 @@
 #ifndef __P_CACHE_HOSTING_H__
 #define __P_CACHE_HOSTING_H__
 #include "P_Cache.h"
+#include "ts/MatcherUtils.h"
+#include "ts/HostLookup.h"
 
 #define CACHE_MEM_FREE_TIMEOUT HRTIME_SECONDS(1)
 
diff --git a/iocore/cache/P_CacheInternal.h b/iocore/cache/P_CacheInternal.h
index 2d852bc..8f860a0 100644
--- a/iocore/cache/P_CacheInternal.h
+++ b/iocore/cache/P_CacheInternal.h
@@ -1021,16 +1021,6 @@ cache_hash(const INK_MD5 &md5)
   return mhash;
 }
 
-#define CLUSTER_CACHE
-
-#ifdef CLUSTER_CACHE
-#include "P_Net.h"
-#include "P_ClusterInternal.h"
-// Note: This include must occur here in order to avoid numerous forward
-//       reference problems.
-#include "P_ClusterInline.h"
-#endif
-
 LINK_DEFINITION(CacheVC, opendir_link)
 
 #endif /* _P_CACHE_INTERNAL_H__ */
diff --git a/iocore/cluster/ClusterAPI.cc b/iocore/cluster/ClusterAPI.cc
deleted file mode 100644
index 8d5d5a8..0000000
--- a/iocore/cluster/ClusterAPI.cc
+++ /dev/null
@@ -1,560 +0,0 @@
-/** @file
-
-  A brief file description
-
-  @section license License
-
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
- */
-
-/****************************************************************************
-
-  ClusterAPI.cc
-
-        Support for Cluster RPC API.
-****************************************************************************/
-#include "P_Cluster.h"
-
-#include "InkAPIInternal.h"
-
-class ClusterAPIPeriodicSM;
-static void send_machine_online_list(TSClusterStatusHandle_t *);
-
-typedef struct node_callout_entry {
-  Ptr<ProxyMutex> mutex;
-  TSClusterStatusFunction func;
-  int state; // See NE_STATE_XXX defines
-} node_callout_entry_t;
-
-#define NE_STATE_FREE 0
-#define NE_STATE_INITIALIZED 1
-
-#define MAX_CLUSTERSTATUS_CALLOUTS 32
-
-static ProxyMutex *ClusterAPI_mutex;
-static ClusterAPIPeriodicSM *periodicSM;
-
-static node_callout_entry_t status_callouts[MAX_CLUSTERSTATUS_CALLOUTS];
-static TSClusterRPCFunction RPC_Functions[API_END_CLUSTER_FUNCTION];
-
-#define INDEX_TO_CLUSTER_STATUS_HANDLE(i) ((TSClusterStatusHandle_t)((i)))
-#define CLUSTER_STATUS_HANDLE_TO_INDEX(h) ((int)((h)))
-#define NODE_HANDLE_TO_IP(h) (*((struct in_addr *)&((h))))
-#define RPC_FUNCTION_KEY_TO_CLUSTER_NUMBER(k) ((int)((k)))
-#define IP_TO_NODE_HANDLE(ip) ((TSNodeHandle_t)((ip)))
-#define SIZEOF_RPC_MSG_LESS_DATA (sizeof(TSClusterRPCMsg_t) - (sizeof(TSClusterRPCMsg_t) - sizeof(TSClusterRPCHandle_t)))
-
-typedef struct RPCHandle {
-  union { // Note: All union elements are assumed to be the same size
-    //       sizeof(u.internal) == sizeof(u.external)
-    TSClusterRPCHandle_t external;
-    struct real_format {
-      int cluster_function;
-      int magic;
-    } internal;
-  } u;
-} RPCHandle_t;
-
-#define RPC_HANDLE_MAGIC 0x12345678
-
-class MachineStatusSM;
-typedef int (MachineStatusSM::*MachineStatusSMHandler)(int, void *);
-class MachineStatusSM : public Continuation
-{
-public:
-  // Broadcast constructor
-  MachineStatusSM(TSNodeHandle_t h, TSNodeStatus_t s)
-    : _node_handle(h), _node_status(s), _status_handle(0), _broadcast(1), _restart(0), _next_n(0)
-  {
-    SET_HANDLER((MachineStatusSMHandler)&MachineStatusSM::MachineStatusSMEvent);
-  }
-  // Unicast constructor
-  MachineStatusSM(TSNodeHandle_t h, TSNodeStatus_t s, TSClusterStatusHandle_t sh)
-    : _node_handle(h), _node_status(s), _status_handle(sh), _broadcast(0), _restart(0), _next_n(0)
-  {
-    SET_HANDLER((MachineStatusSMHandler)&MachineStatusSM::MachineStatusSMEvent);
-  }
-  // Send machine online list constructor
-  MachineStatusSM(TSClusterStatusHandle_t sh)
-    : _node_handle(0), _node_status(NODE_ONLINE), _status_handle(sh), _broadcast(0), _restart(0), _next_n(0)
-  {
-    SET_HANDLER((MachineStatusSMHandler)&MachineStatusSM::MachineStatusSMEvent);
-  }
-  ~MachineStatusSM() override {}
-  int MachineStatusSMEvent(Event *e, void *d);
-
-private:
-  TSNodeHandle_t _node_handle;
-  TSNodeStatus_t _node_status;
-  TSClusterStatusHandle_t _status_handle; // Valid only if !_broadcast
-  int _broadcast;
-  int _restart;
-  int _next_n;
-};
-
-int
-MachineStatusSM::MachineStatusSMEvent(Event * /* e ATS_UNUSED */, void * /* d ATS_UNUSED */)
-{
-  int n;
-  EThread *et = this_ethread();
-
-  if (_broadcast) {
-    /////////////////////////////////////////////////////
-    // Broadcast node transition to all subscribers
-    /////////////////////////////////////////////////////
-    n = _restart ? _next_n : 0;
-    for (; n < MAX_CLUSTERSTATUS_CALLOUTS; ++n) {
-      if (status_callouts[n].func && (status_callouts[n].state == NE_STATE_INITIALIZED)) {
-        MUTEX_TRY_LOCK(lock, status_callouts[n].mutex, et);
-        if (lock.is_locked()) {
-          status_callouts[n].func(&_node_handle, _node_status);
-          Debug("cluster_api", "callout: n %d ([%u.%u.%u.%u], %d)", n, DOT_SEPARATED(_node_handle), _node_status);
-        } else {
-          _restart = 1;
-          _next_n  = n;
-          return EVENT_CONT;
-        }
-      }
-    }
-  } else {
-    if (!_node_handle) {
-      /////////////////////////////////////////////////////
-      // Send online node list to a specific subscriber
-      /////////////////////////////////////////////////////
-      n = CLUSTER_STATUS_HANDLE_TO_INDEX(_status_handle);
-      if (status_callouts[n].func) {
-        MUTEX_TRY_LOCK(lock, status_callouts[n].mutex, et);
-        if (lock.is_locked()) {
-          int mi;
-          unsigned int my_ipaddr = (this_cluster_machine())->ip;
-          ClusterConfiguration *cc;
-
-          TSNodeHandle_t nh;
-
-          cc = this_cluster()->current_configuration();
-          if (cc) {
-            for (mi = 0; mi < cc->n_machines; ++mi) {
-              if (cc->machines[mi]->ip != my_ipaddr) {
-                nh = IP_TO_NODE_HANDLE(cc->machines[mi]->ip);
-                status_callouts[n].func(&nh, NODE_ONLINE);
-
-                Debug("cluster_api", "initial callout: n %d ([%u.%u.%u.%u], %d)", n, DOT_SEPARATED(cc->machines[mi]->ip),
-                      NODE_ONLINE);
-              }
-            }
-          }
-          status_callouts[n].state = NE_STATE_INITIALIZED;
-
-        } else {
-          _restart = 1;
-          _next_n  = n;
-          return EVENT_CONT;
-        }
-      }
-    } else {
-      /////////////////////////////////////////////////////
-      // Send node status to a specific subscriber
-      /////////////////////////////////////////////////////
-      n = CLUSTER_STATUS_HANDLE_TO_INDEX(_status_handle);
-      if (status_callouts[n].func) {
-        MUTEX_TRY_LOCK(lock, status_callouts[n].mutex, et);
-        if (lock.is_locked()) {
-          status_callouts[n].func(&_node_handle, _node_status);
-
-          Debug("cluster_api", "directed callout: n %d ([%u.%u.%u.%u], %d)", n, DOT_SEPARATED(_node_handle), _node_status);
-        } else {
-          _restart = 1;
-          _next_n  = n;
-          return EVENT_CONT;
-        }
-      }
-    }
-  }
-  delete this;
-  return EVENT_DONE;
-}
-
-class ClusterAPIPeriodicSM;
-typedef int (ClusterAPIPeriodicSM::*ClusterAPIPeriodicSMHandler)(int, void *);
-class ClusterAPIPeriodicSM : public Continuation
-{
-public:
-  ClusterAPIPeriodicSM(ProxyMutex *m) : Continuation(m), _active_msmp(nullptr)
-  {
-    SET_HANDLER((ClusterAPIPeriodicSMHandler)&ClusterAPIPeriodicSM::ClusterAPIPeriodicSMEvent);
-  }
-  ~ClusterAPIPeriodicSM() override {}
-  int ClusterAPIPeriodicSMEvent(int, void *);
-  MachineStatusSM *GetNextSM();
-
-private:
-  MachineStatusSM *_active_msmp;
-};
-
-static InkAtomicList status_callout_atomic_q;
-static Queue<MachineStatusSM> status_callout_q;
-
-MachineStatusSM *
-ClusterAPIPeriodicSM::GetNextSM()
-{
-  MachineStatusSM *msmp;
-  MachineStatusSM *msmp_next;
-
-  while (true) {
-    msmp = status_callout_q.pop();
-    if (!msmp) {
-      msmp = (MachineStatusSM *)ink_atomiclist_popall(&status_callout_atomic_q);
-      if (msmp) {
-        while (msmp) {
-          msmp_next       = (MachineStatusSM *)msmp->link.next;
-          msmp->link.next = nullptr;
-          status_callout_q.push(msmp);
-          msmp = msmp_next;
-        }
-        continue;
-      } else {
-        break;
-      }
-    } else {
-      break;
-    }
-  }
-  return msmp;
-}
-
-int
-ClusterAPIPeriodicSM::ClusterAPIPeriodicSMEvent(int e, void *d)
-{
-  // Maintain node status event order by serializing the processing.
-  int ret;
-
-  while (true) {
-    if (_active_msmp) {
-      ret = _active_msmp->handleEvent(e, d);
-      if (ret != EVENT_DONE) {
-        return EVENT_CONT;
-      }
-    }
-    _active_msmp = GetNextSM();
-    if (!_active_msmp) {
-      break;
-    }
-  }
-  return EVENT_CONT;
-}
-
-void
-clusterAPI_init()
-{
-  MachineStatusSM *mssmp = nullptr;
-  ink_atomiclist_init(&status_callout_atomic_q, "cluster API status_callout_q", (char *)&mssmp->link.next - (char *)mssmp);
-  ClusterAPI_mutex = new_ProxyMutex();
-  MUTEX_TRY_LOCK(lock, ClusterAPI_mutex, this_ethread());
-  ink_release_assert(lock.is_locked()); // Should never fail
-  periodicSM = new ClusterAPIPeriodicSM(ClusterAPI_mutex);
-
-  // TODO: Should we do something with this return value?
-  eventProcessor.schedule_every(periodicSM, HRTIME_SECONDS(1), ET_CALL);
-}
-
-/*
- *  Add the given function to the node status callout list which is
- *  invoked on each machine up/down transition.
- *
- *  Note: Using blocking mutex since interface is synchronous and is only
- *	  called at plugin load time.
- */
-int
-TSAddClusterStatusFunction(TSClusterStatusFunction Status_Function, TSMutex m, TSClusterStatusHandle_t *h)
-{
-  Debug("cluster_api", "TSAddClusterStatusFunction func %p", Status_Function);
-  int n;
-  EThread *e = this_ethread();
-
-  ink_release_assert(Status_Function);
-  MUTEX_TAKE_LOCK(ClusterAPI_mutex, e);
-  for (n = 0; n < MAX_CLUSTERSTATUS_CALLOUTS; ++n) {
-    if (!status_callouts[n].func) {
-      status_callouts[n].mutex = (ProxyMutex *)m;
-      status_callouts[n].func  = Status_Function;
-      MUTEX_UNTAKE_LOCK(ClusterAPI_mutex, e);
-      *h = INDEX_TO_CLUSTER_STATUS_HANDLE(n);
-
-      Debug("cluster_api", "TSAddClusterStatusFunction: func %p n %d", Status_Function, n);
-      return 0;
-    }
-  }
-  MUTEX_UNTAKE_LOCK(ClusterAPI_mutex, e);
-  return 1;
-}
-
-/*
- *  Remove the given function from the node status callout list
- *  established via TSAddClusterStatusFunction().
- *
- *  Note: Using blocking mutex since interface is synchronous and is only
- *	  called at plugin unload time (unload currently not supported).
- */
-int
-TSDeleteClusterStatusFunction(TSClusterStatusHandle_t *h)
-{
-  int n      = CLUSTER_STATUS_HANDLE_TO_INDEX(*h);
-  EThread *e = this_ethread();
-
-  ink_release_assert((n >= 0) && (n < MAX_CLUSTERSTATUS_CALLOUTS));
-  Debug("cluster_api", "TSDeleteClusterStatusFunction: n %d", n);
-
-  MUTEX_TAKE_LOCK(ClusterAPI_mutex, e);
-  status_callouts[n].mutex = nullptr;
-  status_callouts[n].func  = (TSClusterStatusFunction) nullptr;
-  status_callouts[n].state = NE_STATE_FREE;
-  MUTEX_UNTAKE_LOCK(ClusterAPI_mutex, e);
-
-  return 0;
-}
-
-int
-TSNodeHandleToIPAddr(TSNodeHandle_t *h, struct in_addr *in)
-{
-  *in = NODE_HANDLE_TO_IP(*h);
-  return 0;
-}
-
-void
-TSGetMyNodeHandle(TSNodeHandle_t *h)
-{
-  *h = IP_TO_NODE_HANDLE((this_cluster_machine())->ip);
-}
-
-/*
- *  Enable node status callouts for the added callout entry.
- *  Issued once after the call to TSAddClusterStatusFunction()
- *  to get the current node configuration.  All subsequent
- *  callouts are updates to the state obtained at this point.
- */
-void
-TSEnableClusterStatusCallout(TSClusterStatusHandle_t *h)
-{
-  int ci = CLUSTER_STATUS_HANDLE_TO_INDEX(*h);
-  // This isn't used.
-  // int my_ipaddr = (this_cluster_machine())->ip;
-  ink_release_assert((ci >= 0) && (ci < MAX_CLUSTERSTATUS_CALLOUTS));
-
-  if (status_callouts[ci].state == NE_STATE_INITIALIZED) {
-    return;
-  }
-
-  Debug("cluster_api", "TSEnableClusterStatusCallout: n %d", ci);
-  send_machine_online_list(h);
-}
-
-static void
-send_machine_online_list(TSClusterStatusHandle_t *h)
-{
-  MachineStatusSM *msm = new MachineStatusSM(*h);
-
-  ink_atomiclist_push(&status_callout_atomic_q, (void *)msm);
-}
-
-/*
- *  Send node online to a specific cluster status entry.
- */
-// This doesn't seem to be used...
-#ifdef NOT_USED_HERE
-static void
-directed_machine_online(int Ipaddr, TSClusterStatusHandle_t *h)
-{
-  MachineStatusSM *msm = new MachineStatusSM(IP_TO_NODE_HANDLE(Ipaddr), NODE_ONLINE, *h);
-
-  ink_atomiclist_push(&status_callout_atomic_q, (void *)msm);
-}
-#endif
-
-/*
- *  Called directly by the Cluster upon detection of node online.
- */
-void
-machine_online_APIcallout(int Ipaddr)
-{
-  MachineStatusSM *msm = new MachineStatusSM(IP_TO_NODE_HANDLE(Ipaddr), NODE_ONLINE);
-
-  ink_atomiclist_push(&status_callout_atomic_q, (void *)msm);
-}
-
-/*
- *  Called directly by the Cluster upon detection of node offline.
- */
-void
-machine_offline_APIcallout(int Ipaddr)
-{
-  MachineStatusSM *msm = new MachineStatusSM(IP_TO_NODE_HANDLE(Ipaddr), NODE_OFFLINE);
-
-  ink_atomiclist_push(&status_callout_atomic_q, (void *)msm);
-}
-
-/*
- *  Associate the given RPC function with the given key.
- *
- *  Note: Using blocking mutex since interface is synchronous and is only
- *	  called at plugin load time.
- */
-int
-TSAddClusterRPCFunction(TSClusterRPCKey_t k, TSClusterRPCFunction func, TSClusterRPCHandle_t *h)
-{
-  RPCHandle_t handle;
-  int n      = RPC_FUNCTION_KEY_TO_CLUSTER_NUMBER(k);
-  EThread *e = this_ethread();
-
-  ink_release_assert(func);
-  ink_release_assert((n >= API_STARECT_CLUSTER_FUNCTION) && (n <= API_END_CLUSTER_FUNCTION));
-  Debug("cluster_api", "TSAddClusterRPCFunction: key %d func %p", k, func);
-
-  handle.u.internal.cluster_function = n;
-  handle.u.internal.magic            = RPC_HANDLE_MAGIC;
-
-  MUTEX_TAKE_LOCK(ClusterAPI_mutex, e);
-  if (n < API_END_CLUSTER_FUNCTION)
-    RPC_Functions[n] = func;
-  MUTEX_UNTAKE_LOCK(ClusterAPI_mutex, e);
-
-  *h = handle.u.external;
-  return 0;
-}
-
-/*
- *  Remove the given RPC function added via TSAddClusterRPCFunction().
- *
- *  Note: Using blocking mutex since interface is synchronous and is only
- *	  called at plugin unload time (unload currently not supported).
- */
-int
-TSDeleteClusterRPCFunction(TSClusterRPCHandle_t *rpch)
-{
-  RPCHandle_t *h = (RPCHandle_t *)rpch;
-  EThread *e     = this_ethread();
-
-  ink_release_assert(((h->u.internal.cluster_function >= API_STARECT_CLUSTER_FUNCTION) &&
-                      (h->u.internal.cluster_function <= API_END_CLUSTER_FUNCTION)));
-  Debug("cluster_api", "TSDeleteClusterRPCFunction: n %d", h->u.internal.cluster_function);
-
-  MUTEX_TAKE_LOCK(ClusterAPI_mutex, e);
-  RPC_Functions[h->u.internal.cluster_function] = nullptr;
-  MUTEX_UNTAKE_LOCK(ClusterAPI_mutex, e);
-  return 0;
-}
-
-/*
- *  Cluster calls us here for each RPC API function.
- */
-void
-default_api_ClusterFunction(ClusterHandler *ch, void *data, int len)
-{
-  Debug("cluster_api", "default_api_ClusterFunction: [%u.%u.%u.%u] data %p len %d", DOT_SEPARATED(ch->machine->ip), data, len);
-
-  TSClusterRPCMsg_t *msg = (TSClusterRPCMsg_t *)data;
-  RPCHandle_t *rpch      = (RPCHandle_t *)&msg->m_handle;
-  int cluster_function   = rpch->u.internal.cluster_function;
-
-  ink_release_assert((size_t)len >= sizeof(TSClusterRPCMsg_t));
-  ink_release_assert(((cluster_function >= API_STARECT_CLUSTER_FUNCTION) && (cluster_function <= API_END_CLUSTER_FUNCTION)));
-
-  if (cluster_function < API_END_CLUSTER_FUNCTION && RPC_Functions[cluster_function]) {
-    int msg_data_len  = len - SIZEOF_RPC_MSG_LESS_DATA;
-    TSNodeHandle_t nh = IP_TO_NODE_HANDLE(ch->machine->ip);
-    (*RPC_Functions[cluster_function])(&nh, msg, msg_data_len);
-  } else {
-    clusterProcessor.free_remote_data((char *)data, len);
-  }
-}
-
-/*
- *  Free TSClusterRPCMsg_t received via the RPC function.
- */
-void
-TSFreeRPCMsg(TSClusterRPCMsg_t *msg, int msg_data_len)
-{
-  RPCHandle_t *rpch = (RPCHandle_t *)&msg->m_handle;
-  ink_release_assert(rpch->u.internal.magic == RPC_HANDLE_MAGIC);
-  Debug("cluster_api", "TSFreeRPCMsg: msg %p msg_data_len %d", msg, msg_data_len);
-
-  clusterProcessor.free_remote_data((char *)msg, msg_data_len + SIZEOF_RPC_MSG_LESS_DATA);
-}
-
-/*
- *  Allocate a message structure for use in the call to TSSendClusterRPC().
- */
-TSClusterRPCMsg_t *
-TSAllocClusterRPCMsg(TSClusterRPCHandle_t *h, int data_size)
-{
-  ink_assert(data_size >= 4);
-  if (data_size < 4) {
-    /* Message must be at least 4 bytes in length */
-    return (TSClusterRPCMsg_t *)nullptr;
-  }
-
-  TSClusterRPCMsg_t *rpcm;
-  OutgoingControl *c = OutgoingControl::alloc();
-
-  c->len = sizeof(OutgoingControl *) + SIZEOF_RPC_MSG_LESS_DATA + data_size;
-  c->alloc_data();
-  *((OutgoingControl **)c->data) = c;
-
-  rpcm           = (TSClusterRPCMsg_t *)(c->data + sizeof(OutgoingControl *));
-  rpcm->m_handle = *h;
-
-  /*
-   * Note: We have carefully constructed TSClusterRPCMsg_t so
-   *       m_data[] is 8 byte aligned.  This allows the user to
-   *       cast m_data[] to any type without any consideration
-   *       for alignment issues.
-   */
-  return rpcm;
-}
-
-/*
- *  Send the given message to the specified node.
- */
-int
-TSSendClusterRPC(TSNodeHandle_t *nh, TSClusterRPCMsg_t *msg)
-{
-  struct in_addr ipaddr = NODE_HANDLE_TO_IP(*nh);
-  RPCHandle_t *rpch     = (RPCHandle_t *)&msg->m_handle;
-
-  OutgoingControl *c       = *((OutgoingControl **)((char *)msg - sizeof(OutgoingControl *)));
-  ClusterConfiguration *cc = this_cluster()->current_configuration();
-  ClusterMachine *m;
-
-  ink_release_assert(rpch->u.internal.magic == RPC_HANDLE_MAGIC);
-
-  if ((m = cc->find(ipaddr.s_addr))) {
-    int len = c->len - sizeof(OutgoingControl *);
-    ink_release_assert((size_t)len >= sizeof(TSClusterRPCMsg_t));
-
-    Debug("cluster_api", "TSSendClusterRPC: msg %p dlen %d [%u.%u.%u.%u] sent", msg, len, DOT_SEPARATED(ipaddr.s_addr));
-    clusterProcessor.invoke_remote(m->pop_ClusterHandler(), rpch->u.internal.cluster_function, msg, len,
-                                   (CLUSTER_OPT_STEAL | CLUSTER_OPT_DATA_IS_OCONTROL));
-  } else {
-    Debug("cluster_api", "TSSendClusterRPC: msg %p to [%u.%u.%u.%u] dropped", msg, DOT_SEPARATED(ipaddr.s_addr));
-    c->freeall();
-  }
-
-  return 0;
-}
-
-/*
- *  End of ClusterAPI.cc
- */
diff --git a/iocore/cluster/ClusterCache.cc b/iocore/cluster/ClusterCache.cc
deleted file mode 100644
index 0a3a816..0000000
--- a/iocore/cluster/ClusterCache.cc
+++ /dev/null
@@ -1,3059 +0,0 @@
-/** @file
-
-  A brief file description
-
-  @section license License
-
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
- */
-
-/****************************************************************************
-
-  ClusterCache.cc
-****************************************************************************/
-
-#include "P_Cluster.h"
-
-#ifdef DEBUG
-#define CLUSTER_TEST_DEBUG 1
-#endif
-
-#ifdef ENABLE_TIME_TRACE
-int callback_time_dist[TIME_DIST_BUCKETS_SIZE];
-int cache_callbacks = 0;
-
-int rmt_callback_time_dist[TIME_DIST_BUCKETS_SIZE];
-int rmt_cache_callbacks = 0;
-
-int lkrmt_callback_time_dist[TIME_DIST_BUCKETS_SIZE];
-int lkrmt_cache_callbacks = 0;
-
-int cntlck_acquire_time_dist[TIME_DIST_BUCKETS_SIZE];
-int cntlck_acquire_events = 0;
-
-int open_delay_time_dist[TIME_DIST_BUCKETS_SIZE];
-int open_delay_events = 0;
-
-#endif // ENABLE_TIME_TRACE
-
-// default will be read from config
-int cache_migrate_on_demand = false;
-
-/////////////////
-// Static Data //
-/////////////////
-static ClassAllocator<CacheContinuation> cacheContAllocator("cacheContAllocator");
-
-static Queue<CacheContinuation> remoteCacheContQueue[REMOTE_CONNECT_HASH];
-static Ptr<ProxyMutex> remoteCacheContQueueMutex[REMOTE_CONNECT_HASH];
-
-// 0 is an illegal sequence number
-#define CACHE_NO_RESPONSE 0
-static int cluster_sequence_number = 1;
-
-#ifdef CLUSTER_TEST_DEBUG
-static ink_hrtime cache_cluster_timeout = HRTIME_SECONDS(65536);
-#else
-static ink_hrtime cache_cluster_timeout = CACHE_CLUSTER_TIMEOUT;
-#endif
-
-///////////////////
-// Declarations  //
-///////////////////
-static CacheContinuation *find_cache_continuation(unsigned int, unsigned int);
-
-static unsigned int new_cache_sequence_number();
-
-#define DOT_SEPARATED(_x) \
-  ((unsigned char *)&(_x))[0], ((unsigned char *)&(_x))[1], ((unsigned char *)&(_x))[2], ((unsigned char *)&(_x))[3]
-
-#define ET_CACHE_CONT_SM ET_NET
-#define ALLOW_THREAD_STEAL true
-
-/**********************************************************************/
-#ifdef CACHE_MSG_TRACE
-/**********************************************************************/
-
-/**********************************************************************/
-// Debug trace support for cache RPC messages
-/**********************************************************************/
-
-#define MAX_TENTRIES 4096
-struct traceEntry {
-  unsigned int seqno;
-  int op;
-  char *type;
-};
-struct traceEntry recvTraceTable[MAX_TENTRIES];
-struct traceEntry sndTraceTable[MAX_TENTRIES];
-
-static recvTraceTable_index = 0;
-static sndTraceTable_index  = 0;
-
-void
-log_cache_op_msg(unsigned int seqno, int op, char *type)
-{
-  int t                   = ink_atomic_increment(&recvTraceTable_index, 1);
-  int n                   = recvTraceTable_index % MAX_TENTRIES;
-  recvTraceTable[n].seqno = seqno;
-  recvTraceTable[n].op    = op;
-  recvTraceTable[n].type  = type;
-}
-
-void
-log_cache_op_sndmsg(unsigned int seqno, int op, char *type)
-{
-  int t                  = ink_atomic_increment(&sndTraceTable_index, 1);
-  int n                  = sndTraceTable_index % MAX_TENTRIES;
-  sndTraceTable[n].seqno = seqno;
-  sndTraceTable[n].op    = op;
-  sndTraceTable[n].type  = type;
-}
-
-void
-dump_recvtrace_table()
-{
-  int n;
-  printf("\n");
-  for (n = 0; n < MAX_TENTRIES; ++n)
-    printf("[%d] seqno=%d, op=%d type=%s\n", n, recvTraceTable[n].seqno, recvTraceTable[n].op,
-           recvTraceTable[n].type ? recvTraceTable[n].type : "");
-}
-
-void
-dump_sndtrace_table()
-{
-  int n;
-  printf("\n");
-  for (n = 0; n < MAX_TENTRIES; ++n)
-    printf("[%d] seqno=%d, op=%d type=%s\n", n, sndTraceTable[n].seqno, sndTraceTable[n].op,
-           sndTraceTable[n].type ? sndTraceTable[n].type : "");
-}
-
-/**********************************************************************/
-#endif // CACHE_MSG_TRACE
-/**********************************************************************/
-
-///////////////////////////////////////////////////////////////////////
-// Cluster write VC cache.
-///////////////////////////////////////////////////////////////////////
-//
-// In the event that a remote open read fails (HTTP only), an
-// open write is issued and if successful a open write connection
-// is returned for the open read.  We cache the open write VC and
-// resolve the subsequent open write locally from the write VC cache
-// using the INK_MD5 of the URL.
-// Note that this is a global per node cache.
-///////////////////////////////////////////////////////////////////////
-
-class ClusterVConnectionCache
-{
-public:
-  ClusterVConnectionCache() { memset(hash_event, 0, sizeof(hash_event)); }
-  void init();
-  int MD5ToIndex(INK_MD5 *p);
-  int insert(INK_MD5 *, ClusterVConnection *);
-  ClusterVConnection *lookup(INK_MD5 *);
-
-public:
-  struct Entry {
-    LINK(Entry, link);
-    bool mark_for_delete;
-    INK_MD5 key;
-    ClusterVConnection *vc;
-
-    Entry() : mark_for_delete(false), vc(nullptr) {}
-    ~Entry() {}
-  };
-
-  enum {
-    MAX_TABLE_ENTRIES = 256, // must be power of 2
-    SCAN_INTERVAL     = 10   // seconds
-  };
-  Queue<Entry> hash_table[MAX_TABLE_ENTRIES];
-  Ptr<ProxyMutex> hash_lock[MAX_TABLE_ENTRIES];
-  Event *hash_event[MAX_TABLE_ENTRIES];
-};
-
-static ClassAllocator<ClusterVConnectionCache::Entry> ClusterVCCacheEntryAlloc("ClusterVConnectionCache::Entry");
-
-ClusterVConnectionCache *GlobalOpenWriteVCcache = nullptr;
-
-/////////////////////////////////////////////////////////////////
-// Perform periodic purges of ClusterVConnectionCache entries
-/////////////////////////////////////////////////////////////////
-class ClusterVConnectionCacheEvent : public Continuation
-{
-public:
-  ClusterVConnectionCacheEvent(ClusterVConnectionCache *c, int n) : Continuation(new_ProxyMutex()), cache(c), hash_index(n)
-  {
-    SET_HANDLER(&ClusterVConnectionCacheEvent::eventHandler);
-  }
-  int eventHandler(int, Event *);
-
-private:
-  ClusterVConnectionCache *cache;
-  int hash_index;
-};
-
-void
-ClusterVConnectionCache::init()
-{
-  int n;
-  ClusterVConnectionCacheEvent *eh;
-
-  for (n = 0; n < MAX_TABLE_ENTRIES; ++n) {
-    hash_lock[n] = new_ProxyMutex();
-  }
-  for (n = 0; n < MAX_TABLE_ENTRIES; ++n) {
-    // Setup up periodic purge events on each hash list
-
-    eh            = new ClusterVConnectionCacheEvent(this, n);
-    hash_event[n] = eventProcessor.schedule_in(eh, HRTIME_SECONDS(ClusterVConnectionCache::SCAN_INTERVAL), ET_CACHE_CONT_SM);
-  }
-}
-inline int
-ClusterVConnectionCache::MD5ToIndex(INK_MD5 *p)
-{
-  uint64_t i = p->fold();
-  int32_t h, l;
-
-  h = i >> 32;
-  l = i & 0xFFFFFFFF;
-  return ((h ^ l) % MAX_TABLE_ENTRIES) & (MAX_TABLE_ENTRIES - 1);
-}
-
-int
-ClusterVConnectionCache::insert(INK_MD5 *key, ClusterVConnection *vc)
-{
-  int index = MD5ToIndex(key);
-  Entry *e;
-  EThread *thread   = this_ethread();
-  ProxyMutex *mutex = thread->mutex.get();
-
-  MUTEX_TRY_LOCK(lock, hash_lock[index], thread);
-  if (!lock.is_locked()) {
-    CLUSTER_INCREMENT_DYN_STAT(CLUSTER_VC_CACHE_INSERT_LOCK_MISSES_STAT);
-    return 0; // lock miss, retry later
-
-  } else {
-    // Add entry to list
-
-    e      = ClusterVCCacheEntryAlloc.alloc();
-    e->key = *key;
-    e->vc  = vc;
-    hash_table[index].enqueue(e);
-    CLUSTER_INCREMENT_DYN_STAT(CLUSTER_VC_CACHE_INSERTS_STAT);
-  }
-  return 1; // Success
-}
-
-ClusterVConnection *
-ClusterVConnectionCache::lookup(INK_MD5 *key)
-{
-  int index = MD5ToIndex(key);
-  Entry *e;
-  ClusterVConnection *vc = nullptr;
-  EThread *thread        = this_ethread();
-  ProxyMutex *mutex      = thread->mutex.get();
-
-  MUTEX_TRY_LOCK(lock, hash_lock[index], thread);
-  if (!lock.is_locked()) {
-    CLUSTER_INCREMENT_DYN_STAT(CLUSTER_VC_CACHE_LOOKUP_LOCK_MISSES_STAT);
-    return vc; // lock miss, retry later
-
-  } else {
-    e = hash_table[index].head;
-    while (e) {
-      if (*key == e->key) { // Hit
-        vc = e->vc;
-        hash_table[index].remove(e);
-        ClusterVCCacheEntryAlloc.free(e);
-        CLUSTER_INCREMENT_DYN_STAT(CLUSTER_VC_CACHE_LOOKUP_HITS_STAT);
-        return vc;
-
-      } else {
-        e = e->link.next;
-      }
-    }
-  }
-  CLUSTER_INCREMENT_DYN_STAT(CLUSTER_VC_CACHE_LOOKUP_MISSES_STAT);
-  return (ClusterVConnection *)-1; // Miss
-}
-
-int
-ClusterVConnectionCacheEvent::eventHandler(int /* event ATS_UNUSED */, Event *e)
-{
-  CLUSTER_INCREMENT_DYN_STAT(CLUSTER_VC_CACHE_SCANS_STAT);
-  MUTEX_TRY_LOCK(lock, cache->hash_lock[hash_index], this_ethread());
-  if (!lock.is_locked()) {
-    CLUSTER_INCREMENT_DYN_STAT(CLUSTER_VC_CACHE_SCAN_LOCK_MISSES_STAT);
-    e->schedule_in(HRTIME_MSECONDS(10));
-    return EVENT_DONE;
-  }
-  // Perform purge action on unreferenced VC(s).
-
-  ClusterVConnectionCache::Entry *entry;
-  ClusterVConnectionCache::Entry *next_entry;
-  entry = cache->hash_table[hash_index].head;
-
-  while (entry) {
-    if (entry->mark_for_delete) {
-      next_entry = entry->link.next;
-
-      cache->hash_table[hash_index].remove(entry);
-      entry->vc->allow_remote_close();
-      entry->vc->do_io(VIO::CLOSE);
-
-      ClusterVCCacheEntryAlloc.free(entry);
-      entry = next_entry;
-      CLUSTER_INCREMENT_DYN_STAT(CLUSTER_VC_CACHE_PURGES_STAT);
-
-    } else {
-      entry->mark_for_delete = true;
-      entry                  = entry->link.next;
-    }
-  }
-
-  // Setup for next purge event
-
-  e->schedule_in(HRTIME_SECONDS(ClusterVConnectionCache::SCAN_INTERVAL), ET_CACHE_CONT_SM);
-  return EVENT_DONE;
-}
-
-///////////////////////////////////////////////////////////////////////
-
-////////////////////////////////////////////////////
-// init()
-//   Global initializations for CacheContinuation
-////////////////////////////////////////////////////
-int
-CacheContinuation::init()
-{
-  int n;
-  for (n                         = 0; n < REMOTE_CONNECT_HASH; ++n)
-    remoteCacheContQueueMutex[n] = new_ProxyMutex();
-
-  GlobalOpenWriteVCcache = new ClusterVConnectionCache;
-  GlobalOpenWriteVCcache->init();
-  return 0;
-}
-
-///////////////////////////////////////////////////////////////////////
-// do_op()
-//   Main function to do a cluster cache operation
-///////////////////////////////////////////////////////////////////////
-Action *
-CacheContinuation::do_op(Continuation *c, ClusterMachine *mp, void *args, int user_opcode, char *data, int data_len, int nbytes,
-                         MIOBuffer *b)
-{
-  CacheContinuation *cc = nullptr;
-  Action *act           = nullptr;
-  char *msg             = nullptr;
-  ClusterHandler *ch    = mp->pop_ClusterHandler();
-
-  /////////////////////////////////////////////////////////////////////
-  // Unconditionally map open read buffer interfaces to open read.
-  // open read buffer interfaces are now deprecated.
-  /////////////////////////////////////////////////////////////////////
-  int opcode = user_opcode;
-  switch (opcode) {
-  case CACHE_OPEN_READ_BUFFER:
-    opcode = CACHE_OPEN_READ;
-    break;
-  case CACHE_OPEN_READ_BUFFER_LONG:
-    opcode = CACHE_OPEN_READ_LONG;
-    break;
-  default:
-    break;
-  }
-
-  if (!ch)
-    goto no_send_exit;
-
-  if (c) {
-    cc                   = cacheContAllocator_alloc();
-    cc->ch               = ch;
-    cc->target_machine   = mp;
-    cc->request_opcode   = opcode;
-    cc->mutex            = c->mutex;
-    cc->action           = c;
-    cc->action.cancelled = false;
-    cc->start_time       = Thread::get_hrtime();
-    cc->from             = mp;
-    cc->result           = op_failure(opcode);
-    SET_CONTINUATION_HANDLER(cc, (CacheContHandler)&CacheContinuation::remoteOpEvent);
-    act = &cc->action;
-
-    // set up sequence number so we can find this continuation
-
-    cc->target_ip  = mp->ip;
-    cc->seq_number = new_cache_sequence_number();
-
-    // establish timeout for cache op
-
-    unsigned int hash = FOLDHASH(cc->target_ip, cc->seq_number);
-    MUTEX_TRY_LOCK(queuelock, remoteCacheContQueueMutex[hash], this_ethread());
-    if (!queuelock.is_locked()) {
-      // failed to acquire lock: no problem, retry later
-      cc->timeout = eventProcessor.schedule_in(cc, CACHE_RETRY_PERIOD, ET_CACHE_CONT_SM);
-    } else {
-      remoteCacheContQueue[hash].enqueue(cc);
-      MUTEX_RELEASE(queuelock);
-      cc->timeout = eventProcessor.schedule_in(cc, cache_cluster_timeout, ET_CACHE_CONT_SM);
-    }
-  }
-  //
-  // Determine the type of the "Over The Wire" (OTW) message header and
-  //   initialize it.
-  //
-  Debug("cache_msg", "do_op opcode=%d seqno=%d Machine=%p data=%p datalen=%d mio=%p", opcode,
-        (c ? cc->seq_number : CACHE_NO_RESPONSE), mp, data, data_len, b);
-
-  switch (opcode) {
-  case CACHE_OPEN_WRITE_BUFFER:
-  case CACHE_OPEN_WRITE_BUFFER_LONG: {
-    ink_release_assert(!"write buffer not supported");
-    break;
-  }
-  case CACHE_OPEN_READ_BUFFER:
-  case CACHE_OPEN_READ_BUFFER_LONG: {
-    ink_release_assert(!"read buffer not supported");
-    break;
-  }
-  case CACHE_OPEN_WRITE:
-  case CACHE_OPEN_READ: {
-    ink_release_assert(c != nullptr);
-    //////////////////////
-    // Use short format //
-    //////////////////////
-    if (!data) {
-      data_len = op_to_sizeof_fixedlen_msg(opcode);
-      data     = (char *)ALLOCA_DOUBLE(data_len);
-    }
-    msg                 = (char *)data;
-    CacheOpMsg_short *m = (CacheOpMsg_short *)msg;
-    m->init();
-    m->opcode     = opcode;
-    m->cfl_flags  = ((CacheOpArgs_General *)args)->cfl_flags;
-    m->md5        = *((CacheOpArgs_General *)args)->url_md5;
-    cc->url_md5   = m->md5;
-    m->seq_number = (c ? cc->seq_number : CACHE_NO_RESPONSE);
-    m->frag_type  = ((CacheOpArgs_General *)args)->frag_type;
-    if (opcode == CACHE_OPEN_WRITE) {
-      m->nbytes = nbytes;
-      m->data   = (uint32_t)((CacheOpArgs_General *)args)->pin_in_cache;
-    } else {
-      m->nbytes = 0;
-      m->data   = 0;
-    }
-
-    if (opcode == CACHE_OPEN_READ) {
-      //
-      // Set upper limit on initial data received with response
-      // for open read response
-      //
-      m->buffer_size = DEFAULT_MAX_BUFFER_SIZE;
-    } else {
-      m->buffer_size = 0;
-    }
-
-    //
-    // Establish the local VC
-    //
-    int res = setup_local_vc(msg, data_len, cc, mp, &act);
-    if (!res) {
-      /////////////////////////////////////////////////////
-      // Unable to setup local VC, request aborted.
-      // Remove request from pending list and deallocate.
-      /////////////////////////////////////////////////////
-      cc->remove_and_delete(0, (Event *)nullptr);
-      return act;
-
-    } else if (res != -1) {
-      ///////////////////////////////////////
-      // VC established, send request
-      ///////////////////////////////////////
-      break;
-
-    } else {
-      //////////////////////////////////////////////////////
-      // Unable to setup VC, delay required, await callback
-      //////////////////////////////////////////////////////
-      goto no_send_exit;
-    }
-  }
-
-  case CACHE_OPEN_READ_LONG:
-  case CACHE_OPEN_WRITE_LONG: {
-    ink_release_assert(c != nullptr);
-    //////////////////////
-    // Use long format  //
-    //////////////////////
-    msg                = data;
-    CacheOpMsg_long *m = (CacheOpMsg_long *)msg;
-    m->init();
-    m->opcode     = opcode;
-    m->cfl_flags  = ((CacheOpArgs_General *)args)->cfl_flags;
-    m->url_md5    = *((CacheOpArgs_General *)args)->url_md5;
-    cc->url_md5   = m->url_md5;
-    m->seq_number = (c ? cc->seq_number : CACHE_NO_RESPONSE);
-    m->nbytes     = nbytes;
-    m->data       = (uint32_t)((CacheOpArgs_General *)args)->pin_in_cache;
-    m->frag_type  = (uint32_t)((CacheOpArgs_General *)args)->frag_type;
-
-    if (opcode == CACHE_OPEN_READ_LONG) {
-      //
-      // Set upper limit on initial data received with response
-      // for open read response
-      //
-      m->buffer_size = DEFAULT_MAX_BUFFER_SIZE;
-    } else {
-      m->buffer_size = 0;
-    }
-    //
-    // Establish the local VC
-    //
-    int res = setup_local_vc(msg, data_len, cc, mp, &act);
-    if (!res) {
-      /////////////////////////////////////////////////////
-      // Unable to setup local VC, request aborted.
-      // Remove request from pending list and deallocate.
-      /////////////////////////////////////////////////////
-      cc->remove_and_delete(0, (Event *)nullptr);
-      return act;
-
-    } else if (res != -1) {
-      ///////////////////////////////////////
-      // VC established, send request
-      ///////////////////////////////////////
-      break;
-
-    } else {
-      //////////////////////////////////////////////////////
-      // Unable to setup VC, delay required, await callback
-      //////////////////////////////////////////////////////
-      goto no_send_exit;
-    }
-  }
-  case CACHE_UPDATE:
-  case CACHE_REMOVE:
-  case CACHE_DEREF: {
-    //////////////////////
-    // Use short format //
-    //////////////////////
-    msg                 = data;
-    CacheOpMsg_short *m = (CacheOpMsg_short *)msg;
-    m->init();
-    m->opcode    = opcode;
-    m->frag_type = ((CacheOpArgs_Deref *)args)->frag_type;
-    m->cfl_flags = ((CacheOpArgs_Deref *)args)->cfl_flags;
-    if (opcode == CACHE_DEREF)
-      m->md5 = *((CacheOpArgs_Deref *)args)->md5;
-    else
-      m->md5      = *((CacheOpArgs_General *)args)->url_md5;
-    m->seq_number = (c ? cc->seq_number : CACHE_NO_RESPONSE);
-    break;
-  }
-  case CACHE_LINK: {
-    ////////////////////////
-    // Use short_2 format //
-    ////////////////////////
-    msg                   = data;
-    CacheOpMsg_short_2 *m = (CacheOpMsg_short_2 *)msg;
-    m->init();
-    m->opcode     = opcode;
-    m->cfl_flags  = ((CacheOpArgs_Link *)args)->cfl_flags;
-    m->md5_1      = *((CacheOpArgs_Link *)args)->from;
-    m->md5_2      = *((CacheOpArgs_Link *)args)->to;
-    m->seq_number = (c ? cc->seq_number : CACHE_NO_RESPONSE);
-    m->frag_type  = ((CacheOpArgs_Link *)args)->frag_type;
-    break;
-  }
-  default:
-    msg = nullptr;
-    break;
-  }
-#ifdef CACHE_MSG_TRACE
-  log_cache_op_sndmsg((c ? cc->seq_number : CACHE_NO_RESPONSE), 0, "do_op");
-#endif
-  clusterProcessor.invoke_remote(
-    ch, op_needs_marshalled_coi(opcode) ? CACHE_OP_MALLOCED_CLUSTER_FUNCTION : CACHE_OP_CLUSTER_FUNCTION, (char *)msg, data_len);
-
-no_send_exit:
-  if (c) {
-    return act;
-  } else {
-    return (Action *)nullptr;
-  }
-}
-
-int
-CacheContinuation::setup_local_vc(char *data, int data_len, CacheContinuation *cc, ClusterMachine *mp, Action **act)
-{
-  bool read_op   = op_is_read(cc->request_opcode);
-  bool short_msg = op_is_shortform(cc->request_opcode);
-
-  // Alloc buffer, copy message and attach to continuation
-  cc->setMsgBufferLen(data_len);
-  cc->allocMsgBuffer();
-  memcpy(cc->getMsgBuffer(), data, data_len);
-
-  SET_CONTINUATION_HANDLER(cc, (CacheContHandler)&CacheContinuation::localVCsetupEvent);
-
-  if (short_msg) {
-    Debug("cache_proto", "open_local-s (%s) seqno=%d", (read_op ? "R" : "W"), ((CacheOpMsg_short *)data)->seq_number);
-  } else {
-    Debug("cache_proto", "open_local-l (%s) seqno=%d", (read_op ? "R" : "W"), ((CacheOpMsg_long *)data)->seq_number);
-  }
-
-  // Create local VC
-  ClusterVConnection *vc;
-
-  if (!read_op && (cc->request_opcode == CACHE_OPEN_WRITE_LONG)) {
-    // Determine if the open_write has already been established.
-    vc = cc->lookupOpenWriteVC();
-
-  } else {
-    vc = clusterProcessor.open_local(cc, mp, cc->open_local_token,
-                                     (CLUSTER_OPT_ALLOW_IMMEDIATE | (read_op ? CLUSTER_OPT_CONN_READ : CLUSTER_OPT_CONN_WRITE)));
-  }
-  if (!vc) {
-    // Error, abort request
-    if (short_msg) {
-      Debug("cache_proto", "0open_local-s (%s) failed, seqno=%d", (read_op ? "R" : "W"), ((CacheOpMsg_short *)data)->seq_number);
-    } else {
-      Debug("cache_proto", "1open_local-l (%s) failed, seqno=%d", (read_op ? "R" : "W"), ((CacheOpMsg_long *)data)->seq_number);
-    }
-    cc->freeMsgBuffer();
-    if (cc->timeout)
-      cc->timeout->cancel();
-    cc->timeout = nullptr;
-
-    // Post async failure callback on a different continuation.
-    *act = callback_failure(&cc->action, (read_op ? CACHE_EVENT_OPEN_READ_FAILED : CACHE_EVENT_OPEN_WRITE_FAILED), 0);
-    return 0;
-
-  } else if (vc != CLUSTER_DELAYED_OPEN) {
-    // We have established the VC
-    if (read_op) {
-      cc->read_cluster_vc = vc;
-    } else {
-      cc->write_cluster_vc = vc;
-    }
-    cc->cluster_vc_channel = vc->channel;
-    vc->current_cont       = cc;
-
-    if (short_msg) {
-      CacheOpMsg_short *ms = (CacheOpMsg_short *)data;
-      ms->channel          = vc->channel;
-      ms->token            = cc->open_local_token;
-      Debug("cache_proto", "0open_local-s (%s) success, seqno=%d chan=%d token=%d,%d VC=%p", (read_op ? "R" : "W"), ms->seq_number,
-            vc->channel, ms->token.ip_created, ms->token.sequence_number, vc);
-    } else {
-      CacheOpMsg_long *ml = (CacheOpMsg_long *)data;
-      ml->channel         = vc->channel;
-      ml->token           = cc->open_local_token;
-      Debug("cache_proto", "1open_local-l (%s) success, seqno=%d chan=%d token=%d,%d VC=%p", (read_op ? "R" : "W"), ml->seq_number,
-            vc->channel, ml->token.ip_created, ml->token.sequence_number, vc);
-    }
-    cc->freeMsgBuffer();
-    SET_CONTINUATION_HANDLER(cc, (CacheContHandler)&CacheContinuation::remoteOpEvent);
-    return 1;
-
-  } else {
-    //////////////////////////////////////////////////////
-    // Unable to setup VC, delay required, await callback
-    //////////////////////////////////////////////////////
-    return -1;
-  }
-}
-
-ClusterVConnection *
-CacheContinuation::lookupOpenWriteVC()
-{
-  ///////////////////////////////////////////////////////////////
-  // See if we already have an open_write ClusterVConnection
-  // which was established in a previous remote open_read which
-  // failed.
-  ///////////////////////////////////////////////////////////////
-  ClusterVConnection *vc;
-  CacheOpMsg_long *ml = (CacheOpMsg_long *)getMsgBuffer();
-
-  vc = GlobalOpenWriteVCcache->lookup(&ml->url_md5);
-
-  if (vc == ((ClusterVConnection *)nullptr)) {
-    // Retry lookup
-    SET_CONTINUATION_HANDLER(this, (CacheContHandler)&CacheContinuation::lookupOpenWriteVCEvent);
-    //
-    // Note: In the lookupOpenWriteVCEvent handler, we use EVENT_IMMEDIATE
-    //       to distinguish the lookup retry from a request timeout
-    //       which uses EVENT_INTERVAL.
-    //
-    lookup_open_write_vc_event = eventProcessor.schedule_imm(this, ET_CACHE_CONT_SM);
-
-  } else if (vc != ((ClusterVConnection *)-1)) {
-    // Hit, found open_write VC in cache.
-    // Post open_write completion by simulating a
-    // remote cache op result message.
-
-    vc->action_ = action; // establish new continuation
-
-    SET_CONTINUATION_HANDLER(this, (CacheContHandler)&CacheContinuation::localVCsetupEvent);
-    this->handleEvent(CLUSTER_EVENT_OPEN_EXISTS, vc);
-
-    CacheOpReplyMsg msg;
-    int msglen;
-
-    msglen         = CacheOpReplyMsg::sizeof_fixedlen_msg();
-    msg.result     = CACHE_EVENT_OPEN_WRITE;
-    msg.seq_number = seq_number;
-    msg.token      = vc->token;
-
-    cache_op_result_ClusterFunction(ch, (void *)&msg, msglen);
-
-  } else {
-    // Miss, establish local VC and send remote open_write request
-
-    SET_CONTINUATION_HANDLER(this, (CacheContHandler)&CacheContinuation::localVCsetupEvent);
-    vc = clusterProcessor.open_local(this, from, open_local_token, (CLUSTER_OPT_ALLOW_IMMEDIATE | CLUSTER_OPT_CONN_WRITE));
-    if (!vc) {
-      this->handleEvent(CLUSTER_EVENT_OPEN_FAILED, nullptr);
-
-    } else if (vc != CLUSTER_DELAYED_OPEN) {
-      this->handleEvent(CLUSTER_EVENT_OPEN, vc);
-    }
-  }
-  return CLUSTER_DELAYED_OPEN; // force completion in callback
-}
-
-int
-CacheContinuation::lookupOpenWriteVCEvent(int event, Event *e)
-{
-  if (event == EVENT_IMMEDIATE) {
-    // Retry open_write VC lookup
-    lookupOpenWriteVC();
-
-  } else {
-    lookup_open_write_vc_event->cancel();
-    SET_CONTINUATION_HANDLER(this, (CacheContHandler)&CacheContinuation::localVCsetupEvent);
-    this->handleEvent(event, e);
-  }
-  return EVENT_DONE;
-}
-
-int
-CacheContinuation::remove_and_delete(int /* event ATS_UNUSED */, Event *e)
-{
-  unsigned int hash = FOLDHASH(target_ip, seq_number);
-  MUTEX_TRY_LOCK(queuelock, remoteCacheContQueueMutex[hash], this_ethread());
-  if (queuelock.is_locked()) {
-    if (remoteCacheContQueue[hash].in(this)) {
-      remoteCacheContQueue[hash].remove(this);
-    }
-    MUTEX_RELEASE(queuelock);
-    if (use_deferred_callback)
-      callback_failure(&action, result, result_error, this);
-    else
-      cacheContAllocator_free(this);
-
-  } else {
-    SET_HANDLER((CacheContHandler)&CacheContinuation::remove_and_delete);
-    if (!e) {
-      timeout = eventProcessor.schedule_in(this, cache_cluster_timeout, ET_CACHE_CONT_SM);
-    } else {
-      e->schedule_in(cache_cluster_timeout);
-    }
-  }
-  return EVENT_DONE;
-}
-
-int
-CacheContinuation::localVCsetupEvent(int event, ClusterVConnection *vc)
-{
-  ink_assert(magicno == (int)MagicNo);
-  ink_assert(getMsgBuffer());
-  bool short_msg = op_is_shortform(request_opcode);
-  bool read_op   = op_is_read(request_opcode);
-
-  if (event == EVENT_INTERVAL) {
-    Event *e          = (Event *)vc;
-    unsigned int hash = FOLDHASH(target_ip, seq_number);
-
-    MUTEX_TRY_LOCK(queuelock, remoteCacheContQueueMutex[hash], e->ethread);
-    if (!queuelock.is_locked()) {
-      e->schedule_in(CACHE_RETRY_PERIOD);
-      return EVENT_CONT;
-    }
-
-    if (!remoteCacheContQueue[hash].in(this)) {
-      ////////////////////////////////////////////////////
-      // Not yet queued on outstanding operations list
-      ////////////////////////////////////////////////////
-      remoteCacheContQueue[hash].enqueue(this);
-      ink_assert(timeout == e);
-      MUTEX_RELEASE(queuelock);
-      e->schedule_in(cache_cluster_timeout);
-      return EVENT_CONT;
-
-    } else {
-      /////////////////////////////////////////////////////
-      // Timeout occurred
-      /////////////////////////////////////////////////////
-      remoteCacheContQueue[hash].remove(this);
-      MUTEX_RELEASE(queuelock);
-      Debug("cluster_timeout", "0cluster op timeout %d", seq_number);
-      CLUSTER_INCREMENT_DYN_STAT(CLUSTER_REMOTE_OP_TIMEOUTS_STAT);
-      timeout = (Event *)1; // Note timeout
-      /////////////////////////////////////////////////////////////////
-      // Note: Failure callback is sent now, but the deallocation of
-      //       the CacheContinuation is deferred until we receive the
-      //       open_local() callback.
-      /////////////////////////////////////////////////////////////////
-      if (!action.cancelled)
-        action.continuation->handleEvent((read_op ? CACHE_EVENT_OPEN_READ_FAILED : CACHE_EVENT_OPEN_WRITE_FAILED), nullptr);
-      return EVENT_DONE;
-    }
-
-  } else if (((event == CLUSTER_EVENT_OPEN) || (event == CLUSTER_EVENT_OPEN_EXISTS)) &&
-             (((ptrdiff_t)timeout & (ptrdiff_t)1) == 0)) {
-    ink_hrtime now;
-    now = Thread::get_hrtime();
-    CLUSTER_SUM_DYN_STAT(CLUSTER_OPEN_DELAY_TIME_STAT, now - start_time);
-    LOG_EVENT_TIME(start_time, open_delay_time_dist, open_delay_events);
-    if (read_op) {
-      read_cluster_vc = vc;
-    } else {
-      write_cluster_vc = vc;
-    }
-    cluster_vc_channel = vc->channel;
-    vc->current_cont   = this;
-
-    if (short_msg) {
-      CacheOpMsg_short *ms = (CacheOpMsg_short *)getMsgBuffer();
-      ms->channel          = vc->channel;
-      ms->token            = open_local_token;
-
-      Debug("cache_proto", "2open_local-s (%s) success, seqno=%d chan=%d token=%d,%d VC=%p", (read_op ? "R" : "W"), ms->seq_number,
-            vc->channel, ms->token.ip_created, ms->token.sequence_number, vc);
-
-    } else {
-      CacheOpMsg_long *ml = (CacheOpMsg_long *)getMsgBuffer();
-      ml->channel         = vc->channel;
-      ml->token           = open_local_token;
-
-      Debug("cache_proto", "3open_local-l (%s) success, seqno=%d chan=%d token=%d,%d VC=%p", (read_op ? "R" : "W"), ml->seq_number,
-            vc->channel, ml->token.ip_created, ml->token.sequence_number, vc);
-    }
-    SET_HANDLER((CacheContHandler)&CacheContinuation::remoteOpEvent);
-
-    if (event != CLUSTER_EVENT_OPEN_EXISTS) {
-      // Send request message
-      clusterProcessor.invoke_remote(
-        ch, (op_needs_marshalled_coi(request_opcode) ? CACHE_OP_MALLOCED_CLUSTER_FUNCTION : CACHE_OP_CLUSTER_FUNCTION),
-        (char *)getMsgBuffer(), getMsgBufferLen());
-    }
-
-  } else {
-    int send_failure_callback = 1;
-
-    if (((ptrdiff_t)timeout & (ptrdiff_t)1) == 0) {
-      if (short_msg) {
-        Debug("cache_proto", "2open_local-s (%s) failed, seqno=%d", (read_op ? "R" : "W"),
-              ((CacheOpMsg_short *)getMsgBuffer())->seq_number);
-      } else {
-        Debug("cache_proto", "3open_local-l (%s) failed, seqno=%d", (read_op ? "R" : "W"),
-              ((CacheOpMsg_long *)getMsgBuffer())->seq_number);
-      }
-
-    } else {
-      Debug("cache_proto", "4open_local cancelled due to timeout, seqno=%d", seq_number);
-      this->timeout = nullptr;
-
-      // Deallocate VC if successfully acquired
-
-      if (event == CLUSTER_EVENT_OPEN) {
-        vc->pending_remote_fill = 0;
-        vc->remote_closed       = 1; // avoid remote close msg
-        vc->do_io(VIO::CLOSE);
-      }
-      send_failure_callback = 0; // already sent.
-    }
-
-    if (this->timeout)
-      this->timeout->cancel();
-    this->timeout = nullptr;
-
-    freeMsgBuffer();
-    if (send_failure_callback) {
-      //
-      // Action corresponding to "this" already sent back to user,
-      //   use "this" to establish the failure callback after
-      //   removing ourselves from the active list.
-      //
-      this->use_deferred_callback = true;
-      this->result                = (read_op ? CACHE_EVENT_OPEN_READ_FAILED : CACHE_EVENT_OPEN_WRITE_FAILED);
-      this->result_error          = 0;
-      remove_and_delete(0, (Event *)nullptr);
-
-    } else {
-      cacheContAllocator_free(this);
-    }
-    return EVENT_DONE;
-  }
-  // Free message
-  freeMsgBuffer();
-
-  return EVENT_DONE;
-}
-
-///////////////////////////////////////////////////////////////////////////
-// cache_op_ClusterFunction()
-//   On the receiving side, handle a general cluster cache operation
-///////////////////////////////////////////////////////////////////////////
-
-////////////////////////////////////////////////////////////////////////
-// Marshaling functions for OTW message headers
-////////////////////////////////////////////////////////////////////////
-
-inline CacheOpMsg_long *
-unmarshal_CacheOpMsg_long(void *data, int NeedByteSwap)
-{
-  if (NeedByteSwap)
-    ((CacheOpMsg_long *)data)->SwapBytes();
-  return (CacheOpMsg_long *)data;
-}
-
-inline CacheOpMsg_short *
-unmarshal_CacheOpMsg_short(void *data, int NeedByteSwap)
-{
-  if (NeedByteSwap)
-    ((CacheOpMsg_short *)data)->SwapBytes();
-  return (CacheOpMsg_short *)data;
-}
-
-inline CacheOpMsg_short_2 *
-unmarshal_CacheOpMsg_short_2(void *data, int NeedByteSwap)
-{
-  if (NeedByteSwap)
-    ((CacheOpMsg_short_2 *)data)->SwapBytes();
-  return (CacheOpMsg_short_2 *)data;
-}
-
-// init_from_long() support routine for cache_op_ClusterFunction()
-inline void
-init_from_long(CacheContinuation *cont, CacheOpMsg_long *msg, ClusterMachine *m)
-{
-  cont->no_reply_message   = (msg->seq_number == CACHE_NO_RESPONSE);
-  cont->seq_number         = msg->seq_number;
-  cont->cfl_flags          = msg->cfl_flags;
-  cont->from               = m;
-  cont->url_md5            = msg->url_md5;
-  cont->cluster_vc_channel = msg->channel;
-  cont->frag_type          = (CacheFragType)msg->frag_type;
-  if ((cont->request_opcode == CACHE_OPEN_WRITE_LONG) || (cont->request_opcode == CACHE_OPEN_READ_LONG)) {
-    cont->pin_in_cache = (time_t)msg->data;
-  } else {
-    cont->pin_in_cache = 0;
-  }
-  cont->token  = msg->token;
-  cont->nbytes = (((int)msg->nbytes < 0) ? 0 : msg->nbytes);
-
-  if (cont->request_opcode == CACHE_OPEN_READ_LONG) {
-    cont->caller_buf_freebytes = msg->buffer_size;
-  } else {
-    cont->caller_buf_freebytes = 0;
-  }
-}
-
-// init_from_short() support routine for cache_op_ClusterFunction()
-inline void
-init_from_short(CacheContinuation *cont, CacheOpMsg_short *msg, ClusterMachine *m)
-{
-  cont->no_reply_message   = (msg->seq_number == CACHE_NO_RESPONSE);
-  cont->seq_number         = msg->seq_number;
-  cont->cfl_flags          = msg->cfl_flags;
-  cont->from               = m;
-  cont->url_md5            = msg->md5;
-  cont->cluster_vc_channel = msg->channel;
-  cont->token              = msg->token;
-  cont->nbytes             = (((int)msg->nbytes < 0) ? 0 : msg->nbytes);
-  cont->frag_type          = (CacheFragType)msg->frag_type;
-
-  if (cont->request_opcode == CACHE_OPEN_WRITE) {
-    cont->pin_in_cache = (time_t)msg->data;
-  } else {
-    cont->pin_in_cache = 0;
-  }
-
-  if (cont->request_opcode == CACHE_OPEN_READ) {
-    cont->caller_buf_freebytes = msg->buffer_size;
-  } else {
-    cont->caller_buf_freebytes = 0;
-  }
-}
-
-// init_from_short_2() support routine for cache_op_ClusterFunction()
-inline void
-init_from_short_2(CacheContinuation *cont, CacheOpMsg_short_2 *msg, ClusterMachine *m)
-{
-  cont->no_reply_message = (msg->seq_number == CACHE_NO_RESPONSE);
-  cont->seq_number       = msg->seq_number;
-  cont->cfl_flags        = msg->cfl_flags;
-  cont->from             = m;
-  cont->url_md5          = msg->md5_1;
-  cont->frag_type        = (CacheFragType)msg->frag_type;
-}
-
-void
-cache_op_ClusterFunction(ClusterHandler *ch, void *data, int len)
-{
-  EThread *thread   = this_ethread();
-  ProxyMutex *mutex = thread->mutex.get();
-  ////////////////////////////////////////////////////////
-  // Note: we are running on the ET_CLUSTER thread
-  ////////////////////////////////////////////////////////
-  CLUSTER_INCREMENT_DYN_STAT(CLUSTER_CACHE_OUTSTANDING_STAT);
-
-  int opcode;
-  ClusterMessageHeader *mh = (ClusterMessageHeader *)data;
-
-  if (mh->GetMsgVersion() != CacheOpMsg_long::CACHE_OP_LONG_MESSAGE_VERSION) { ////////////////////////////////////////////////
-    // Convert from old to current message format
-    ////////////////////////////////////////////////
-    ink_release_assert(!"cache_op_ClusterFunction() bad msg version");
-  }
-  opcode = ((CacheOpMsg_long *)data)->opcode;
-
-  // If necessary, create a continuation to reflect the response back
-
-  CacheContinuation *c = CacheContinuation::cacheContAllocator_alloc();
-  c->mutex             = new_ProxyMutex();
-  MUTEX_TRY_LOCK(lock, c->mutex, this_ethread());
-  c->request_opcode = opcode;
-  c->token.clear();
-  c->start_time = Thread::get_hrtime();
-  c->ch         = ch;
-  SET_CONTINUATION_HANDLER(c, (CacheContHandler)&CacheContinuation::replyOpEvent);
-
-  switch (opcode) {
-  case CACHE_OPEN_WRITE_BUFFER:
-  case CACHE_OPEN_WRITE_BUFFER_LONG:
-    ink_release_assert(!"cache_op_ClusterFunction WRITE_BUFFER not supported");
-    break;
-
-  case CACHE_OPEN_READ_BUFFER:
-  case CACHE_OPEN_READ_BUFFER_LONG:
-    ink_release_assert(!"cache_op_ClusterFunction READ_BUFFER not supported");
-    break;
-
-  case CACHE_OPEN_READ: {
-    CacheOpMsg_short *msg = unmarshal_CacheOpMsg_short(data, mh->NeedByteSwap());
-    init_from_short(c, msg, ch->machine);
-    Debug("cache_msg", "cache_op-s op=%d seqno=%d data=%p len=%d machine=%p", opcode, c->seq_number, data, len, ch->machine);
-    //
-    // Establish the remote side of the ClusterVConnection
-    //
-    c->write_cluster_vc = clusterProcessor.connect_local((Continuation *)nullptr, &c->token, c->cluster_vc_channel,
-                                                         (CLUSTER_OPT_IMMEDIATE | CLUSTER_OPT_CONN_READ));
-    if (!c->write_cluster_vc) {
-      // Unable to setup channel, abort processing.
-      CLUSTER_INCREMENT_DYN_STAT(CLUSTER_CHAN_INUSE_STAT);
-      Debug("chan_inuse", "1Remote chan=%d inuse tok.ip=%u.%u.%u.%u tok.seqno=%d seqno=%d", c->cluster_vc_channel,
-            DOT_SEPARATED(c->token.ip_created), c->token.sequence_number, c->seq_number);
-
-      // Send cluster op failed reply
-      c->replyOpEvent(CACHE_EVENT_OPEN_READ_FAILED, (VConnection *)-ECLUSTER_CHANNEL_INUSE);
-      break;
-
-    } else {
-      c->write_cluster_vc->current_cont = c;
-    }
-    ink_release_assert(c->write_cluster_vc != CLUSTER_DELAYED_OPEN);
-    ink_release_assert((opcode == CACHE_OPEN_READ) || c->write_cluster_vc->pending_remote_fill);
-
-    SET_CONTINUATION_HANDLER(c, (CacheContHandler)&CacheContinuation::setupVCdataRead);
-    Debug("cache_proto", "0read op, seqno=%d chan=%d bufsize=%d token=%d,%d", msg->seq_number, msg->channel, msg->buffer_size,
-          msg->token.ip_created, msg->token.sequence_number);
-#ifdef CACHE_MSG_TRACE
-    log_cache_op_msg(msg->seq_number, len, "cache_op_open_read");
-#endif
-    CacheKey key(msg->md5);
-
-    char *hostname = nullptr;
-    int host_len   = len - op_to_sizeof_fixedlen_msg(opcode);
-    if (host_len) {
-      hostname = (char *)msg->moi.byte;
-    }
-    Cache *call_cache = caches[c->frag_type];
-    c->cache_action   = call_cache->open_read(c, &key, c->frag_type, hostname, host_len);
-    break;
-  }
-  case CACHE_OPEN_READ_LONG: {
-    // Cache needs message data, copy it.
-    c->setMsgBufferLen(len);
-    c->allocMsgBuffer();
-    memcpy(c->getMsgBuffer(), (char *)data, len);
-
-    int flen             = CacheOpMsg_long::sizeof_fixedlen_msg();
-    CacheOpMsg_long *msg = unmarshal_CacheOpMsg_long(c->getMsgBuffer(), mh->NeedByteSwap());
-    init_from_long(c, msg, ch->machine);
-    Debug("cache_msg", "cache_op-l op=%d seqno=%d data=%p len=%d machine=%p", opcode, c->seq_number, data, len, ch->machine);
-#ifdef CACHE_MSG_TRACE
-    log_cache_op_msg(msg->seq_number, len, "cache_op_open_read_long");
-#endif
-    //
-    // Establish the remote side of the ClusterVConnection
-    //
-    c->write_cluster_vc = clusterProcessor.connect_local((Continuation *)nullptr, &c->token, c->cluster_vc_channel,
-                                                         (CLUSTER_OPT_IMMEDIATE | CLUSTER_OPT_CONN_READ));
-    if (!c->write_cluster_vc) {
-      // Unable to setup channel, abort processing.
-      CLUSTER_INCREMENT_DYN_STAT(CLUSTER_CHAN_INUSE_STAT);
-      Debug("chan_inuse", "2Remote chan=%d inuse tok.ip=%u.%u.%u.%u tok.seqno=%d seqno=%d", c->cluster_vc_channel,
-            DOT_SEPARATED(c->token.ip_created), c->token.sequence_number, c->seq_number);
-
-      // Send cluster op failed reply
-      c->replyOpEvent(CACHE_EVENT_OPEN_READ_FAILED, (VConnection *)-ECLUSTER_CHANNEL_INUSE);
-      break;
-
-    } else {
-      c->write_cluster_vc->current_cont = c;
-    }
-    ink_release_assert(c->write_cluster_vc != CLUSTER_DELAYED_OPEN);
-    ink_release_assert((opcode == CACHE_OPEN_READ_LONG) || c->write_cluster_vc->pending_remote_fill);
-
-    SET_CONTINUATION_HANDLER(c, (CacheContHandler)&CacheContinuation::setupReadWriteVC);
-    Debug("cache_proto", "1read op, seqno=%d chan=%d bufsize=%d token=%d,%d", msg->seq_number, msg->channel, msg->buffer_size,
-          msg->token.ip_created, msg->token.sequence_number);
-
-    const char *p = (const char *)msg + flen;
-    int moi_len   = len - flen;
-    int res;
-
-    ink_assert(moi_len > 0);
-
-    // Unmarshal CacheHTTPHdr
-    res = c->ic_request.unmarshal((char *)p, moi_len, nullptr);
-    ink_assert(res > 0);
-    ink_assert(c->ic_request.valid());
-    c->request_purge =
-      c->ic_request.method_get_wksidx() == HTTP_WKSIDX_PURGE || c->ic_request.method_get_wksidx() == HTTP_WKSIDX_DELETE;
-    moi_len -= res;
-    p += res;
-    ink_assert(moi_len > 0);
-
-    c->http_config_params = HttpConfig::acquire();
-
-    char *hostname = nullptr;
-    int host_len   = 0;
-
-    if (moi_len) {
-      hostname = (char *)p;
-      host_len = moi_len;
-
-      // Save hostname and attach it to the continuation since we may
-      //  need it if we convert this to an open_write.
-
-      c->ic_hostname     = new_IOBufferData(iobuffer_size_to_index(host_len));
-      c->ic_hostname_len = host_len;
-
-      memcpy(c->ic_hostname->data(), hostname, host_len);
-    }
-
-    // This is technically not correct, since we're not treating these properly as overridable. But,
-    // this is dead code (mostly), and they weren't overridable before.
-    CacheKey key(msg->url_md5);
-    Cache *call_cache = caches[c->frag_type];
-    Action *a = call_cache->open_read(c, &key, &c->ic_request, &c->http_config_params->oride, c->frag_type, hostname, host_len);
-    // Get rid of purify warnings since 'c' can be freed by open_read.
-    if (a != ACTION_RESULT_DONE) {
-      c->cache_action = a;
-    }
-    break;
-  }
-  case CACHE_OPEN_WRITE: {
-    CacheOpMsg_short *msg = unmarshal_CacheOpMsg_short(data, mh->NeedByteSwap());
-    init_from_short(c, msg, ch->machine);
-    Debug("cache_msg", "cache_op-s op=%d seqno=%d data=%p len=%d machine=%p", opcode, c->seq_number, data, len, ch->machine);
-#ifdef CACHE_MSG_TRACE
-    log_cache_op_msg(msg->seq_number, len, "cache_op_open_write");
-#endif
-    //
-    // Establish the remote side of the ClusterVConnection
-    //
-    c->read_cluster_vc = clusterProcessor.connect_local((Continuation *)nullptr, &c->token, c->cluster_vc_channel,
-                                                        (CLUSTER_OPT_IMMEDIATE | CLUSTER_OPT_CONN_WRITE));
-    if (!c->read_cluster_vc) {
-      // Unable to setup channel, abort processing.
-      CLUSTER_INCREMENT_DYN_STAT(CLUSTER_CHAN_INUSE_STAT);
-      Debug("chan_inuse", "3Remote chan=%d inuse tok.ip=%u.%u.%u.%u tok.seqno=%d seqno=%d", c->cluster_vc_channel,
-            DOT_SEPARATED(c->token.ip_created), c->token.sequence_number, c->seq_number);
-
-      // Send cluster op failed reply
-      c->replyOpEvent(CACHE_EVENT_OPEN_WRITE_FAILED, (VConnection *)-ECLUSTER_CHANNEL_INUSE);
-      break;
-
-    } else {
-      c->read_cluster_vc->current_cont = c;
-    }
-    ink_release_assert(c->read_cluster_vc != CLUSTER_DELAYED_OPEN);
-
-    CacheKey key(msg->md5);
-
-    char *hostname = nullptr;
-    int host_len   = len - op_to_sizeof_fixedlen_msg(opcode);
-    if (host_len) {
-      hostname = (char *)msg->moi.byte;
-    }
-
-    Cache *call_cache = caches[c->frag_type];
-    Action *a =
-      call_cache->open_write(c, &key, c->frag_type, !!(c->cfl_flags & CFL_OVERWRITE_ON_WRITE), c->pin_in_cache, hostname, host_len);
-    if (a != ACTION_RESULT_DONE) {
-      c->cache_action = a;
-    }
-    break;
-  }
-  case CACHE_OPEN_WRITE_LONG: {
-    // Cache needs message data, copy it.
-    c->setMsgBufferLen(len);
-    c->allocMsgBuffer();
-    memcpy(c->getMsgBuffer(), (char *)data, len);
-
-    int flen             = CacheOpMsg_long::sizeof_fixedlen_msg();
-    CacheOpMsg_long *msg = unmarshal_CacheOpMsg_long(c->getMsgBuffer(), mh->NeedByteSwap());
-    init_from_long(c, msg, ch->machine);
-    Debug("cache_msg", "cache_op-l op=%d seqno=%d data=%p len=%d machine=%p", opcode, c->seq_number, data, len, ch->machine);
-#ifdef CACHE_MSG_TRACE
-    log_cache_op_msg(msg->seq_number, len, "cache_op_open_write_long");
-#endif
-    //
-    // Establish the remote side of the ClusterVConnection
-    //
-    c->read_cluster_vc = clusterProcessor.connect_local((Continuation *)nullptr, &c->token, c->cluster_vc_channel,
-                                                        (CLUSTER_OPT_IMMEDIATE | CLUSTER_OPT_CONN_WRITE));
-    if (!c->read_cluster_vc) {
-      // Unable to setup channel, abort processing.
-      CLUSTER_INCREMENT_DYN_STAT(CLUSTER_CHAN_INUSE_STAT);
-      Debug("chan_inuse", "4Remote chan=%d inuse tok.ip=%u.%u.%u.%u tok.seqno=%d seqno=%d", c->cluster_vc_channel,
-            DOT_SEPARATED(c->token.ip_created), c->token.sequence_number, c->seq_number);
-
-      // Send cluster op failed reply
-      c->replyOpEvent(CACHE_EVENT_OPEN_WRITE_FAILED, (VConnection *)-ECLUSTER_CHANNEL_INUSE);
-      break;
-
-    } else {
-      c->read_cluster_vc->current_cont = c;
-    }
-    ink_release_assert(c->read_cluster_vc != CLUSTER_DELAYED_OPEN);
-
-    CacheHTTPInfo *ci = nullptr;
-    const char *p     = (const char *)msg + flen;
-    int res           = 0;
-    int moi_len       = len - flen;
-
-    if (moi_len && c->cfl_flags & CFL_LOPENWRITE_HAVE_OLDINFO) {
-      // Unmarshal old CacheHTTPInfo
-      res = HTTPInfo::unmarshal((char *)p, moi_len, nullptr);
-      ink_assert(res > 0);
-      c->ic_old_info.get_handle((char *)p, moi_len);
-      ink_assert(c->ic_old_info.valid());
-      ci = &c->ic_old_info;
-    }
-    if (c->cfl_flags & CFL_ALLOW_MULTIPLE_WRITES) {
-      ink_assert(!ci);
-      ci = (CacheHTTPInfo *)CACHE_ALLOW_MULTIPLE_WRITES;
-    }
-    moi_len -= res;
-    p += res;
-
-    CacheKey key(msg->url_md5);
-    char *hostname = nullptr;
-
-    if (moi_len) {
-      hostname = (char *)p;
-    }
-
-    Cache *call_cache = caches[c->frag_type];
-    Action *a         = call_cache->open_write(c, &key, ci, c->pin_in_cache, nullptr, c->frag_type, hostname, moi_len);
-    if (a != ACTION_RESULT_DONE) {
-      c->cache_action = a;
-    }
-    break;
-  }
-  case CACHE_REMOVE: {
-    CacheOpMsg_short *msg = unmarshal_CacheOpMsg_short(data, mh->NeedByteSwap());
-    init_from_short(c, msg, ch->machine);
-    Debug("cache_msg", "cache_op op=%d seqno=%d data=%p len=%d machine=%p", opcode, c->seq_number, data, len, ch->machine);
-#ifdef CACHE_MSG_TRACE
-    log_cache_op_msg(msg->seq_number, len, "cache_op_remove");
-#endif
-    CacheKey key(msg->md5);
-
-    char *hostname = nullptr;
-    int host_len   = len - op_to_sizeof_fixedlen_msg(opcode);
-    if (host_len) {
-      hostname = (char *)msg->moi.byte;
-    }
-
-    Cache *call_cache = caches[c->frag_type];
-    Action *a         = call_cache->remove(c, &key, c->frag_type, hostname, host_len);
-    if (a != ACTION_RESULT_DONE) {
-      c->cache_action = a;
-    }
-    break;
-  }
-  case CACHE_LINK: {
-    CacheOpMsg_short_2 *msg = unmarshal_CacheOpMsg_short_2(data, mh->NeedByteSwap());
-    init_from_short_2(c, msg, ch->machine);
-    Debug("cache_msg", "cache_op op=%d seqno=%d data=%p len=%d machine=%p", opcode, c->seq_number, data, len, ch->machine);
-#ifdef CACHE_MSG_TRACE
-    log_cache_op_msg(msg->seq_number, len, "cache_op_link");
-#endif
-
-    CacheKey key1(msg->md5_1);
-    CacheKey key2(msg->md5_2);
-
-    char *hostname = nullptr;
-    int host_len   = len - op_to_sizeof_fixedlen_msg(opcode);
-    if (host_len) {
-      hostname = (char *)msg->moi.byte;
-    }
-
-    Cache *call_cache = caches[c->frag_type];
-    Action *a         = call_cache->link(c, &key1, &key2, c->frag_type, hostname, host_len);
-    if (a != ACTION_RESULT_DONE) {
-      c->cache_action = a;
-    }
-    break;
-  }
-  case CACHE_DEREF: {
-    CacheOpMsg_short *msg = unmarshal_CacheOpMsg_short(data, mh->NeedByteSwap());
-    init_from_short(c, msg, ch->machine);
-    Debug("cache_msg", "cache_op op=%d seqno=%d data=%p len=%d machine=%p", opcode, c->seq_number, data, len, ch->machine);
-#ifdef CACHE_MSG_TRACE
-    log_cache_op_msg(msg->seq_number, len, "cache_op_deref");
-#endif
-
-    CacheKey key(msg->md5);
-
-    char *hostname = nullptr;
-    int host_len   = len - op_to_sizeof_fixedlen_msg(opcode);
-    if (host_len) {
-      hostname = (char *)msg->moi.byte;
-    }
-
-    Cache *call_cache = caches[c->frag_type];
-    Action *a         = call_cache->deref(c, &key, c->frag_type, hostname, host_len);
-    if (a != ACTION_RESULT_DONE) {
-      c->cache_action = a;
-    }
-    break;
-  }
-
-  default: {
-    ink_release_assert(0);
-  }
-  } // End of switch
-}
-
-void
-cache_op_malloc_ClusterFunction(ClusterHandler *ch, void *data, int len)
-{
-  cache_op_ClusterFunction(ch, data, len);
-  // We own the message data, free it back to the Cluster subsystem
-  clusterProcessor.free_remote_data((char *)data, len);
-}
-
-int
-CacheContinuation::setupVCdataRead(int event, VConnection *vc)
-{
-  ink_assert(magicno == (int)MagicNo);
-  //
-  // Setup the initial data read for the given Cache VC.
-  // This data is sent back in the response message.
-  //
-  if (event == CACHE_EVENT_OPEN_READ) {
-    //////////////////////////////////////////
-    // Allocate buffer and initiate read.
-    //////////////////////////////////////////
-    Debug("cache_proto", "setupVCdataRead CACHE_EVENT_OPEN_READ seqno=%d", seq_number);
-    ink_release_assert(caller_buf_freebytes);
-    SET_HANDLER((CacheContHandler)&CacheContinuation::VCdataRead);
-
-    int64_t size_index = iobuffer_size_to_index(caller_buf_freebytes);
-    MIOBuffer *buf     = new_MIOBuffer(size_index);
-    readahead_reader   = buf->alloc_reader();
-
-    MUTEX_TRY_LOCK(lock, mutex, this_ethread()); // prevent immediate callback
-    readahead_vio = vc->do_io_read(this, caller_buf_freebytes, buf);
-    return EVENT_DONE;
-
-  } else {
-    // Error case, deflect processing to replyOpEvent.
-    SET_HANDLER((CacheContHandler)&CacheContinuation::replyOpEvent);
-    return handleEvent(event, vc);
-  }
-}
-
-int
-CacheContinuation::VCdataRead(int event, VIO *target_vio)
-{
-  ink_release_assert(magicno == (int)MagicNo);
-  ink_release_assert(readahead_vio == target_vio);
-
-  VConnection *vc = target_vio->vc_server;
-  int reply       = CACHE_EVENT_OPEN_READ;
-  int32_t object_size;
-
-  switch (event) {
-  case VC_EVENT_EOS: {
-    if (!target_vio->ndone) {
-      // Doc with zero byte body, handle as read failure
-      goto read_failed;
-    }
-    // Fall through
-  }
-  case VC_EVENT_READ_READY:
-  case VC_EVENT_READ_COMPLETE: {
-    int clone_bytes;
-    int current_ndone = target_vio->ndone;
-
-    ink_assert(current_ndone);
-    ink_assert(current_ndone <= readahead_reader->read_avail());
-
-    object_size   = getObjectSize(vc, request_opcode, &cache_vc_info);
-    have_all_data = ((object_size <= caller_buf_freebytes) && (object_size == current_ndone));
-
-    // Use no more than the caller's max buffer limit
-
-    clone_bytes = current_ndone;
-    if (!have_all_data) {
-      if (current_ndone > caller_buf_freebytes) {
-        clone_bytes = caller_buf_freebytes;
-      }
-    }
-    // Clone data
-
-    IOBufferBlock *tail;
-    readahead_data =
-      clone_IOBufferBlockList(readahead_reader->get_current_block(), readahead_reader->start_offset, clone_bytes, &tail);
-
-    if (have_all_data) {
-      // Close VC, since no more data and also to avoid VC_EVENT_EOS
-
-      MIOBuffer *mbuf = target_vio->buffer.writer();
-      vc->do_io(VIO::CLOSE);
-      free_MIOBuffer(mbuf);
-      readahead_vio = nullptr;
-    }
-    SET_HANDLER((CacheContHandler)&CacheContinuation::replyOpEvent);
-    handleEvent(reply, vc);
-    return EVENT_CONT;
-  }
-  case VC_EVENT_ERROR:
-  case VC_EVENT_INACTIVITY_TIMEOUT:
-  case VC_EVENT_ACTIVE_TIMEOUT:
-  default: {
-  read_failed:
-    // Read failed, deflect to replyOpEvent.
-
-    MIOBuffer *mbuf = target_vio->buffer.writer();
-    vc->do_io(VIO::CLOSE);
-    free_MIOBuffer(mbuf);
-    readahead_vio = nullptr;
-    reply         = CACHE_EVENT_OPEN_READ_FAILED;
-
-    SET_HANDLER((CacheContHandler)&CacheContinuation::replyOpEvent);
-    handleEvent(reply, (VConnection *)-ECLUSTER_ORB_DATA_READ);
-    return EVENT_DONE;
-  }
-  } // End of switch
-}
-
-int
-CacheContinuation::setupReadWriteVC(int event, VConnection *vc)
-{
-  // Only handles OPEN_READ_LONG processing.
-
-  switch (event) {
-  case CACHE_EVENT_OPEN_READ: {
-    // setup readahead
-
-    SET_HANDLER((CacheContHandler)&CacheContinuation::setupVCdataRead);
-    return handleEvent(event, vc);
-    break;
-  }
-  case CACHE_EVENT_OPEN_READ_FAILED: {
-    if (frag_type == CACHE_FRAG_TYPE_HTTP && !request_purge) {
-      // HTTP open read failed, attempt open write now to avoid an additional
-      //  message round trip
-
-      CacheKey key(url_md5);
-
-      Cache *call_cache = caches[frag_type];
-      Action *a         = call_cache->open_write(this, &key, nullptr, pin_in_cache, nullptr, frag_type,
-                                         ic_hostname ? ic_hostname->data() : nullptr, ic_hostname_len);
-      if (a != ACTION_RESULT_DONE) {
-        cache_action = a;
-      }
-    } else {
-      SET_HANDLER((CacheContHandler)&CacheContinuation::replyOpEvent);
-      return handleEvent(CACHE_EVENT_OPEN_READ_FAILED, nullptr);
-    }
-    break;
-  }
-  case CACHE_EVENT_OPEN_WRITE: {
-    // Convert from read to write connection
-
-    ink_assert(!read_cluster_vc && write_cluster_vc);
-    read_cluster_vc = write_cluster_vc;
-    read_cluster_vc->set_type(CLUSTER_OPT_CONN_WRITE);
-    write_cluster_vc = nullptr;
-
-    SET_HANDLER((CacheContHandler)&CacheContinuation::replyOpEvent);
-    return handleEvent(event, vc);
-    break;
-  }
-  case CACHE_EVENT_OPEN_WRITE_FAILED:
-  default: {
-    SET_HANDLER((CacheContHandler)&CacheContinuation::replyOpEvent);
-    return handleEvent(CACHE_EVENT_OPEN_READ_FAILED, nullptr);
-    break;
-  }
-  } // end of switch
-
-  return EVENT_DONE;
-}
-
-/////////////////////////////////////////////////////////////////////////
-// replyOpEvent()
-//   Reflect the (local) reply back to the (remote) requesting node.
-/////////////////////////////////////////////////////////////////////////
-int
-CacheContinuation::replyOpEvent(int event, VConnection *cvc)
-{
-  ink_assert(magicno == (int)MagicNo);
-  Debug("cache_proto", "replyOpEvent(this=%p,event=%d,VC=%p)", this, event, cvc);
-  ink_hrtime now;
-  now = Thread::get_hrtime();
-  CLUSTER_SUM_DYN_STAT(CLUSTER_CACHE_CALLBACK_TIME_STAT, now - start_time);
-  LOG_EVENT_TIME(start_time, callback_time_dist, cache_callbacks);
-  ink_release_assert(expect_cache_callback);
-  expect_cache_callback = false; // make sure we are called back exactly once
-
-  result                        = event;
-  bool open                     = event_is_open(event);
-  bool read_op                  = op_is_read(request_opcode);
-  bool open_read_now_open_write = false;
-
-  // Reply message initializations
-  CacheOpReplyMsg rmsg;
-  CacheOpReplyMsg *msg = &rmsg;
-  msg->result          = event;
-
-  if ((request_opcode == CACHE_OPEN_READ_LONG) && cvc && (event == CACHE_EVENT_OPEN_WRITE)) {
-    //////////////////////////////////////////////////////////////////////////
-    // open read failed, but open write succeeded, set result to
-    // CACHE_EVENT_OPEN_READ_FAILED and make result token non zero to
-    // signal to the remote node that we have established a write connection.
-    //////////////////////////////////////////////////////////////////////////
-    msg->result              = CACHE_EVENT_OPEN_READ_FAILED;
-    open_read_now_open_write = true;
-  }
-
-  msg->seq_number = seq_number;
-  int flen        = CacheOpReplyMsg::sizeof_fixedlen_msg(); // include token
-  int len         = 0;
-  int vers        = 0;
-
-  int results_expected = 1;
-
-  if (no_reply_message) // CACHE_NO_RESPONSE request
-    goto free_exit;
-
-  if (open) {
-    // prepare for CACHE_OPEN_EVENT
-
-    results_expected = 2;
-    cache_vc         = cvc;
-    cache_read       = (event == CACHE_EVENT_OPEN_READ);
-
-    if (read_op && !open_read_now_open_write) {
-      ink_release_assert(write_cluster_vc->pending_remote_fill);
-      ink_assert(have_all_data || (readahead_vio == &((CacheVC *)cache_vc)->vio));
-      Debug("cache_proto", "connect_local success seqno=%d have_all_data=%d", seq_number, (have_all_data ? 1 : 0));
-
-      if (have_all_data) {
-        msg->token.clear(); // Tell sender no conn established
-        write_cluster_vc->type = VC_CLUSTER_WRITE;
-      } else {
-        msg->token = token; // Tell sender conn established
-        setupReadBufTunnel(cache_vc, write_cluster_vc);
-      }
-
-    } else {
-      Debug("cache_proto", "cache_open [%s] success seqno=%d", (cache_read ? "R" : "W"), seq_number);
-      msg->token = token; // Tell sender conn established
-
-      OneWayTunnel *pOWT = OneWayTunnel::OneWayTunnel_alloc();
-      pOWT->init(read_cluster_vc, cache_vc, nullptr, nbytes ? nbytes : DEFAULT_MAX_BUFFER_SIZE, this->mutex.get());
-      read_cluster_vc->allow_remote_close();
-      results_expected--;
-    }
-
-    // For cache reads, marshal the associated CacheHTTPInfo in the reply
-    if (cache_read) {
-      int res;
-
-      msg->is_ram_cache_hit = ((CacheVC *)cache_vc)->is_ram_cache_hit();
-
-      if (!cache_vc_info.valid()) {
-        (void)getObjectSize(cache_vc, request_opcode, &cache_vc_info);
-      }
-      // Determine data length and allocate
-      len                    = cache_vc_info.marshal_length();
-      CacheOpReplyMsg *reply = (CacheOpReplyMsg *)ALLOCA_DOUBLE(flen + len);
-
-      // Initialize reply message header
-      *reply = *msg;
-
-      // Marshal response data into reply message
-      res = cache_vc_info.marshal((char *)reply + flen, len);
-      ink_assert(res >= 0 && res <= len);
-
-      // Make reply message the current message
-      msg = reply;
-    }
-
-  } else {
-    Debug("cache_proto", "cache operation failed result=%d seqno=%d (this=%p)", event, seq_number, this);
-    msg->token.clear(); // Tell sender no conn established
-
-    // Reallocate reply message, allowing for marshalled data
-    len += sizeof(int32_t);
-    CacheOpReplyMsg *reply = (CacheOpReplyMsg *)ALLOCA_DOUBLE(flen + len);
-
-    // Initialize reply message header
-    *reply = *msg;
-
-    if (request_opcode != CACHE_LINK) {
-      //
-      // open read/write failed, close preallocated VC
-      //
-      if (read_cluster_vc) {
-        read_cluster_vc->remote_closed = 1; // avoid remote close msg
-        read_cluster_vc->do_io(VIO::CLOSE);
-      }
-      if (write_cluster_vc) {
-        write_cluster_vc->pending_remote_fill = 0;
-        write_cluster_vc->remote_closed       = 1; // avoid remote close msg
-        write_cluster_vc->do_io(VIO::CLOSE);
-      }
-      reply->moi.u32 = (int32_t)((uintptr_t)cvc & 0xffffffff); // code describing failure
-    }
-    // Make reply message the current message
-    msg = reply;
-  }
-  CLUSTER_DECREMENT_DYN_STAT(CLUSTER_CACHE_OUTSTANDING_STAT);
-
-//
-// Send reply message
-//
-#ifdef CACHE_MSG_TRACE
-  log_cache_op_sndmsg(msg->seq_number, 0, "replyOpEvent");
-#endif
-  vers = CacheOpReplyMsg::protoToVersion(from->msg_proto_major);
-  if (vers == CacheOpReplyMsg::CACHE_OP_REPLY_MESSAGE_VERSION) {
-    if (read_op) {
-      // Transmit reply message and object data in same cluster message
-      Debug("cache_proto", "Sending reply/data seqno=%d buflen=%" PRId64, seq_number,
-            readahead_data ? bytes_IOBufferBlockList(readahead_data, 1) : 0);
-      clusterProcessor.invoke_remote_data(ch, CACHE_OP_RESULT_CLUSTER_FUNCTION, (void *)msg, (flen + len), readahead_data.get(),
-                                          cluster_vc_channel, &token, &CacheContinuation::disposeOfDataBuffer, (void *)this,
-                                          CLUSTER_OPT_STEAL);
-    } else {
-      Debug("cache_proto", "Sending reply seqno=%d, (this=%p)", seq_number, this);
-      clusterProcessor.invoke_remote(ch, CACHE_OP_RESULT_CLUSTER_FUNCTION, (void *)msg, (flen + len), CLUSTER_OPT_STEAL);
-    }
-
-  } else {
-    //////////////////////////////////////////////////////////////
-    // Create the specified down rev version of this message
-    //////////////////////////////////////////////////////////////
-    ink_release_assert(!"replyOpEvent() bad msg version");
-  }
-
-free_exit:
-  results_expected--;
-  if (results_expected <= 0) {
-    Debug("cache_proto", "replyOpEvent: freeing this=%p", this);
-    cacheContAllocator_free(this);
-  }
-  return EVENT_DONE;
-}
-
-void
-CacheContinuation::setupReadBufTunnel(VConnection *cache_read_vc, VConnection *cluster_write_vc)
-{
-  ////////////////////////////////////////////////////////////
-  // Setup OneWayTunnel and tunnel close event handler.
-  // Used in readahead processing on open read connections.
-  ////////////////////////////////////////////////////////////
-  tunnel_cont        = cacheContAllocator_alloc();
-  tunnel_cont->mutex = this->mutex;
-  SET_CONTINUATION_HANDLER(tunnel_cont, (CacheContHandler)&CacheContinuation::tunnelClosedEvent);
-  int64_t ravail = bytes_IOBufferBlockList(readahead_data, 1);
-
-  tunnel_mutex  = tunnel_cont->mutex;
-  tunnel_closed = false;
-
-  tunnel = OneWayTunnel::OneWayTunnel_alloc();
-  readahead_reader->consume(ravail); // allow for bytes sent in initial reply
-  tunnel->init(cache_read_vc, cluster_write_vc, tunnel_cont, readahead_vio, readahead_reader);
-  tunnel_cont->action      = this;
-  tunnel_cont->tunnel      = tunnel;
-  tunnel_cont->tunnel_cont = tunnel_cont;
-
-  // Disable cluster_write_vc
-  ((ClusterVConnection *)cluster_write_vc)->write.enabled = 0;
-
-  // Disable cache read VC
-  readahead_vio->nbytes = readahead_vio->ndone;
-
-  /////////////////////////////////////////////////////////////////////
-  // At this point, the OneWayTunnel is blocked awaiting a reenable
-  // on both the source and target VCs. Reenable occurs after the
-  // message containing the initial data and open read reply are sent.
-  /////////////////////////////////////////////////////////////////////
-}
-
-///////////////////////////////////////////////////////////////////////
-// Tunnnel exited event handler, used for readahead on open read.
-///////////////////////////////////////////////////////////////////////
-int
-CacheContinuation::tunnelClosedEvent(int /* event ATS_UNUSED */, void *c)
-{
-  ink_assert(magicno == (int)MagicNo);
-  // Note: We are called with the tunnel_mutex held.
-  CacheContinuation *tc = (CacheContinuation *)c;
-  ink_release_assert(tc->tunnel_cont == tc);
-  CacheContinuation *real_cc = (CacheContinuation *)tc->action.continuation;
-
-  if (real_cc) {
-    // Notify the real continuation of the tunnel closed event
-    real_cc->tunnel        = nullptr;
-    real_cc->tunnel_cont   = nullptr;
-    real_cc->tunnel_closed = true;
-  }
-  OneWayTunnel::OneWayTunnel_free(tc->tunnel);
-  cacheContAllocator_free(tc);
-
-  return EVENT_DONE;
-}
-
-////////////////////////////////////////////////////////////
-// Retry DisposeOfDataBuffer continuation
-////////////////////////////////////////////////////////////
-struct retryDisposeOfDataBuffer;
-typedef int (retryDisposeOfDataBuffer::*rtryDisOfDBufHandler)(int, void *);
-struct retryDisposeOfDataBuffer : public Continuation {
-  CacheContinuation *c;
-
-  int
-  handleRetryEvent(int event, Event *e)
-  {
-    if (CacheContinuation::handleDisposeEvent(event, c) == EVENT_DONE) {
-      delete this;
-      return EVENT_DONE;
-    } else {
-      e->schedule_in(HRTIME_MSECONDS(10));
-      return EVENT_CONT;
-    }
-  }
-  retryDisposeOfDataBuffer(CacheContinuation *cont) : Continuation(new_ProxyMutex()), c(cont)
-  {
-    SET_HANDLER((rtryDisOfDBufHandler)&retryDisposeOfDataBuffer::handleRetryEvent);
-  }
-};
-
-//////////////////////////////////////////////////////////////////
-// Callback from cluster to dispose of data passed in
-// call to invoke_remote_data().
-//////////////////////////////////////////////////////////////////
-void
-CacheContinuation::disposeOfDataBuffer(void *d)
-{
-  ink_assert(d);
-  CacheContinuation *cc = (CacheContinuation *)d;
-  ink_assert(cc->have_all_data || cc->readahead_vio);
-  ink_assert(cc->have_all_data || (cc->readahead_vio == &((CacheVC *)cc->cache_vc)->vio));
-
-  if (cc->have_all_data) {
-    //
-    // All object data resides in the buffer, no OneWayTunnel
-    // started and the Cache VConnection has already been closed.
-    // Close write_cluster_vc and set remote close to avoid send of
-    // close message to remote node.
-    //
-    cc->write_cluster_vc->pending_remote_fill = 0;
-    cc->write_cluster_vc->remote_closed       = 1;
-    cc->write_cluster_vc->do_io(VIO::CLOSE);
-    cc->readahead_data = nullptr;
-
-    cacheContAllocator_free(cc);
-
-  } else {
-    cc->write_cluster_vc->pending_remote_fill = 0;
-    cc->write_cluster_vc->allow_remote_close();
-    if (handleDisposeEvent(0, cc) == EVENT_CONT) {
-      // Setup retry continuation.
-      retryDisposeOfDataBuffer *retryCont = new retryDisposeOfDataBuffer(cc);
-      eventProcessor.schedule_in(retryCont, HRTIME_MSECONDS(10), ET_CALL);
-    }
-  }
-}
-
-int
-CacheContinuation::handleDisposeEvent(int /* event ATS_UNUSED */, CacheContinuation *cc)
-{
-  ink_assert(cc->magicno == (int)MagicNo);
-  MUTEX_TRY_LOCK(lock, cc->tunnel_mutex, this_ethread());
-  if (lock.is_locked()) {
-    // Write of initial object data is complete.
-
-    if (!cc->tunnel_closed) {
-      // Start tunnel by reenabling source and target VCs.
-
-      cc->tunnel->vioSource->nbytes = getObjectSize(cc->tunnel->vioSource->vc_server, cc->request_opcode, nullptr);
-      cc->tunnel->vioSource->reenable_re();
-
-      // Tunnel may be closed by vioSource->reenable_re(),
-      // we should check it again here:
-      if (!cc->tunnel_closed) {
-        cc->tunnel->vioTarget->reenable();
-
-        // Tell tunnel event we are gone
-        cc->tunnel_cont->action.continuation = nullptr;
-      }
-    }
-    cacheContAllocator_free(cc);
-    return EVENT_DONE;
-
-  } else {
-    // Lock acquire failed, retry operation.
-    return EVENT_CONT;
-  }
-}
-
-/////////////////////////////////////////////////////////////////////////////
-// cache_op_result_ClusterFunction()
-//   Invoked on the machine which initiated a remote op, this
-//   unmarshals the result and calls a continuation in the requesting thread.
-/////////////////////////////////////////////////////////////////////////////
-void
-cache_op_result_ClusterFunction(ClusterHandler *ch, void *d, int l)
-{
-  ////////////////////////////////////////////////////////
-  // Note: we are running on the ET_CACHE_CONT_SM thread
-  ////////////////////////////////////////////////////////
-
-  // Copy reply message data
-  Ptr<IOBufferData> iob = make_ptr(new_IOBufferData(iobuffer_size_to_index(l)));
-  memcpy(iob->data(), (char *)d, l);
-  char *data = iob->data();
-  int flen, len = l;
-  CacheHTTPInfo ci;
-  CacheOpReplyMsg *msg     = (CacheOpReplyMsg *)data;
-  int32_t op_result_error  = 0;
-  ClusterMessageHeader *mh = (ClusterMessageHeader *)data;
-
-  if (mh->GetMsgVersion() != CacheOpReplyMsg::CACHE_OP_REPLY_MESSAGE_VERSION) { ////////////////////////////////////////////////
-    // Convert from old to current message format
-    ////////////////////////////////////////////////
-    ink_release_assert(!"cache_op_result_ClusterFunction() bad msg version");
-  }
-
-  flen = CacheOpReplyMsg::sizeof_fixedlen_msg();
-  if (mh->NeedByteSwap())
-    msg->SwapBytes();
-
-  Debug("cluster_cache", "received cache op result, seqno=%d result=%d", msg->seq_number, msg->result);
-
-  // If applicable, unmarshal any response data
-  if ((len > flen) && event_reply_may_have_moi(msg->result)) {
-    switch (msg->result) {
-    case CACHE_EVENT_OPEN_READ: {
-      char *p = (char *)msg + flen;
-      int res;
-
-      // Unmarshal CacheHTTPInfo
-      res = HTTPInfo::unmarshal(p, len, nullptr);
-      ci.get_handle(p, len);
-      ink_assert(res > 0);
-      ink_assert(ci.valid());
-      break;
-    }
-    case CACHE_EVENT_LINK:
-    case CACHE_EVENT_LINK_FAILED:
-      break;
-    case CACHE_EVENT_OPEN_READ_FAILED:
-    case CACHE_EVENT_OPEN_WRITE_FAILED:
-    case CACHE_EVENT_REMOVE_FAILED:
-    case CACHE_EVENT_UPDATE_FAILED:
-    case CACHE_EVENT_DEREF_FAILED: {
-      // Unmarshal the error code
-      ink_assert(((len - flen) == sizeof(int32_t)));
-      op_result_error = msg->moi.u32;
-      if (mh->NeedByteSwap())
-        ats_swap32((uint32_t *)&op_result_error);
-      op_result_error = -op_result_error;
-      break;
-    }
-    default: {
-      ink_release_assert(!"invalid moi data for received msg");
-      break;
-    }
-    } // end of switch
-  }
-  // See if this response is still expected (expected case == yes)
-
-  unsigned int hash = FOLDHASH(ch->machine->ip, msg->seq_number);
-  EThread *thread   = this_ethread();
-  ProxyMutex *mutex = thread->mutex.get();
-  if (MUTEX_TAKE_TRY_LOCK(remoteCacheContQueueMutex[hash], thread)) {
-    // Find it in pending list
-
-    CacheContinuation *c = find_cache_continuation(msg->seq_number, ch->machine->ip);
-    if (!c) {
-      // Reply took to long, response no longer expected.
-      MUTEX_UNTAKE_LOCK(remoteCacheContQueueMutex[hash], thread);
-      Debug("cluster_timeout", "0cache reply timeout: %d", msg->seq_number);
-      CLUSTER_INCREMENT_DYN_STAT(CLUSTER_REMOTE_OP_REPLY_TIMEOUTS_STAT);
-      if (ci.valid())
-        ci.destroy();
-      return;
-    }
-
-    // Update remote ram cache hit flag
-    if (msg->result == CACHE_EVENT_OPEN_READ)
-      c->read_cluster_vc->set_ram_cache_hit(msg->is_ram_cache_hit);
-
-    // Try to send the message
-
-    MUTEX_TRY_LOCK(lock, c->mutex, thread);
-
-    // Failed to acquire lock, defer
-
-    if (!lock.is_locked()) {
-      MUTEX_UNTAKE_LOCK(remoteCacheContQueueMutex[hash], thread);
-      goto Lretry;
-    }
-    c->result_error = op_result_error;
-
-    // send message, release lock
-
-    c->freeMsgBuffer();
-    if (ci.valid()) {
-      // Unmarshaled CacheHTTPInfo contained in reply message, copy it.
-      c->setMsgBufferLen(len, iob.get());
-      c->ic_new_info = ci;
-    }
-    msg->seq_number = len; // HACK ALERT: reusing variable
-    c->handleEvent(CACHE_EVENT_RESPONSE_MSG, data);
-
-  } else {
-  // Failed to wake it up, defer by creating a timed continuation
-
-  Lretry:
-    CacheContinuation *c = CacheContinuation::cacheContAllocator_alloc();
-    c->mutex             = new_ProxyMutex();
-    c->seq_number        = msg->seq_number;
-    c->target_ip         = ch->machine->ip;
-    SET_CONTINUATION_HANDLER(c, (CacheContHandler)&CacheContinuation::handleReplyEvent);
-    c->start_time = Thread::get_hrtime();
-    c->result     = msg->result;
-    if (event_is_open(msg->result))
-      c->token = msg->token;
-    if (ci.valid()) {
-      // Unmarshaled CacheHTTPInfo contained in reply message, copy it.
-      c->setMsgBufferLen(len, iob.get());
-      c->ic_new_info = ci;
-    }
-    c->result_error = op_result_error;
-    eventProcessor.schedule_in(c, CACHE_RETRY_PERIOD, ET_CACHE_CONT_SM);
-  }
-}
-
-////////////////////////////////////////////////////////////////////////
-// handleReplyEvent()
-//   If we cannot acquire any of the locks to handle the response
-//   inline, it is defered and later handled by this function.
-////////////////////////////////////////////////////////////////////////
-int
-CacheContinuation::handleReplyEvent(int event, Event *e)
-{
-  (void)event;
-
-  // take lock on outstanding message queue
-
-  EThread *t        = e->ethread;
-  unsigned int hash = FOLDHASH(target_ip, seq_number);
-
-  if (!MUTEX_TAKE_TRY_LOCK(remoteCacheContQueueMutex[hash], t)) {
-    e->schedule_in(CACHE_RETRY_PERIOD);
-    return EVENT_CONT;
-  }
-
-  LOG_EVENT_TIME(start_time, cntlck_acquire_time_dist, cntlck_acquire_events);
-
-  // See if this response is still expected
-
-  CacheContinuation *c = find_cache_continuation(seq_number, target_ip);
-  if (c) {
-    // Acquire the lock to the continuation mutex
-
-    MUTEX_TRY_LOCK(lock, c->mutex, e->ethread);
-    if (!lock.is_locked()) {
-      // If we fail to acquire the lock, reschedule
-
-      MUTEX_UNTAKE_LOCK(remoteCacheContQueueMutex[hash], t);
-      e->schedule_in(CACHE_RETRY_PERIOD);
-      return EVENT_CONT;
-    }
-
-    // If unmarshalled CacheHTTPInfo exists, pass it along
-
-    if (ic_new_info.valid()) {
-      c->freeMsgBuffer();
-      c->setMsgBufferLen(getMsgBufferLen(), getMsgBufferIOBData());
-      c->ic_new_info = ic_new_info;
-      ic_new_info.clear();
-    }
-    // send message, release lock
-
-    c->handleEvent(CACHE_EVENT_RESPONSE, this);
-
-  } else {
-    MUTEX_UNTAKE_LOCK(remoteCacheContQueueMutex[hash], t);
-    Debug("cluster_timeout", "cache reply timeout: %d", seq_number);
-    CLUSTER_INCREMENT_DYN_STAT(CLUSTER_REMOTE_OP_REPLY_TIMEOUTS_STAT);
-  }
-
-  // Free this continuation
-
-  cacheContAllocator_free(this);
-  return EVENT_DONE;
-}
-
-//////////////////////////////////////////////////////////////////////////
-// remoteOpEvent()
-//   On the requesting node, handle the timeout and response to the user.
-//   There may be two CacheContinuations involved:
-//    1) One waiting to respond to the user.
-//       This case is CACHE_EVENT_RESPONSE_MSG which is handled
-//       inline (without delay).
-//    2) One which is carrying the response from the remote machine which
-//       has been delayed for a lock.  This case is CACHE_EVENT_RESPONSE.
-//////////////////////////////////////////////////////////////////////////
-int
-CacheContinuation::remoteOpEvent(int event_code, Event *e)
-{
-  ink_assert(magicno == (int)MagicNo);
-  int event = event_code;
-  ink_hrtime now;
-  if (start_time) {
-    int res;
-    if (event != EVENT_INTERVAL) {
-      if (event == CACHE_EVENT_RESPONSE) {
-        CacheContinuation *ccont = (CacheContinuation *)e;
-        res                      = ccont->result;
-      } else {
-        CacheOpReplyMsg *rmsg = (CacheOpReplyMsg *)e;
-        res                   = rmsg->result;
-      }
-      if ((res == CACHE_EVENT_LOOKUP) || (res == CACHE_EVENT_LOOKUP_FAILED)) {
-        now = Thread::get_hrtime();
-        CLUSTER_SUM_DYN_STAT(CLUSTER_CACHE_LKRMT_CALLBACK_TIME_STAT, now - start_time);
-        LOG_EVENT_TIME(start_time, lkrmt_callback_time_dist, lkrmt_cache_callbacks);
-      } else {
-        now = Thread::get_hrtime();
-        CLUSTER_SUM_DYN_STAT(CLUSTER_CACHE_RMT_CALLBACK_TIME_STAT, now - start_time);
-        LOG_EVENT_TIME(start_time, rmt_callback_time_dist, rmt_cache_callbacks);
-      }
-    }
-    start_time = 0;
-  }
-  // for CACHE_EVENT_RESPONSE/XXX the lock was acquired at the higher level
-  intptr_t return_error  = 0;
-  ClusterVCToken *pToken = nullptr;
-
-retry:
-
-  switch (event) {
-  default:
-    ink_assert(!"bad case");
-    return EVENT_DONE;
-
-  case EVENT_INTERVAL: {
-    unsigned int hash = FOLDHASH(target_ip, seq_number);
-
-    MUTEX_TRY_LOCK(queuelock, remoteCacheContQueueMutex[hash], e->ethread);
-    if (!queuelock.is_locked()) {
-      e->schedule_in(CACHE_RETRY_PERIOD);
-      return EVENT_CONT;
-    }
-    // we are not yet enqueued on the list of outstanding operations
-
-    if (!remoteCacheContQueue[hash].in(this)) {
-      remoteCacheContQueue[hash].enqueue(this);
-      ink_assert(timeout == e);
-      MUTEX_RELEASE(queuelock);
-      e->schedule_in(cache_cluster_timeout);
-      return EVENT_CONT;
-    }
-    // a timeout has occurred
-
-    if (find_cache_continuation(seq_number, target_ip)) {
-      // Valid timeout
-      MUTEX_RELEASE(queuelock);
-
-      Debug("cluster_timeout", "cluster op timeout %d", seq_number);
-      CLUSTER_INCREMENT_DYN_STAT(CLUSTER_REMOTE_OP_TIMEOUTS_STAT);
-      request_timeout = true;
-      timeout         = nullptr;
-      //
-      // Post error completion now and defer deallocation of
-      // the continuation until we receive the reply or the
-      // target node goes down.
-      //
-      if (!action.cancelled)
-        action.continuation->handleEvent(result, (void *)-ECLUSTER_OP_TIMEOUT);
-      action.cancelled = 1;
-
-      if (target_machine->dead) {
-        event = CACHE_EVENT_RESPONSE_MSG;
-        goto retry;
-      } else {
-        timeout = e;
-        e->schedule_in(cache_cluster_timeout);
-        return EVENT_DONE;
-      }
-
-    } else {
-      // timeout not expected for continuation; log and ignore
-      MUTEX_RELEASE(queuelock);
-      Debug("cluster_timeout", "unknown cluster op timeout %d", seq_number);
-      Note("Unexpected CacheCont timeout, [%u.%u.%u.%u] seqno=%d", DOT_SEPARATED(target_ip), seq_number);
-      CLUSTER_INCREMENT_DYN_STAT(CLUSTER_REMOTE_OP_TIMEOUTS_STAT);
-      return EVENT_DONE;
-    }
-  }
-
-  case CACHE_EVENT_RESPONSE:
-  case CACHE_EVENT_RESPONSE_MSG: {
-    // the response has arrived, cancel timeout
-
-    if (timeout) {
-      timeout->cancel();
-      timeout = nullptr;
-    }
-    // remove from the pending queue
-    unsigned int hash = FOLDHASH(target_ip, seq_number);
-
-    remoteCacheContQueue[hash].remove(this);
-    MUTEX_UNTAKE_LOCK(remoteCacheContQueueMutex[hash], this_ethread());
-    // Fall through
-  }
-
-  case CACHE_EVENT_RESPONSE_RETRY: {
-    // determine result code
-
-    CacheContinuation *c = (CacheContinuation *)e;
-    CacheOpReplyMsg *msg = (CacheOpReplyMsg *)e;
-    if (event == CACHE_EVENT_RESPONSE_MSG) {
-      result = (request_timeout ? result : msg->result);
-      pToken = (request_timeout ? &token : &msg->token);
-    } else if (event == CACHE_EVENT_RESPONSE) {
-      result = (request_timeout ? result : c->result);
-      pToken = &c->token;
-    } else if (event == CACHE_EVENT_RESPONSE_RETRY) {
-      pToken = &token;
-    } else {
-      ink_release_assert(!"remoteOpEvent bad event code");
-    }
-
-    // handle response
-
-    if (result == CACHE_EVENT_LOOKUP) {
-      callback_user(result, nullptr);
-      return EVENT_DONE;
-
-    } else if (event_is_open(result)) {
-      bool read_op = ((request_opcode == CACHE_OPEN_READ) || (request_opcode == CACHE_OPEN_READ_LONG));
-      if (read_op) {
-        ink_release_assert(read_cluster_vc->pending_remote_fill > 1);
-        read_cluster_vc->pending_remote_fill = 0;
-
-        have_all_data = pToken->is_clear(); // no conn implies all data
-        if (have_all_data) {
-          read_cluster_vc->have_all_data = true;
-        } else {
-          read_cluster_vc->have_all_data = false;
-        }
-        // Move CacheHTTPInfo reply data into VC
-        read_cluster_vc->marshal_buf = this->getMsgBufferIOBData();
-        read_cluster_vc->alternate   = this->ic_new_info;
-        this->ic_new_info.clear();
-        ink_release_assert(read_cluster_vc->alternate.object_size_get());
-
-        if (!action.cancelled) {
-          ClusterVConnection *target_vc = read_cluster_vc;
-          callback_user(result, target_vc); // "this" is deallocated
-          target_vc->allow_remote_close();
-        } else {
-          read_cluster_vc->allow_remote_close();
-          read_cluster_vc->do_io(VIO::ABORT);
-          cacheContAllocator_free(this);
-        }
-
-      } else {
-        ink_assert(result == CACHE_EVENT_OPEN_WRITE);
-        ink_assert(!pToken->is_clear());
-
-        ClusterVConnection *result_vc = write_cluster_vc;
-        if (!action.cancelled) {
-          callback_user(result, result_vc);
-          result_vc->allow_remote_close();
-        } else {
-          result_vc->allow_remote_close();
-          result_vc->do_io(VIO::ABORT);
-          cacheContAllocator_free(this);
-        }
-      }
-      return EVENT_DONE;
-    }
-    break;
-  } // End of case
-  } // End of switch
-
-  // Handle failure cases
-
-  if (result == CACHE_EVENT_LOOKUP_FAILED) {
-    // check for local probes
-
-    ClusterMachine *m = cluster_machine_at_depth(cache_hash(url_md5));
-
-    // if the current configuration indicates that this
-    // machine is the master (or the owner machine has failed), go to
-    // the local machine.  Also if PROBE_LOCAL_CACHE_LAST.
-    //
-    int len        = getMsgBufferLen();
-    char *hostname = (len ? getMsgBuffer() : nullptr);
-
-    if (!m || PROBE_LOCAL_CACHE_LAST) {
-      SET_HANDLER((CacheContHandler)&CacheContinuation::probeLookupEvent);
-      CacheKey key(url_md5);
-
-      Cache *call_cache = caches[frag_type];
-      call_cache->lookup(this, &key, frag_type, hostname, len);
-      return EVENT_DONE;
-    }
-    if (PROBE_LOCAL_CACHE_FIRST) {
-      callback_user(CACHE_EVENT_LOOKUP_FAILED, nullptr);
-    } else {
-      SET_HANDLER((CacheContHandler)&CacheContinuation::probeLookupEvent);
-      CacheKey key(url_md5);
-
-      Cache *call_cache = caches[frag_type];
-      call_cache->lookup(this, &key, frag_type, hostname, len);
-    }
-    return EVENT_DONE;
-
-  } else {
-    // Handle failure of all ops except for lookup
-
-    ClusterVConnection *cacheable_vc = nullptr;
-    if ((request_opcode == CACHE_OPEN_READ_LONG) && !pToken->is_clear()) {
-      ink_assert(read_cluster_vc && !write_cluster_vc);
-      //
-      // OPEN_READ_LONG has failed, but the remote node was able to
-      // establish an OPEN_WRITE_LONG connection.
-      // Convert the cluster read VC to a write VC and insert it
-      // into the global write VC cache.  This will allow us to
-      // locally resolve the subsequent OPEN_WRITE_LONG request.
-      //
-
-      // Note: We do not allow remote close on this VC while
-      //       it resides in cache
-      //
-      read_cluster_vc->set_type(CLUSTER_OPT_CONN_WRITE);
-      // FIX ME. ajitb 12/21/99
-      // Compiler bug in CC: WorkShop Compilers 5.0 98/12/15 C++ 5.0
-      // Does not accept assignment of ((Continuation *) nullptr)
-      {
-        Continuation *temp       = nullptr;
-        read_cluster_vc->action_ = temp;
-      }
-      if (!GlobalOpenWriteVCcache->insert(&url_md5, read_cluster_vc)) {
-        // Unable to insert VC into cache, try later
-        cacheable_vc = read_cluster_vc;
-      }
-      read_cluster_vc = nullptr;
-    }
-    if (read_cluster_vc) {
-      read_cluster_vc->remote_closed = 0; // send remote close
-      read_cluster_vc->allow_remote_close();
-      read_cluster_vc->do_io(VIO::ABORT);
-      read_cluster_vc = nullptr;
-    }
-    if (write_cluster_vc) {
-      write_cluster_vc->remote_closed = 0; // send remote close
-      write_cluster_vc->allow_remote_close();
-      write_cluster_vc->do_io(VIO::ABORT);
-      write_cluster_vc = nullptr;
-    }
-    if (!request_timeout) {
-      if (!return_error) {
-        return_error = result_error;
-      }
-      if (cacheable_vc) {
-        insert_cache_callback_user(cacheable_vc, result, (void *)return_error);
-      } else {
-        callback_user(result, (void *)return_error);
-      }
-    } else {
-      // callback already made at timeout, just free continuation
-      if (cacheable_vc) {
-        cacheable_vc->allow_remote_close();
-        cacheable_vc->do_io(VIO::CLOSE);
-        cacheable_vc = nullptr;
-      }
-      cacheContAllocator_free(this);
-    }
-    return EVENT_DONE;
-  }
-}
-
-//////////////////////////////////////////////////////////////////////////
-// probeLookupEvent()
-//   After a local probe, return the response to the client and cleanup.
-//////////////////////////////////////////////////////////////////////////
-
-int
-CacheContinuation::probeLookupEvent(int event, void * /* d ATS_UNUSED */)
-{
-  ink_assert(magicno == (int)MagicNo);
-  callback_user(event, nullptr);
-  return EVENT_DONE;
-}
-
-///////////////////////////////////////////////////////////
-// lookupEvent()
-//   Result of a local lookup for PROBE_LOCAL_CACHE_FIRST
-///////////////////////////////////////////////////////////
-int
-CacheContinuation::lookupEvent(int /* event ATS_UNUSED */, void * /* d ATS_UNUSED */)
-{
-  ink_release_assert(!"Invalid call CacheContinuation::lookupEvent");
-  return EVENT_DONE;
-}
-
-//////////////////////////////////////////////////////////////////////////
-// do_remote_lookup()
-//   If the object is supposed to be on a remote machine, probe there.
-//   Returns: Non zero (Action *) if a probe was initiated
-//            Zero (Action *) if no probe
-//////////////////////////////////////////////////////////////////////////
-Action *
-CacheContinuation::do_remote_lookup(Continuation *cont, const CacheKey *key, CacheContinuation *c, CacheFragType ft,
-                                    const char *hostname, int hostname_len)
-{
-  int probe_depth                                                = 0;
-  ClusterMachine *past_probes[CONFIGURATION_HISTORY_PROBE_DEPTH] = {nullptr};
-  int mlen            = op_to_sizeof_fixedlen_msg(CACHE_LOOKUP_OP) + ((hostname && hostname_len) ? hostname_len : 0);
-  CacheLookupMsg *msg = (CacheLookupMsg *)ALLOCA_DOUBLE(mlen);
-  msg->init();
-
-  if (key) {
-    msg->url_md5 = *key;
-  } else {
-    ink_assert(c);
-    msg->url_md5 = c->url_md5;
-  }
-
-  ClusterMachine *m = nullptr;
-
-  if (cache_migrate_on_demand) {
-    m = cluster_machine_at_depth(cache_hash(msg->url_md5), c ? &c->probe_depth : &probe_depth, c ? c->past_probes : past_probes);
-  } else {
-    // If migrate-on-demand is off, do not probe beyond one level.
-
-    if (c && c->probe_depth)
-      return (Action *)nullptr;
-    m = cluster_machine_at_depth(cache_hash(msg->url_md5));
-    if (c)
-      c->probe_depth = 1;
-  }
-
-  if (!m)
-    return (Action *)nullptr;
-  ClusterHandler *ch = m->pop_ClusterHandler();
-  if (!ch)
-    return (Action *)nullptr;
-
-  // If we do not have a continuation, build one
-
-  if (!c) {
-    c              = cacheContAllocator_alloc();
-    c->mutex       = cont->mutex;
-    c->probe_depth = probe_depth;
-    memcpy(c->past_probes, past_probes, sizeof(past_probes));
-  }
-  c->ch = ch;
-  // Save hostname data in case we need to do a local lookup.
-  if (hostname && hostname_len) {
-    // Alloc buffer, copy hostname data and attach to continuation
-    c->setMsgBufferLen(hostname_len);
-    c->allocMsgBuffer();
-    memcpy(c->getMsgBuffer(), hostname, hostname_len);
-  }
-
-  c->url_md5          = msg->url_md5;
-  c->action.cancelled = false;
-  c->action           = cont;
-  c->start_time       = Thread::get_hrtime();
-  SET_CONTINUATION_HANDLER(c, (CacheContHandler)&CacheContinuation::remoteOpEvent);
-  c->result = CACHE_EVENT_LOOKUP_FAILED;
-
-  // set up sequence number so we can find this continuation
-
-  c->target_ip    = m->ip;
-  c->seq_number   = new_cache_sequence_number();
-  msg->seq_number = c->seq_number;
-  c->frag_type    = ft;
-  msg->frag_type  = ft;
-
-  // establish timeout for lookup
-
-  unsigned int hash = FOLDHASH(c->target_ip, c->seq_number);
-  MUTEX_TRY_LOCK(queuelock, remoteCacheContQueueMutex[hash], this_ethread());
-  if (!queuelock.is_locked()) {
-    // failed to acquire lock: no problem, retry later
-    c->timeout = eventProcessor.schedule_in(c, CACHE_RETRY_PERIOD, ET_CACHE_CONT_SM);
-  } else {
-    remoteCacheContQueue[hash].enqueue(c);
-    MUTEX_RELEASE(queuelock);
-    c->timeout = eventProcessor.schedule_in(c, cache_cluster_timeout, ET_CACHE_CONT_SM);
-  }
-
-  char *data;
-  int len;
-  int vers = CacheLookupMsg::protoToVersion(m->msg_proto_major);
-
-  if (vers == CacheLookupMsg::CACHE_LOOKUP_MESSAGE_VERSION) {
-    msg->seq_number = c->seq_number;
-    data            = (char *)msg;
-    len             = mlen;
-    if (hostname && hostname_len) {
-      memcpy(msg->moi.byte, hostname, hostname_len);
-    }
-  } else {
-    //////////////////////////////////////////////////////////////
-    // Create the specified down rev version of this message
-    //////////////////////////////////////////////////////////////
-    ink_release_assert(!"CacheLookupMsg bad msg version");
-  }
-
-// send the message
-
-#ifdef CACHE_MSG_TRACE
-  log_cache_op_sndmsg(msg.seq_number, 0, "cache_lookup");
-#endif
-  clusterProcessor.invoke_remote(c->ch, CACHE_LOOKUP_CLUSTER_FUNCTION, data, len);
-  return &c->action;
-}
-
-////////////////////////////////////////////////////////////////////////////
-// cache_lookup_ClusterFunction()
-//   This function is invoked on a remote machine to do a remote lookup.
-//   It unmarshals the URL and does a local lookup, with its own
-//   continuation set to CacheContinuation::replyLookupEvent()
-////////////////////////////////////////////////////////////////////////////
-void
-cache_lookup_ClusterFunction(ClusterHandler *ch, void *data, int len)
-{
-  (void)len;
-  EThread *thread   = this_ethread();
-  ProxyMutex *mutex = thread->mutex.get();
-  ////////////////////////////////////////////////////////
-  // Note: we are running on the ET_CLUSTER thread
-  ////////////////////////////////////////////////////////
-
-  CacheLookupMsg *msg      = (CacheLookupMsg *)data;
-  ClusterMessageHeader *mh = (ClusterMessageHeader *)data;
-
-  if (mh->GetMsgVersion() != CacheLookupMsg::CACHE_LOOKUP_MESSAGE_VERSION) { ////////////////////////////////////////////////
-    // Convert from old to current message format
-    ////////////////////////////////////////////////
-    ink_release_assert(!"cache_lookup_ClusterFunction() bad msg version");
-  }
-
-  if (mh->NeedByteSwap())
-    msg->SwapBytes();
-
-  CLUSTER_INCREMENT_DYN_STAT(CLUSTER_CACHE_OUTSTANDING_STAT);
-
-  CacheContinuation *c = CacheContinuation::cacheContAllocator_alloc();
-  c->mutex             = new_ProxyMutex();
-  MUTEX_TRY_LOCK(lock, c->mutex, this_ethread());
-  c->no_reply_message = (msg->seq_number == CACHE_NO_RESPONSE);
-  c->seq_number       = msg->seq_number;
-  c->from             = ch->machine;
-  c->url_md5          = msg->url_md5;
-  SET_CONTINUATION_HANDLER(c, (CacheContHandler)&CacheContinuation::replyLookupEvent);
-
-  CacheKey key(msg->url_md5);
-#ifdef CACHE_MSG_TRACE
-  log_cache_op_msg(msg->seq_number, 0, "cache_lookup");
-#endif
-
-  // Extract hostname data if passed.
-
-  char *hostname;
-  int hostname_len = len - op_to_sizeof_fixedlen_msg(CACHE_LOOKUP_OP);
-  hostname         = (hostname_len ? (char *)msg->moi.byte : nullptr);
-
-  // Note: Hostname data invalid after return from lookup
-  Cache *call_cache = caches[msg->frag_type];
-  call_cache->lookup(c, &key, (CacheFragType)msg->frag_type, hostname, hostname_len);
-}
-
-/////////////////////////////////////////////////////////////////////////
-// replyLookupEvent()
-//   This function handles the result of a lookup on a remote machine.
-//   It packages up the result and sends it back to the calling machine.
-/////////////////////////////////////////////////////////////////////////
-int
-CacheContinuation::replyLookupEvent(int event, void * /* d ATS_UNUSED */)
-{
-  ink_hrtime now;
-  now = Thread::get_hrtime();
-  CLUSTER_SUM_DYN_STAT(CLUSTER_CACHE_CALLBACK_TIME_STAT, now - start_time);
-  LOG_EVENT_TIME(start_time, callback_time_dist, cache_callbacks);
-
-  int vers = CacheOpReplyMsg::protoToVersion(from->msg_proto_major);
-  if (vers == CacheOpReplyMsg::CACHE_OP_REPLY_MESSAGE_VERSION) {
-    CacheOpReplyMsg *msg;
-    int flen = CacheOpReplyMsg::sizeof_fixedlen_msg();
-    msg      = (CacheOpReplyMsg *)ALLOCA_DOUBLE(flen);
-    msg->init();
-    CLUSTER_DECREMENT_DYN_STAT(CLUSTER_CACHE_OUTSTANDING_STAT);
-    int len = flen - sizeof(msg->token);
-
-    if (!no_reply_message) {
-      msg->seq_number = seq_number;
-      msg->result     = event;
-#ifdef CACHE_MSG_TRACE
-      log_cache_op_sndmsg(seq_number, event, "cache_result");
-#endif
-      clusterProcessor.invoke_remote(ch, CACHE_OP_RESULT_CLUSTER_FUNCTION, msg, len);
-    }
-  } else {
-    //////////////////////////////////////////////////////////////
-    // Create the specified down rev version of this message
-    //////////////////////////////////////////////////////////////
-    ink_release_assert(!"replyLookupEvent() bad msg version");
-  }
-
-  // Free up everything
-
-  cacheContAllocator_free(this);
-  return EVENT_DONE;
-}
-
-int32_t
-CacheContinuation::getObjectSize(VConnection *vc, int opcode, CacheHTTPInfo *ret_ci)
-{
-  CacheHTTPInfo *ci   = nullptr;
-  int64_t object_size = 0;
-
-  if ((opcode == CACHE_OPEN_READ_LONG) || (opcode == CACHE_OPEN_READ_BUFFER_LONG)) {
-    ((CacheVC *)vc)->get_http_info(&ci);
-    if (ci) {
-      object_size = ci->object_size_get();
-
-    } else {
-      ci          = nullptr;
-      object_size = 0;
-    }
-
-  } else {
-    object_size = ((CacheVC *)vc)->get_object_size();
-  }
-
-  if (ret_ci && !ret_ci->valid()) {
-    CacheHTTPInfo new_ci;
-    new_ci.create();
-    if (ci) {
-      // Initialize copy
-      new_ci.copy(ci);
-    } else {
-      new_ci.object_size_set(object_size);
-    }
-    new_ci.m_alt->m_writeable = 1;
-    ret_ci->copy_shallow(&new_ci);
-  }
-  ink_release_assert(object_size);
-  return object_size;
-}
-
-//////////////////////////////////////////////////////////////////////////
-// insert_cache_callback_user()
-//  Insert write VC into global cache prior to performing user callback.
-//////////////////////////////////////////////////////////////////////////
-void
-CacheContinuation::insert_cache_callback_user(ClusterVConnection *vc, int res, void *e)
-{
-  if (GlobalOpenWriteVCcache->insert(&url_md5, vc)) {
-    // Inserted
-    callback_user(res, e);
-
-  } else {
-    // Unable to insert, try later
-    result          = res;
-    callback_data   = e;
-    callback_data_2 = (void *)vc;
-    SET_HANDLER((CacheContHandler)&CacheContinuation::insertCallbackEvent);
-    eventProcessor.schedule_imm(this, ET_CACHE_CONT_SM);
-  }
-}
-
-int
-CacheContinuation::insertCallbackEvent(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */)
-{
-  if (GlobalOpenWriteVCcache->insert(&url_md5, (ClusterVConnection *)callback_data_2)) {
-    // Inserted
-    callback_user(result, callback_data);
-
-  } else {
-    // Unable to insert, try later
-    eventProcessor.schedule_imm(this, ET_CACHE_CONT_SM);
-  }
-  return EVENT_DONE;
-}
-
-///////////////////////////////////////////////////////////////////
-// callback_user()
-//  Invoke handleEvent on the given continuation (cont) with
-//    considerations for Action.
-///////////////////////////////////////////////////////////////////
-void
-CacheContinuation::callback_user(int res, void *e)
-{
-  EThread *et = this_ethread();
-
-  if (!is_ClusterThread(et)) {
-    MUTEX_TRY_LOCK(lock, mutex, et);
-    if (lock.is_locked()) {
-      if (!action.cancelled) {
-        action.continuation->handleEvent(res, e);
-      }
-      cacheContAllocator_free(this);
-
-    } else {
-      // Unable to acquire lock, retry later
-      defer_callback_result(res, e);
-    }
-  } else {
-    // Can not post completion on ET_CLUSTER thread.
-    defer_callback_result(res, e);
-  }
-}
-
-void
-CacheContinuation::defer_callback_result(int r, void *e)
-{
-  result        = r;
-  callback_data = e;
-  SET_HANDLER((CacheContHandler)&CacheContinuation::callbackResultEvent);
-  eventProcessor.schedule_imm(this, ET_CACHE_CONT_SM);
-}
-
-int
-CacheContinuation::callbackResultEvent(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */)
-{
-  if (!action.cancelled)
-    action.continuation->handleEvent(result, callback_data);
-  cacheContAllocator_free(this);
-  return EVENT_DONE;
-}
-
-//-----------------------------------------------------------------
-// CacheContinuation static member functions
-//-----------------------------------------------------------------
-
-///////////////////////////////////////////////////////////////////////
-// cacheContAllocator_alloc()
-///////////////////////////////////////////////////////////////////////
-CacheContinuation *
-CacheContinuation::cacheContAllocator_alloc()
-{
-  return cacheContAllocator.alloc();
-}
-
-///////////////////////////////////////////////////////////////////////
-// cacheContAllocator_free()
-///////////////////////////////////////////////////////////////////////
-void
-CacheContinuation::cacheContAllocator_free(CacheContinuation *c)
-{
-  ink_assert(c->magicno == (int)MagicNo);
-  //  ink_assert(!c->cache_op_ClusterFunction);
-  c->magicno = -1;
-#ifdef ENABLE_TIME_TRACE
-  c->start_time = 0;
-#endif
-  c->free();
-  c->mutex = nullptr;
-  // FIX ME. ajitb 12/21/99
-  // Compiler bug in CC: WorkShop Compilers 5.0 98/12/15 C++ 5.0
-  // Does not accept assignment of ((Continuation *) nullptr)
-  {
-    Continuation *temp = nullptr;
-    c->action          = temp;
-  }
-  c->tunnel_mutex = nullptr;
-  cacheContAllocator.free(c);
-}
-
-/////////////////////////////////////////////////////////////////////////
-// callback_failure()
-//   Post error completion using a continuation.
-/////////////////////////////////////////////////////////////////////////
-Action *
-CacheContinuation::callback_failure(Action *a, int result, int err, CacheContinuation *this_cc)
-{
-  CacheContinuation *cc;
-  if (!this_cc) {
-    cc         = cacheContAllocator_alloc();
-    cc->mutex  = a->mutex;
-    cc->action = *a;
-
-  } else {
-    cc = this_cc;
-  }
-  cc->result       = result;
-  cc->result_error = err;
-  SET_CONTINUATION_HANDLER(cc, (CacheContHandler)&CacheContinuation::callbackEvent);
-  eventProcessor.schedule_imm(cc, ET_CACHE_CONT_SM);
-  return &cc->action;
-}
-
-///////////////////////////////////////////////////////////////////////
-// callbackEvent()
-//  Invoke callback and deallocate continuation.
-///////////////////////////////////////////////////////////////////////
-int
-CacheContinuation::callbackEvent(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */)
-{
-  if (!action.cancelled)
-    action.continuation->handleEvent(result, (void *)(intptr_t)result_error);
-  cacheContAllocator_free(this);
-  return EVENT_DONE;
-}
-
-//------------------------------------------------------------------
-// File static functions
-//------------------------------------------------------------------
-
-////////////////////////////////////////////////////////////////////////
-// find_cache_continuation()
-//   Find a currently pending cache continuation expecting a response.
-//   Requires taking the lock on the remoteCacheContQueueMutex first.
-////////////////////////////////////////////////////////////////////////
-static CacheContinuation *
-find_cache_continuation(unsigned int seq_number, unsigned int from_ip)
-{
-  unsigned int hash        = FOLDHASH(from_ip, seq_number);
-  CacheContinuation *c     = nullptr;
-  CacheContinuation *lastc = nullptr;
-  for (c = (CacheContinuation *)remoteCacheContQueue[hash].head; c; c = (CacheContinuation *)c->link.next) {
-    if (seq_number == c->seq_number && from_ip == c->target_ip) {
-      if (lastc) {
-        ink_release_assert(c->link.prev == lastc);
-      } else {
-        ink_release_assert(!c->link.prev);
-      }
-      break;
-    }
-
-    lastc = c;
-  }
-  return c;
-}
-
-/////////////////////////////////////////////////////////////////////////////
-// new_cache_sequence_number()
-//  Generate unique request sequence numbers
-/////////////////////////////////////////////////////////////////////////////
-static unsigned int
-new_cache_sequence_number()
-{
-  unsigned int res = 0;
-
-  do {
-    res = (unsigned int)ink_atomic_increment(&cluster_sequence_number, 1);
-  } while (!res);
-
-  return res;
-}
-
-/***************************************************************************/
-#ifdef OMIT
-/***************************************************************************/
-/////////////////////////////////////////////////////////////////////////////
-// forwardEvent()
-//   for migrate-on-demand, make a connection between the
-//   the node which has the object and the node which should have it.
-//
-//   prepared for either OPEN_READ (from current owner)
-//   or OPEN_WRITE (from new owner)
-/////////////////////////////////////////////////////////////////////////////
-int
-CacheContinuation::forwardEvent(int event, VConnection *c)
-{
-  int ret    = EVENT_CONT;
-  cluster_vc = 0;
-
-  cache_read = false;
-  switch (event) {
-  default:
-    ink_assert(!"bad case");
-  case CACHE_EVENT_OPEN_WRITE_FAILED:
-    ret = EVENT_DONE;
-    break;
-  case CACHE_EVENT_OPEN_WRITE:
-    cluster_vc = c;
-    break;
-  case CACHE_EVENT_OPEN_READ_FAILED:
-    cache_read = true;
-    ret        = EVENT_DONE;
-    break;
-  case CACHE_EVENT_OPEN_READ:
-    cache_read = true;
-    cluster_vc = c;
-    break;
-  }
-  SET_HANDLER((CacheContHandler)&CacheContinuation::forwardWaitEvent);
-  return ret;
-}
-
-////////////////////////////////////////////////////////////////////////
-// forwardWaitEvent()
-//   For migrate-on-demand, make a connection as above (forwardEvent)
-//   second either OPEN_READ or OPEN_WRITE,
-//   the data for the first is stored in (cluster_vc,cache_read)
-////////////////////////////////////////////////////////////////////////
-int
-CacheContinuation::forwardWaitEvent(int event, VConnection *c)
-{
-  int ret         = EVENT_CONT;
-  int res         = CACHE_EVENT_OPEN_READ_FAILED;
-  void *res_data  = nullptr;
-  VConnection *vc = nullptr;
-
-  switch (event) {
-  default:
-    ink_assert(!"bad case");
-  case CACHE_EVENT_OPEN_WRITE_FAILED:
-  case CACHE_EVENT_OPEN_READ_FAILED:
-    ret = EVENT_DONE;
-    break;
-  case CACHE_EVENT_OPEN_WRITE:
-  case CACHE_EVENT_OPEN_READ:
-    vc = c;
-    break;
-  }
-  VConnection *read_vc  = (cache_read ? cluster_vc : vc);
-  VConnection *write_vc = (!cache_read ? cluster_vc : vc);
-
-  res      = read_vc ? CACHE_EVENT_OPEN_READ : CACHE_EVENT_OPEN_READ_FAILED;
-  res_data = read_vc;
-
-  // if the read and write are sucessful, tunnel the read to the write
-  if (read_vc && write_vc) {
-    res_data = new VCTee(read_vc, write_vc, vio);
-    if (vio) { // CACHE_EVENT_OPEN_READ_VIO
-      res      = event;
-      res_data = &((VCTee *)read_vc)->vio;
-    }
-  }
-  // if the read is sucessful return it to the user
-  //
-  c->handleEvent(res, res_data);
-  return ret;
-}
-
-/////////////////////////////////////////////////////////////////////
-// tunnelEvent()
-//   If the reply requires data, tunnel the data from the cache
-//   to the cluster.
-/////////////////////////////////////////////////////////////////////
-int
-CacheContinuation::tunnelEvent(int event, VConnection *vc)
-{
-  int ret       = EVENT_DONE;
-  int flen      = CacheOpReplyMsg::sizeof_fixedlen_msg(); // include token
-  int len       = 0;
-  bool read_buf = ((request_opcode == CACHE_OPEN_READ_BUFFER) || (request_opcode == CACHE_OPEN_READ_BUFFER_LONG));
-  ink_release_assert(!read_buf);
-
-  CacheOpReplyMsg rmsg;
-  CacheOpReplyMsg *msg = &rmsg;
-  msg->result          = result;
-  msg->seq_number      = seq_number;
-  msg->token           = token;
-  int expect_reply     = 1;
-
-  if (event == CLUSTER_EVENT_OPEN) {
-    if (cache_read) {
-      if (read_buf) {
-        ink_assert(have_all_data || (readahead_vio == &((CacheVConnection *)cluster_vc)->vio));
-        write_cluster_vc = (ClusterVConnection *)vc;
-
-        if (have_all_data) {
-          msg->token.clear(); // Tell sender no conn established
-        } else {
-          msg->token = token; // Tell sender conn established
-          setupReadBufTunnel(cluster_vc, vc);
-        }
-
-      } else {
-        OneWayTunnel *pOWT = OneWayTunnel::OneWayTunnel_alloc();
-        pOWT->init(cluster_vc, vc, nullptr, nbytes, this->mutex);
-        --expect_reply;
-      }
-
-      ////////////////////////////////////////////////////////
-      // cache_read requires CacheHTTPInfo in reply message.
-      ////////////////////////////////////////////////////////
-      int res;
-      CacheHTTPInfo *ci;
-
-      if (!cache_vc_info) {
-        // OPEN_READ case
-        (void)getObjectSize(cluster_vc, request_opcode, &cache_vc_info);
-      }
-      ci = cache_vc_info;
-
-      // Determine data length and allocate
-      len                    = ci->marshal_length();
-      CacheOpReplyMsg *reply = (CacheOpReplyMsg *)ALLOCA_DOUBLE(flen + len);
-
-      // Initialize reply message header
-      *reply = *msg;
-
-      // Marshal response data into reply message
-      res = ci->marshal((char *)reply->moi.byte, len);
-      ink_assert(res > 0);
-
-      // Make reply message the current message
-      msg = reply;
-
-    } else {
-      OneWayTunnel *pOWT = OneWayTunnelAllocator.alloc();
-      pOWT->init(vc, cluster_vc, nullptr, nbytes, this->mutex);
-      --expect_reply;
-    }
-    ret = EVENT_CONT;
-  } else {
-    ink_release_assert(event == CLUSTER_EVENT_OPEN_FAILED);
-    msg->result = CACHE_EVENT_SET_FAILED(result);
-
-    if (read_buf) {
-      Debug("cluster_timeout", "unable to make cluster connection2");
-      initial_buf     = 0; // Do not send data
-      initial_bufsize = 0;
-
-      if (!have_all_data) {
-        // Shutdown cache connection and free MIOBuffer
-        MIOBuffer *mbuf = readahead_vio->buffer.writer();
-        cluster_vc->do_io(VIO::CLOSE);
-        free_MIOBuffer(mbuf);
-      }
-    } else {
-      Debug("cluster_timeout", "unable to make cluster connection2A");
-      cluster_vc->do_io(VIO::CLOSE);
-    }
-    len = 0 - (int)sizeof(msg->token);
-    --expect_reply;
-  }
-
-  int vers = CacheOpReplyMsg::protoToVersion(from->msg_proto_major);
-  if (vers == CacheOpReplyMsg::CACHE_OP_REPLY_MESSAGE_VERSION) {
-    if (read_buf) {
-      // Transmit reply message and object data in same cluster message
-      clusterProcessor.invoke_remote_data(from, CACHE_OP_RESULT_CLUSTER_FUNCTION, (void *)msg, (flen + len), initial_buf,
-                                          initial_bufsize, cluster_vc_channel, &token, &CacheContinuation::disposeOfDataBuffer,
-                                          (void *)this, CLUSTER_OPT_STEAL);
-
-    } else {
-      clusterProcessor.invoke_remote(from, CACHE_OP_RESULT_CLUSTER_FUNCTION, (void *)msg, (flen + len), CLUSTER_OPT_STEAL);
-    }
-  } else {
-    //////////////////////////////////////////////////////////////
-    // Create the specified down rev version of this message
-    //////////////////////////////////////////////////////////////
-    ink_release_assert(!"tunnelEvent() bad msg version");
-  }
-  if (expect_reply <= 0)
-    cacheContAllocator_free(this);
-  return ret;
-}
-
-/////////////////////////////////////////////////////////////////////
-// remoteConnectEvent()
-//   If this was an open, make a connection on this side before
-//   responding to the user.
-/////////////////////////////////////////////////////////////////////
-int
-CacheContinuation::remoteConnectEvent(int event, VConnection *cvc)
-{
-  ClusterVConnection *vc = (ClusterVConnection *)cvc;
-
-  if (event == CLUSTER_EVENT_OPEN) {
-    if (result == CACHE_EVENT_OPEN_READ) {
-      // Move CacheHTTPInfo reply data into VC
-      vc->alternate = this->ic_new_info;
-      this->ic_new_info.clear();
-    }
-    callback_user(result, vc);
-    return EVENT_CONT;
-  } else {
-    Debug("cluster_cache", "unable to make cluster connection");
-    callback_user(CACHE_EVENT_SET_FAILED(result), vc);
-    return EVENT_DONE;
-  }
-}
-
-/***************************************************************************/
-#endif // OMIT
-/***************************************************************************/
-
-// End of ClusterCache.cc
diff --git a/iocore/cluster/ClusterConfig.cc b/iocore/cluster/ClusterConfig.cc
deleted file mode 100644
index 33cbc7d..0000000
--- a/iocore/cluster/ClusterConfig.cc
+++ /dev/null
@@ -1,450 +0,0 @@
-/** @file
-
-  A brief file description
-
-  @section license License
-
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
- */
-
-/****************************************************************************
-
-  ClusterConfig.cc
-****************************************************************************/
-
-#include "P_Cluster.h"
-// updated from the cluster port configuration variable
-int cluster_port = DEFAULT_CLUSTER_PORT_NUMBER;
-
-ClusterAccept::ClusterAccept(int *port, int send_bufsize, int recv_bufsize)
-  : Continuation(nullptr),
-    p_cluster_port(port),
-    socket_send_bufsize(send_bufsize),
-    socket_recv_bufsize(recv_bufsize),
-    current_cluster_port(-1),
-    accept_action(nullptr),
-    periodic_event(nullptr)
-{
-  mutex = new_ProxyMutex();
-  SET_HANDLER(&ClusterAccept::ClusterAcceptEvent);
-}
-
-ClusterAccept::~ClusterAccept()
-{
-  mutex = nullptr;
-}
-
-void
-ClusterAccept::Init()
-{
-  // Setup initial accept by simulating EVENT_INTERVAL
-  // where cluster accept port has changed.
-
-  current_cluster_port = ~*p_cluster_port;
-  ClusterAcceptEvent(EVENT_INTERVAL, nullptr);
-
-  // Setup periodic event to handle changing cluster accept port.
-  periodic_event = eventProcessor.schedule_every(this, HRTIME_SECONDS(60));
-}
-
-void
-ClusterAccept::ShutdownDelete()
-{
-  MUTEX_TRY_LOCK(lock, this->mutex, this_ethread());
-  if (!lock.is_locked()) {
-    eventProcessor.schedule_imm(this, ET_CALL);
-    return;
-  }
-  // Kill all events and delete.
-  if (accept_action) {
-    accept_action->cancel();
-    accept_action = nullptr;
-  }
-  if (periodic_event) {
-    periodic_event->cancel();
-    periodic_event = nullptr;
-  }
-  delete this;
-}
-
-int
-ClusterAccept::ClusterAcceptEvent(int event, void *data)
-{
-  switch (event) {
-  case EVENT_IMMEDIATE: {
-    ShutdownDelete();
-    return EVENT_DONE;
-  }
-  case EVENT_INTERVAL: {
-    int cluster_port = *p_cluster_port;
-
-    if (cluster_port != current_cluster_port) {
-      // Configuration changed cluster port, redo accept on new port.
-      if (accept_action) {
-        accept_action->cancel();
-        accept_action = nullptr;
-      }
-
-      NetProcessor::AcceptOptions opt;
-      opt.recv_bufsize   = socket_recv_bufsize;
-      opt.send_bufsize   = socket_send_bufsize;
-      opt.etype          = ET_CLUSTER;
-      opt.local_port     = cluster_port;
-      opt.ip_family      = AF_INET;
-      opt.localhost_only = false;
-
-      accept_action = netProcessor.main_accept(this, NO_FD, opt);
-      if (!accept_action) {
-        Warning("Unable to accept cluster connections on port: %d", cluster_port);
-      } else {
-        current_cluster_port = cluster_port;
-      }
-    }
-    return EVENT_CONT;
-  }
-  case NET_EVENT_ACCEPT: {
-    ClusterAcceptMachine((NetVConnection *)data);
-    return EVENT_DONE;
-  }
-  default: {
-    Warning("ClusterAcceptEvent: received unknown event %d", event);
-    return EVENT_DONE;
-  }
-  } // End of switch
-}
-
-int
-ClusterAccept::ClusterAcceptMachine(NetVConnection *NetVC)
-{
-  // Validate remote IP address.
-  unsigned int remote_ip = NetVC->get_remote_ip();
-  MachineList *mc        = the_cluster_machines_config();
-
-  if (mc && !mc->find(remote_ip)) {
-    Note("Illegal cluster connection from %u.%u.%u.%u", DOT_SEPARATED(remote_ip));
-    NetVC->do_io(VIO::CLOSE);
-    return 0;
-  }
-
-  Debug(CL_NOTE, "Accepting machine %u.%u.%u.%u", DOT_SEPARATED(remote_ip));
-  ClusterHandler *ch = new ClusterHandler;
-  ch->machine        = new ClusterMachine(nullptr, remote_ip);
-  ch->ip             = remote_ip;
-  ch->net_vc         = NetVC;
-  eventProcessor.schedule_imm_signal(ch, ET_CLUSTER);
-  return 1;
-}
-
-static void
-make_cluster_connections(MachineList *l)
-{
-  //
-  // Connect to all new machines.
-  //
-  uint32_t ip         = this_cluster_machine()->ip;
-  int num_connections = this_cluster_machine()->num_connections;
-
-  if (l) {
-    for (int i = 0; i < l->n; i++) {
-#ifdef LOCAL_CLUSTER_TEST_MODE
-      if (ip < l->machine[i].ip || (ip == l->machine[i].ip && (cluster_port < l->machine[i].port))) {
-#else
-      if (ip < l->machine[i].ip) {
-#endif
-        for (int j = 0; j < num_connections; j++) {
-          clusterProcessor.connect(l->machine[i].ip, l->machine[i].port, j);
-        }
-      }
-    }
-  }
-}
-
-int
-machine_config_change(const char * /* name ATS_UNUSED */, RecDataT /* data_type ATS_UNUSED */, RecData data, void *cookie)
-{
-  // Handle changes to the cluster.config or machines.config
-  // file.  cluster.config is the list of machines in the
-  // cluster proper ( in the cluster hash table ).  machines.config
-  // is the list of machines which communicate with the cluster.
-  // This may include front-end load redirectors, machines going
-  // up or coming down etc.
-  //
-  char *filename   = (char *)data.rec_string;
-  MachineList *l   = read_MachineList(filename);
-  MachineList *old = nullptr;
-#ifdef USE_SEPARATE_MACHINE_CONFIG
-  switch ((int)cookie) {
-  case MACHINE_CONFIG:
-    old             = machines_config;
-    machines_config = l;
-    break;
-  case CLUSTER_CONFIG:
-    old            = cluster_config;
-    cluster_config = l;
-    make_cluster_connections(l);
-    break;
-  }
-#else
-  (void)cookie;
-  old             = cluster_config;
-  machines_config = l;
-  cluster_config  = l;
-  make_cluster_connections(l);
-#endif
-  if (old)
-    free_MachineList(old);
-  return 0;
-}
-
-void
-do_machine_config_change(void *d, const char *s)
-{
-  char cluster_config_filename[PATH_NAME_MAX] = "";
-  REC_ReadConfigString(cluster_config_filename, s, sizeof(cluster_config_filename) - 1);
-  RecData data;
-  data.rec_string = cluster_config_filename;
-  machine_config_change(s, RECD_STRING, data, d);
-}
-
-/*************************************************************************/
-// ClusterConfiguration member functions (Public Class)
-/*************************************************************************/
-ClusterConfiguration::ClusterConfiguration() : n_machines(0), changed(0)
-{
-  memset(machines, 0, sizeof(machines));
-  memset(hash_table, 0, sizeof(hash_table));
-}
-
-/*************************************************************************/
-// ConfigurationContinuation member functions (Internal Class)
-/*************************************************************************/
-struct ConfigurationContinuation;
-typedef int (ConfigurationContinuation::*CfgContHandler)(int, void *);
-struct ConfigurationContinuation : public Continuation {
-  ClusterConfiguration *c;
-  ClusterConfiguration *prev;
-
-  int
-  zombieEvent(int /* event ATS_UNUSED */, Event *e)
-  {
-    prev->link.next = nullptr; // remove that next pointer
-    SET_HANDLER((CfgContHandler)&ConfigurationContinuation::dieEvent);
-    e->schedule_in(CLUSTER_CONFIGURATION_ZOMBIE);
-    return EVENT_CONT;
-  }
-
-  int
-  dieEvent(int event, Event *e)
-  {
-    (void)event;
-    (void)e;
-    delete c;
-    delete this;
-    return EVENT_DONE;
-  }
-
-  ConfigurationContinuation(ClusterConfiguration *cc, ClusterConfiguration *aprev) : Continuation(nullptr), c(cc), prev(aprev)
-  {
-    mutex = new_ProxyMutex();
-    SET_HANDLER((CfgContHandler)&ConfigurationContinuation::zombieEvent);
-  }
-};
-
-static void
-free_configuration(ClusterConfiguration *c, ClusterConfiguration *prev)
-{
-  //
-  // Delete the configuration after a time.
-  // The problem is that configurations change infrequently, and
-  // are used in different threads, so reference counts are
-  // relatively difficult and expensive.  The solution I have
-  // chosen is to simply delete the object after some (very long)
-  // time after it has ceased to be accessible.
-  //
-  eventProcessor.schedule_in(new ConfigurationContinuation(c, prev), CLUSTER_CONFIGURATION_TIMEOUT, ET_CALL);
-}
-
-ClusterConfiguration *
-configuration_add_machine(ClusterConfiguration *c, ClusterMachine *m)
-{
-  // Build a new cluster configuration with the new machine.
-  // Machines are stored in ip sorted order.
-  //
-  EThread *thread          = this_ethread();
-  ProxyMutex *mutex        = thread->mutex.get();
-  int i                    = 0;
-  ClusterConfiguration *cc = new ClusterConfiguration(*c);
-
-  // Find the place to insert this new machine
-  //
-  for (i = 0; i < cc->n_machines; i++) {
-    if (cc->machines[i]->ip > m->ip)
-      break;
-  }
-
-  // Move the other machines out of the way
-  //
-  for (int j            = cc->n_machines - 1; j >= i; j--)
-    cc->machines[j + 1] = cc->machines[j];
-
-  // Insert it
-  //
-  cc->machines[i] = m;
-  cc->n_machines++;
-
-  cc->link.next = c;
-  cc->changed   = Thread::get_hrtime();
-  ink_assert(cc->n_machines < CLUSTER_MAX_MACHINES);
-
-  build_cluster_hash_table(cc);
-  INK_MEMORY_BARRIER; // commit writes before freeing old hash table
-  CLUSTER_INCREMENT_DYN_STAT(CLUSTER_CONFIGURATION_CHANGES_STAT);
-
-  free_configuration(c, cc);
-  return cc;
-}
-
-ClusterConfiguration *
-configuration_remove_machine(ClusterConfiguration *c, ClusterMachine *m)
-{
-  ProxyMutex *mutex = this_thread()->mutex.get();
-
-  //
-  // Build a new cluster configuration without a machine
-  //
-  ClusterConfiguration *cc = new ClusterConfiguration(*c);
-  //
-  // remove m and move others down
-  //
-  for (int i = 0; i < cc->n_machines - 1; i++)
-    if (m == cc->machines[i])
-      m = cc->machines[i] = cc->machines[i + 1];
-  cc->n_machines--;
-
-  ink_assert(cc->n_machines > 0);
-
-  cc->link.next = c;
-  cc->changed   = Thread::get_hrtime();
-
-  build_cluster_hash_table(cc);
-  INK_MEMORY_BARRIER; // commit writes before freeing old hash table
-  CLUSTER_INCREMENT_DYN_STAT(CLUSTER_CONFIGURATION_CHANGES_STAT);
-
-  free_configuration(c, cc);
-  return cc;
-}
-
-//
-// cluster_machine_at_depth()
-//   Find a machine at a particular depth into the past.
-//   We don't want to probe the current machine or machines
-//   we have probed before, so we store a list of "past_probes".
-//   If probe_depth and past_probes are nullptr we only want the
-//   owner (machine now as opposed to in the past).
-//
-ClusterMachine *
-cluster_machine_at_depth(unsigned int hash, int *pprobe_depth, ClusterMachine **past_probes)
-{
-#ifdef CLUSTER_TOMCAT
-  if (!cache_clustering_enabled)
-    return nullptr;
-#endif
-  ClusterConfiguration *cc      = this_cluster()->current_configuration();
-  ClusterConfiguration *next_cc = cc;
-  ink_hrtime now                = Thread::get_hrtime();
-  int fake_probe_depth          = 0;
-  int &probe_depth              = pprobe_depth ? (*pprobe_depth) : fake_probe_depth;
-  int tprobe_depth              = probe_depth;
-
-#ifdef CLUSTER_TEST
-  if (cc->n_machines > 1) {
-    for (int i = 0; i < cc->n_machines; ++i) {
-      if (cc->machines[i] != this_cluster_machine()) {
-        return cc->machines[i];
-      }
-    }
-  }
-#endif // CLUSTER_TEST
-
-  while (true) {
-    // If we are out of our depth, fail
-    //
-    if (probe_depth > CONFIGURATION_HISTORY_PROBE_DEPTH)
-      break;
-
-    // If there is no configuration, fail
-    //
-    if (!cc || !next_cc)
-      break;
-
-    cc      = next_cc;
-    next_cc = next_cc->link.next;
-
-    // Find the correct configuration
-    //
-    if (tprobe_depth) {
-      if (cc->changed > (now + CLUSTER_CONFIGURATION_TIMEOUT))
-        break;
-      tprobe_depth--;
-      continue;
-    }
-
-    ClusterMachine *m = cc->machine_hash(hash);
-
-    // If it is not this machine, or a machine we have done before
-    // and one that is still up, try again
-    //
-    bool ok = !(m == this_cluster_machine() || (past_probes && machine_in_vector(m, past_probes, probe_depth)) || m->dead);
-
-    // Store the all but the last probe, so that we never return
-    // the same machine
-    //
-    if (past_probes && probe_depth < CONFIGURATION_HISTORY_PROBE_DEPTH)
-      past_probes[probe_depth] = m;
-    probe_depth++;
-
-    if (!ok) {
-      if (!pprobe_depth)
-        break; // don't go down if we don't have a depth
-      continue;
-    }
-
-    return (m != this_cluster_machine()) ? m : nullptr;
-  }
-  return nullptr;
-}
-
-//
-// initialize_thread_for_cluster()
-//   This is not required since we have a separate handler
-//   for each machine-machine pair, the pointers to which are
-//   stored in the ClusterMachine structures
-//
-void
-initialize_thread_for_cluster(EThread *e)
-{
-  (void)e;
-}
-
-/*************************************************************************/
-// Cluster member functions (Public Class)
-/*************************************************************************/
-Cluster::Cluster()
-{
-}
-
-// End of ClusterConfig.cc
diff --git a/iocore/cluster/ClusterHandler.cc b/iocore/cluster/ClusterHandler.cc
deleted file mode 100644
index 09f1db9..0000000
--- a/iocore/cluster/ClusterHandler.cc
+++ /dev/null
@@ -1,3068 +0,0 @@
-/** @file
-
-  A brief file description
-
-  @section license License
-
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
- */
-
-/****************************************************************************
-
-  ClusterHandler.cc
-****************************************************************************/
-
-#define DEFINE_CLUSTER_FUNCTIONS
-#include "P_Cluster.h"
-
-/*************************************************************************/
-// Global Data
-/*************************************************************************/
-// Initialize clusterFunction[] size
-unsigned SIZE_clusterFunction = countof(clusterFunction);
-
-// hook for testing
-ClusterFunction *ptest_ClusterFunction = nullptr;
-
-// global bit buckets for closed channels
-static char channel_dummy_input[DEFAULT_MAX_BUFFER_SIZE];
-char channel_dummy_output[DEFAULT_MAX_BUFFER_SIZE];
-
-// outgoing control continuations
-ClassAllocator<OutgoingControl> outControlAllocator("outControlAllocator");
-
-// incoming control descriptors
-ClassAllocator<IncomingControl> inControlAllocator("inControlAllocator");
-
-static int dump_msgs = 0;
-
-/////////////////////////////////////////
-// VERIFY_PETERS_DATA support code
-/////////////////////////////////////////
-#ifdef VERIFY_PETERS_DATA
-#define DO_VERIFY_PETERS_DATA(_p, _l) verify_peters_data(_p, _l)
-#else
-#define DO_VERIFY_PETERS_DATA(_p, _l)
-#endif
-
-void
-verify_peters_data(char *ap, int l)
-{
-  unsigned char *p = (unsigned char *)ap;
-  for (int i = 0; i < l - 1; i++) {
-    unsigned char x1 = p[i];
-    unsigned char x2 = p[i + 1];
-    x1 += 1;
-    if (x1 != x2) {
-      fprintf(stderr, "verify peter's data failed at %d\n", i);
-      break;
-    }
-  }
-}
-
-/*************************************************************************/
-// ClusterHandler member functions (Internal Class)
-/*************************************************************************/
-//
-// Overview:
-//  In a steady state cluster environment, all cluster nodes have an
-//  established TCP socket connection to each node in the cluster.
-//  An instance of the class ClusterHandler exists for each known node
-//  in the cluster.  All specific node-node data/state is encapsulated
-//  by this class.
-//
-//  ClusterHandler::mainClusterEvent() is the key periodic event which
-//  drives the read/write action over the node-node socket connection.
-//  A high level overview of ClusterHandler::mainClusterEvent() action is
-//  as follows:
-//      1) Perform cluster interconnect load monitoring functions.
-//         If interconnect is overloaded, convert all remote cluster
-//         operations to proxy only.
-//      2) Process delayed reads.  Delayed read refers to data associated
-//         with a VC (Virtual Connection) which resides in an intermediate
-//         buffer and is unknown to the VC.  This is required in cases
-//         where we are unable to acquire the VC mutex at the time of the
-//         read from the node-node socket.  Delayed read processing
-//         consists of acquiring the VC mutex and moving the data into the
-//         VC and posting read completion.
-//      3) Process pending read data on the node-node TCP socket.  In the
-//         typical case, read processing is performed using three read
-//         operations.  The actions are as follows:
-//              a) read the fixed size message header
-//                 (struct ClusterMsgHeader) consisting of the
-//                 number of data descriptors and the size of the inline
-//                 control messages following the data descriptors.
-//              b) Setup buffer for data descriptors and inline control
-//                 messages and issue read.
-//              c) Setup read buffers and acquire applicable locks for
-//                 VC/Control data described by data descriptors and issue
-//                 read.
-//              d) Perform read completion actions on control and VC data.
-//              e) Free VC locks
-//      4) Process write bank data.  Write bank data is outstanding data
-//         which we were unable to push out in the last write over the
-//         node-node TCP socket.  Write bank data must be successfully pushed
-//         before performing any additional write processing.
-//      5) Build a write message consisting of the following data:
-//          1) Write data for a Virtual Connection in the current write data
-//             bucket (write_vcs)
-//          2) Virtual Connection free space for VCs in the current read
-//             data bucket (read_vcs)
-//          3) Control message data (outgoing_control)
-//      6) Push write data
-//
-//  Thread stealing refers to executing the control message processing
-//  portion of mainClusterEvent()  by a thread not associated with the
-//  periodic event.  This a mechanism to avoid the latency on control
-//  messages by allowing them to be pushed immediately.
-//
-/*************************************************************************/
-
-ClusterHandler::ClusterHandler()
-  : net_vc(nullptr),
-    thread(nullptr),
-    ip(0),
-    port(0),
-    hostname(nullptr),
-    machine(nullptr),
-    ifd(-1),
-    id(-1),
-    dead(true),
-    downing(false),
-    active(false),
-    on_stolen_thread(false),
-    n_channels(0),
-    channels(nullptr),
-    channel_data(nullptr),
-    connector(false),
-    cluster_connect_state(ClusterHandler::CLCON_INITIAL),
-    needByteSwap(false),
-    configLookupFails(0),
-    cluster_periodic_event(nullptr),
-    read(this, true),
-    write(this, false),
-    current_time(0),
-    last(0),
-    last_report(0),
-    n_since_last_report(0),
-    last_cluster_op_enable(0),
-    last_trace_dump(0),
-    clm(nullptr),
-    disable_remote_cluster_ops(false),
-    pw_write_descriptors_built(0),
-    pw_freespace_descriptors_built(0),
-    pw_controldata_descriptors_built(0),
-    pw_time_expired(0),
-    started_on_stolen_thread(false),
-    control_message_write(false)
-#ifdef CLUSTER_STATS
-    ,
-    _vc_writes(0),
-    _vc_write_bytes(0),
-    _control_write_bytes(0),
-    _dw_missed_lock(0),
-    _dw_not_enabled(0),
-    _dw_wait_remote_fill(0),
-    _dw_no_active_vio(0),
-    _dw_not_enabled_or_no_write(0),
-    _dw_set_data_pending(0),
-    _dw_no_free_space(0),
-    _fw_missed_lock(0),
-    _fw_not_enabled(0),
-    _fw_wait_remote_fill(0),
-    _fw_no_active_vio(0),
-    _fw_not_enabled_or_no_read(0),
-    _process_read_calls(0),
-    _n_read_start(0),
-    _n_read_header(0),
-    _n_read_await_header(0),
-    _n_read_setup_descriptor(0),
-    _n_read_descriptor(0),
-    _n_read_await_descriptor(0),
-    _n_read_setup_data(0),
-    _n_read_data(0),
-    _n_read_await_data(0),
-    _n_read_post_complete(0),
-    _n_read_complete(0),
-    _process_write_calls(0),
-    _n_write_start(0),
-    _n_write_setup(0),
-    _n_write_initiate(0),
-    _n_write_await_completion(0),
-    _n_write_post_complete(0),
-    _n_write_complete(0)
-#endif
-{
-#ifdef MSG_TRACE
-  t_fd = fopen("msgtrace.log", "w");
-#endif
-  // we need to lead by at least 1
-
-  min_priority = 1;
-  SET_HANDLER((ClusterContHandler)&ClusterHandler::startClusterEvent);
-
-  mutex = new_ProxyMutex();
-  OutgoingControl oc;
-  int n;
-  for (n = 0; n < CLUSTER_CMSG_QUEUES; ++n) {
-    ink_atomiclist_init(&outgoing_control_al[n], "OutGoingControlQueue", (char *)&oc.link.next - (char *)&oc);
-  }
-
-  IncomingControl ic;
-  ink_atomiclist_init(&external_incoming_control, "ExternalIncomingControlQueue", (char *)&ic.link.next - (char *)&ic);
-
-  ClusterVConnection ivc;
-  ink_atomiclist_init(&external_incoming_open_local, "ExternalIncomingOpenLocalQueue", (char *)&ivc.link.next - (char *)&ivc);
-  ink_atomiclist_init(&read_vcs_ready, "ReadVcReadyQueue", offsetof(ClusterVConnection, ready_alink.next));
-  ink_atomiclist_init(&write_vcs_ready, "WriteVcReadyQueue", offsetof(ClusterVConnection, ready_alink.next));
-  memset((char *)&callout_cont[0], 0, sizeof(callout_cont));
-  memset((char *)&callout_events[0], 0, sizeof(callout_events));
-}
-
-ClusterHandler::~ClusterHandler()
-{
-  bool free_m = false;
-  if (net_vc) {
-    net_vc->do_io(VIO::CLOSE);
-    net_vc = nullptr;
-  }
-  if (machine) {
-    MUTEX_TAKE_LOCK(the_cluster_config_mutex, this_ethread());
-    if (++machine->free_connections >= machine->num_connections)
-      free_m = true;
-    MUTEX_UNTAKE_LOCK(the_cluster_config_mutex, this_ethread());
-    if (free_m)
-      free_ClusterMachine(machine);
-  }
-  machine = nullptr;
-  ats_free(hostname);
-  hostname = nullptr;
-  ats_free(channels);
-  channels = nullptr;
-  if (channel_data) {
-    for (int i = 0; i < n_channels; ++i) {
-      if (channel_data[i]) {
-        ats_free(channel_data[i]);
-        channel_data[i] = nullptr;
-      }
-    }
-    ats_free(channel_data);
-    channel_data = nullptr;
-  }
-  if (read_vcs)
-    delete[] read_vcs;
-  read_vcs = nullptr;
-
-  if (write_vcs)
-    delete[] write_vcs;
-  write_vcs = nullptr;
-
-  if (clm) {
-    delete clm;
-    clm = nullptr;
-  }
-#ifdef CLUSTER_STATS
-  message_blk = nullptr;
-#endif
-}
-
-void
-ClusterHandler::close_ClusterVConnection(ClusterVConnection *vc)
-{
-  //
-  // Close down a ClusterVConnection
-  //
-  if (vc->inactivity_timeout)
-    vc->inactivity_timeout->cancel(vc);
-  if (vc->active_timeout)
-    vc->active_timeout->cancel(vc);
-  if (vc->read.queue)
-    ClusterVC_remove_read(vc);
-  if (vc->write.queue)
-    ClusterVC_remove_write(vc);
-  vc->read.vio.mutex  = nullptr;
-  vc->write.vio.mutex = nullptr;
-
-  ink_assert(!vc->read_locked);
-  ink_assert(!vc->write_locked);
-  int channel = vc->channel;
-  free_channel(vc);
-
-  if (vc->byte_bank_q.head) {
-    delayed_reads.remove(vc);
-
-    // Deallocate byte bank descriptors
-    ByteBankDescriptor *d;
-    while ((d = vc->byte_bank_q.dequeue())) {
-      ByteBankDescriptor::ByteBankDescriptor_free(d);
-    }
-  }
-  vc->read_block = nullptr;
-
-  ink_assert(!vc->write_list);
-  ink_assert(!vc->write_list_tail);
-  ink_assert(!vc->write_list_bytes);
-  ink_assert(!vc->write_bytes_in_transit);
-
-  if (((!vc->remote_closed && !vc->have_all_data) || (vc->remote_closed == FORCE_CLOSE_ON_OPEN_CHANNEL)) && vc->ch) {
-    CloseMessage msg;
-    int vers = CloseMessage::protoToVersion(vc->ch->machine->msg_proto_major);
-    void *data;
-    int len;
-
-    if (vers == CloseMessage::CLOSE_CHAN_MESSAGE_VERSION) {
-      msg.channel         = channel;
-      msg.status          = (vc->remote_closed == FORCE_CLOSE_ON_OPEN_CHANNEL) ? FORCE_CLOSE_ON_OPEN_CHANNEL : vc->closed;
-      msg.lerrno          = vc->lerrno;
-      msg.sequence_number = vc->token.sequence_number;
-      data                = (void *)&msg;
-      len                 = sizeof(CloseMessage);
-
-    } else {
-      //////////////////////////////////////////////////////////////
-      // Create the specified down rev version of this message
-      //////////////////////////////////////////////////////////////
-      ink_release_assert(!"close_ClusterVConnection() bad msg version");
-    }
-    clusterProcessor.invoke_remote(vc->ch, CLOSE_CHANNEL_CLUSTER_FUNCTION, data, len);
-  }
-  ink_hrtime now = Thread::get_hrtime();
-  CLUSTER_DECREMENT_DYN_STAT(CLUSTER_CONNECTIONS_OPEN_STAT);
-  CLUSTER_SUM_DYN_STAT(CLUSTER_CON_TOTAL_TIME_STAT, now - vc->start_time);
-  if (!local_channel(channel)) {
-    CLUSTER_SUM_DYN_STAT(CLUSTER_REMOTE_CONNECTION_TIME_STAT, now - vc->start_time);
-  } else {
-    CLUSTER_SUM_DYN_STAT(CLUSTER_LOCAL_CONNECTION_TIME_STAT, now - vc->start_time);
-  }
-  clusterVCAllocator_free(vc);
-}
-
-inline bool
-ClusterHandler::vc_ok_write(ClusterVConnection *vc)
-{
-  return (((vc->closed > 0) && (vc->write_list || vc->write_bytes_in_transit)) ||
-          (!vc->closed && vc->write.enabled && vc->write.vio.op == VIO::WRITE && vc->write.vio.buffer.writer()));
-}
-
-inline bool
-ClusterHandler::vc_ok_read(ClusterVConnection *vc)
-{
-  return (!vc->closed && vc->read.vio.op == VIO::READ && vc->read.vio.buffer.writer());
-}
-
-void
-ClusterHandler::close_free_lock(ClusterVConnection *vc, ClusterVConnState *s)
-{
-  Ptr<ProxyMutex> m(s->vio.mutex);
-  if (s == &vc->read) {
-    if (vc->read_locked)
-      MUTEX_UNTAKE_LOCK(vc->read_locked, thread);
-    vc->read_locked = nullptr;
-  } else {
-    if (vc->write_locked)
-      MUTEX_UNTAKE_LOCK(vc->write_locked, thread);
-    vc->write_locked = nullptr;
-  }
-  close_ClusterVConnection(vc);
-}
-
-bool
-ClusterHandler::build_data_vector(char *d, int len, bool read_flag)
-{
-  // Internal interface to general network i/o facility allowing
-  // single vector read/write to static data buffer.
-
-  ClusterState &s = (read_flag ? read : write);
-  ink_assert(d);
-  ink_assert(len);
-  ink_assert(s.iov);
-
-  s.msg.count       = 1;
-  s.iov[0].iov_base = nullptr;
-  s.iov[0].iov_len  = len;
-  s.block[0]        = new_IOBufferBlock();
-  s.block[0]->set(new_constant_IOBufferData(d, len));
-
-  if (read_flag) {
-    // Make block write_avail == len
-    s.block[0]->_buf_end = s.block[0]->end() + len;
-  } else {
-    // Make block read_avail == len
-    s.block[0]->fill(len);
-  }
-
-  s.to_do = len;
-  s.did   = 0;
-  s.n_iov = 1;
-
-  return true;
-}
-
-bool
-ClusterHandler::build_initial_vector(bool read_flag)
-{
-  //
-  // Build initial read/write struct iovec and corresponding IOBufferData
-  // structures from the given struct descriptor(s).
-  // Required vector adjustments for partial i/o conditions is handled
-  // by adjust_vector().
-  //
-  ///////////////////////////////////////////////////////////////////
-  // Descriptor to struct iovec layout
-  ///////////////////////////////////////////////////////////////////
-  // Write iovec[] layout
-  //    iov[0] ----> struct ClusterMsgHeader
-  //    iov[1] ----> struct descriptor [count]
-  //                 char short_control_messages[control_bytes]
-  //
-  //    iov[2] ----> struct descriptor data (element #1)
-  //    ......
-  //    iov[2+count] ----> struct descriptor data (element #count)
-  //
-  ///////////////////////////////////////////////////////////////////
-  // Read iovec[] layout phase #1 read
-  //    iov[0] ----> struct ClusterMsgHeader
-  ///////////////////////////////////////////////////////////////////
-  // Read iovec[] layout phase #2 read
-  //    iov[0] ----> struct descriptor[count]
-  //                 char short_control_messages[control_bytes]
-  ///////////////////////////////////////////////////////////////////
-  // Read iovec[] layout phase #3 read
-  //    iov[0] ----> struct descriptor data (element #1)
-  //    ......
-  //    iov[count-1] ----> struct descriptor data (element #count)
-  ///////////////////////////////////////////////////////////////////
-  int i, n;
-  // This isn't used.
-  // MIOBuffer      *w;
-
-  ink_hrtime now      = Thread::get_hrtime();
-  ClusterState &s     = (read_flag ? read : write);
-  OutgoingControl *oc = s.msg.outgoing_control.head;
-  IncomingControl *ic = incoming_control.head;
-  int new_n_iov       = 0;
-  int to_do           = 0;
-  int len;
-
-  ink_assert(s.iov);
-
-  if (!read_flag) {
-    //////////////////////////////////////////////////////////////////////
-    // Setup vector for write of header, descriptors and control data
-    //////////////////////////////////////////////////////////////////////
-    len                       = sizeof(ClusterMsgHeader) + (s.msg.count * sizeof(Descriptor)) + s.msg.control_bytes;
-    s.iov[new_n_iov].iov_base = nullptr;
-    s.iov[new_n_iov].iov_len  = len;
-    s.block[new_n_iov]        = s.msg.get_block_header();
-
-    // Make read_avail == len
-    s.block[new_n_iov]->fill(len);
-
-    to_do += len;
-    ++new_n_iov;
-
-  } else {
-    if (s.msg.state == 0) {
-      ////////////////////////////////////
-      // Setup vector for read of header
-      ////////////////////////////////////
-      len                       = sizeof(ClusterMsgHeader);
-      s.iov[new_n_iov].iov_base = nullptr;
-      s.iov[new_n_iov].iov_len  = len;
-      s.block[new_n_iov]        = s.msg.get_block_header();
-
-      // Make write_avail == len
-      s.block[new_n_iov]->_buf_end = s.block[new_n_iov]->end() + len;
-
-      to_do += len;
-      ++new_n_iov;
-
-    } else if (s.msg.state == 1) {
-      /////////////////////////////////////////////////////////
-      // Setup vector for read of Descriptors+control data
-      /////////////////////////////////////////////////////////
-      len                       = (s.msg.count * sizeof(Descriptor)) + s.msg.control_bytes;
-      s.iov[new_n_iov].iov_base = nullptr;
-      s.iov[new_n_iov].iov_len  = len;
-      s.block[new_n_iov]        = s.msg.get_block_descriptor();
-
-      // Make write_avail == len
-      s.block[new_n_iov]->_buf_end = s.block[new_n_iov]->end() + len;
-
-      to_do += s.iov[new_n_iov].iov_len;
-      ++new_n_iov;
-    }
-  }
-
-  ////////////////////////////////////////////////////////////
-  // Build vector for data section of the cluster message.
-  // For read, we only do this if we are in data phase
-  // of the read (msg.state == 2)
-  //////////////////////////////////////////////////////////////
-  //  Note: We are assuming that free space descriptors follow
-  //        the data descriptors.
-  //////////////////////////////////////////////////////////////
-  for (i = 0; i < (read_flag ? ((s.msg.state >= 2) ? s.msg.count : 0) : s.msg.count); i++) {
-    if (s.msg.descriptor[i].type == CLUSTER_SEND_DATA) {
-      ///////////////////////////////////
-      // Control channel data
-      ///////////////////////////////////
-      if (s.msg.descriptor[i].channel == CLUSTER_CONTROL_CHANNEL) {
-        if (read_flag) {
-          ///////////////////////
-          // Incoming Control
-          ///////////////////////
-          if (!ic) {
-            ic                  = IncomingControl::alloc();
-            ic->recognized_time = now;
-            CLUSTER_INCREMENT_DYN_STAT(CLUSTER_CTRL_MSGS_RECVD_STAT);
-            ic->len = s.msg.descriptor[i].length;
-            ic->alloc_data();
-            if (!ic->fast_data()) {
-              CLUSTER_INCREMENT_DYN_STAT(CLUSTER_SLOW_CTRL_MSGS_RECVD_STAT);
-            }
-            // Mark message data as invalid
-            *((uint32_t *)ic->data) = UNDEFINED_CLUSTER_FUNCTION;
-            incoming_control.enqueue(ic);
-          }
-          s.iov[new_n_iov].iov_base = nullptr;
-          s.iov[new_n_iov].iov_len  = ic->len;
-          s.block[new_n_iov]        = ic->get_block();
-          to_do += s.iov[new_n_iov].iov_len;
-          ++new_n_iov;
-          ic = (IncomingControl *)ic->link.next;
-        } else {
-          ///////////////////////
-          // Outgoing Control
-          ///////////////////////
-          ink_assert(oc);
-          s.iov[new_n_iov].iov_base = nullptr;
-          s.iov[new_n_iov].iov_len  = oc->len;
-          s.block[new_n_iov]        = oc->get_block();
-          to_do += s.iov[new_n_iov].iov_len;
-          ++new_n_iov;
-          oc = (OutgoingControl *)oc->link.next;
-        }
-      } else {
-        ///////////////////////////////
-        // User channel data
-        ///////////////////////////////
-        ClusterVConnection *vc = channels[s.msg.descriptor[i].channel];
-
-        if (VALID_CHANNEL(vc) && (s.msg.descriptor[i].sequence_number) == CLUSTER_SEQUENCE_NUMBER(vc->token.sequence_number)) {
-          if (read_flag) {
-            ink_release_assert(!vc->initial_data_bytes);
-            /////////////////////////////////////
-            // Try to get the read VIO mutex
-            /////////////////////////////////////
-            ink_release_assert(!vc->read_locked);
-#ifdef CLUSTER_TOMCAT
-            if (!vc->read.vio.mutex ||
-                !MUTEX_TAKE_TRY_LOCK_FOR_SPIN(vc->read.vio.mutex, thread, vc->read.vio._cont, READ_LOCK_SPIN_COUNT))
-#else
-            if (!MUTEX_TAKE_TRY_LOCK_FOR_SPIN(vc->read.vio.mutex, thread, vc->read.vio._cont, READ_LOCK_SPIN_COUNT))
-#endif
-            {
-              vc->read_locked = nullptr;
-            } else {
-              vc->read_locked = vc->read.vio.mutex;
-            }
-
-            ///////////////////////////////////////
-            // Allocate read data block
-            ///////////////////////////////////////
-            if (s.msg.descriptor[i].length) {
-              vc->iov_map = new_n_iov;
-            } else {
-              vc->iov_map = CLUSTER_IOV_NONE;
-            }
-            if (vc->pending_remote_fill || vc_ok_read(vc)) {
-              //////////////////////////////////////////////////////////
-              // Initial and subsequent data on open read channel.
-              // Allocate IOBufferBlock.
-              //////////////////////////////////////////////////////////
-              ink_release_assert(s.msg.descriptor[i].length <= DEFAULT_MAX_BUFFER_SIZE);
-              vc->read_block = new_IOBufferBlock();
-              int64_t index  = buffer_size_to_index(s.msg.descriptor[i].length, MAX_BUFFER_SIZE_INDEX);
-              vc->read_block->alloc(index);
-
-              s.iov[new_n_iov].iov_base = nullptr;
-              s.block[new_n_iov]        = vc->read_block->clone();
-
-            } else {
-              Debug(CL_NOTE, "dumping cluster read data");
-              s.iov[new_n_iov].iov_base = nullptr;
-              s.block[new_n_iov]        = new_IOBufferBlock();
-              s.block[new_n_iov]->set(new_constant_IOBufferData(channel_dummy_input, DEFAULT_MAX_BUFFER_SIZE));
-            }
-
-            // Make block write_avail == descriptor[].length
-            s.block[new_n_iov]->_buf_end = s.block[new_n_iov]->end() + s.msg.descriptor[i].length;
-
-          } else {
-            bool remote_write_fill = (vc->pending_remote_fill && vc->remote_write_block);
-            // Sanity check, assert we have the lock
-            if (!remote_write_fill) {
-              ink_assert(vc->write_locked);
-            }
-            if (vc_ok_write(vc) || remote_write_fill) {
-              if (remote_write_fill) {
-                s.iov[new_n_iov].iov_base = nullptr;
-                ink_release_assert((int)s.msg.descriptor[i].length == bytes_IOBufferBlockList(vc->remote_write_block, 1));
-                s.block[new_n_iov] = vc->remote_write_block;
-
-              } else {
-                s.iov[new_n_iov].iov_base = nullptr;
-                ink_release_assert((int)s.msg.descriptor[i].length <= vc->write_list_bytes);
-                s.block[new_n_iov] = vc->write_list;
-                vc->write_list     = consume_IOBufferBlockList(vc->write_list, (int)s.msg.descriptor[i].length);
-                vc->write_list_bytes -= (int)s.msg.descriptor[i].length;
-                vc->write_bytes_in_transit += (int)s.msg.descriptor[i].length;
-
-                vc->write_list_tail = vc->write_list.get();
-                while (vc->write_list_tail && vc->write_list_tail->next) {
-                  vc->write_list_tail = vc->write_list_tail->next.get();
-                }
-              }
-            } else {
-              Debug(CL_NOTE, "faking cluster write data");
-              s.iov[new_n_iov].iov_base = nullptr;
-              s.block[new_n_iov]        = new_IOBufferBlock();
-              s.block[new_n_iov]->set(new_constant_IOBufferData(channel_dummy_output, DEFAULT_MAX_BUFFER_SIZE));
-              // Make block read_avail == descriptor[].length
-              s.block[new_n_iov]->fill(s.msg.descriptor[i].length);
-            }
-          }
-        } else {
-          // VC has been deleted, need to dump the bits...
-          s.iov[new_n_iov].iov_base = nullptr;
-          s.block[new_n_iov]        = new_IOBufferBlock();
-
-          if (read_flag) {
-            s.block[new_n_iov]->set(new_constant_IOBufferData(channel_dummy_input, DEFAULT_MAX_BUFFER_SIZE));
-
-            // Make block write_avail == descriptor[].length
-            s.block[new_n_iov]->_buf_end = s.block[new_n_iov]->end() + s.msg.descriptor[i].length;
-
-          } else {
-            s.block[new_n_iov]->set(new_constant_IOBufferData(channel_dummy_output, DEFAULT_MAX_BUFFER_SIZE));
-
-            // Make block read_avail == descriptor[].length
-            s.block[new_n_iov]->fill(s.msg.descriptor[i].length);
-          }
-        }
-        s.iov[new_n_iov].iov_len = s.msg.descriptor[i].length;
-        to_do += s.iov[new_n_iov].iov_len;
-        ++new_n_iov;
-      }
-    }
-  }
-  // Release IOBufferBlock references used in previous i/o operation
-  for (n = new_n_iov; n < MAX_TCOUNT; ++n) {
-    s.block[n] = nullptr;
-  }
-
-  // Initialize i/o state variables
-  s.to_do = to_do;
-  s.did   = 0;
-  s.n_iov = new_n_iov;
-  return true;
-
-// TODO: This is apparently dead code, I added the #if 0 to avoid compiler
-// warnings, but is this really intentional??
-#if 0
-  // Release all IOBufferBlock references.
-  for (n = 0; n < MAX_TCOUNT; ++n) {
-    s.block[n] = 0;
-  }
-  CLUSTER_INCREMENT_DYN_STAT(CLUSTER_OP_DELAYED_FOR_LOCK_STAT);
-  Debug(CL_WARN, "%s delayed for locks", read_flag ? "read" : "write");
-  free_locks(read_flag, i);
-  return false;
-#endif
-}
-
-bool
-ClusterHandler::get_read_locks()
-{
-  ///////////////////////////////////////////////////////////////////////
-  // Reacquire locks for the request setup by build_initial_vector().
-  // We are called after each read completion prior to posting completion
-  ///////////////////////////////////////////////////////////////////////
-  ClusterState &s = read;
-  int i, n;
-  int bytes_processed;
-  int vec_bytes_remainder;
-  int iov_done[MAX_TCOUNT];
-
-  memset((char *)iov_done, 0, sizeof(int) * MAX_TCOUNT);
-
-  // Compute bytes transferred on a per vector basis
-  bytes_processed = s.did - s.bytes_xfered; // not including bytes in this xfer
-
-  i = -1;
-  for (n = 0; n < s.n_iov; ++n) {
-    bytes_processed -= s.iov[n].iov_len;
-    if (bytes_processed >= 0) {
-      iov_done[n] = s.iov[n].iov_len;
-    } else {
-      iov_done[n] = s.iov[n].iov_len + bytes_processed;
-      if (i < 0) {
-        i = n; // note i/o start vector
-
-        // Now at vector where last transfer started,
-        // make considerations for the last transfer on this vector.
-
-        vec_bytes_remainder = (s.iov[n].iov_len - iov_done[n]);
-        bytes_processed     = s.bytes_xfered;
-
-        bytes_processed -= vec_bytes_remainder;
-        if (bytes_processed >= 0) {
-          iov_done[n] = vec_bytes_remainder;
-        } else {
-          iov_done[n] = vec_bytes_remainder + bytes_processed;
-          break;
-        }
-      } else {
-        break;
-      }
-    }
-  }
-  ink_release_assert(i >= 0);
-
-  // Start lock acquisition at the first vector where we started
-  //  the last read.
-  //
-  //  Note: We are assuming that free space descriptors follow
-  //        the data descriptors.
-
-  for (; i < s.n_iov; ++i) {
-    if ((s.msg.descriptor[i].type == CLUSTER_SEND_DATA) && (s.msg.descriptor[i].channel != CLUSTER_CONTROL_CHANNEL)) {
-      // Only user channels require locks
-
-      ClusterVConnection *vc = channels[s.msg.descriptor[i].channel];
-      if (!VALID_CHANNEL(vc) || ((s.msg.descriptor[i].sequence_number) != CLUSTER_SEQUENCE_NUMBER(vc->token.sequence_number)) ||
-          !vc_ok_read(vc)) {
-        // Channel no longer valid, lock not needed since we
-        //  already have a reference to the buffer
-        continue;
-      }
-
-      ink_assert(!vc->read_locked);
-      vc->read_locked = vc->read.vio.mutex;
-      if (vc->byte_bank_q.head ||
-          !MUTEX_TAKE_TRY_LOCK_FOR_SPIN(vc->read.vio.mutex, thread, vc->read.vio._cont, READ_LOCK_SPIN_COUNT)) {
-        // Pending byte bank completions or lock acquire failure.
-
-        vc->read_locked = nullptr;
-        continue;
-      }
-      // Since we now have the mutex, really see if reads are allowed.
-
-      if (!vc_ok_read(vc)) {
-        MUTEX_UNTAKE_LOCK(vc->read.vio.mutex, thread);
-        vc->read_locked = nullptr;
-        continue;
-      }
-      // Lock acquire success, move read bytes into VC
-
-      int64_t read_avail = vc->read_block->read_avail();
-
-      if (!vc->pending_remote_fill && read_avail) {
-        Debug("cluster_vc_xfer", "Deferred fill ch %d %p %" PRId64 " bytes", vc->channel, vc, read_avail);
-
-        vc->read.vio.buffer.writer()->append_block(vc->read_block->clone());
-        if (complete_channel_read(read_avail, vc)) {
-          vc->read_block->consume(read_avail);
-        }
-      }
-    }
-  }
-  return true; // success
-}
-
-bool
-ClusterHandler::get_write_locks()
-{
-  ///////////////////////////////////////////////////////////////////////
-  // Reacquire locks for the request setup by build_initial_vector().
-  // We are called after the entire write completes prior to
-  // posting completion.
-  ///////////////////////////////////////////////////////////////////////
-  ClusterState &s = write;
-  int i;
-
-  for (i = 0; i < s.msg.count; ++i) {
-    if ((s.msg.descriptor[i].type == CLUSTER_SEND_DATA) && (s.msg.descriptor[i].channel != CLUSTER_CONTROL_CHANNEL)) {
-      // Only user channels require locks
-
-      ClusterVConnection *vc = channels[s.msg.descriptor[i].channel];
-      if (!VALID_CHANNEL(vc) || (s.msg.descriptor[i].sequence_number) != CLUSTER_SEQUENCE_NUMBER(vc->token.sequence_number)) {
-        // Channel no longer valid, lock not needed since we
-        //  already have a reference to the buffer
-        continue;
-      }
-      ink_assert(!vc->write_locked);
-      vc->write_locked = vc->write.vio.mutex;
-#ifdef CLUSTER_TOMCAT
-      if (vc->write_locked &&
-          !MUTEX_TAKE_TRY_LOCK_FOR_SPIN(vc->write.vio.mutex, thread, vc->write.vio._cont, WRITE_LOCK_SPIN_COUNT)) {
-#else
-      if (!MUTEX_TAKE_TRY_LOCK_FOR_SPIN(vc->write.vio.mutex, thread, vc->write.vio._cont, WRITE_LOCK_SPIN_COUNT)) {
-#endif
-        // write lock acquire failed, free all acquired locks and retry later
-        vc->write_locked = nullptr;
-        free_locks(CLUSTER_WRITE, i);
-        return false;
-      }
-    }
-  }
-  return true;
-}
-
-void
-ClusterHandler::swap_descriptor_bytes()
-{
-  for (int i = 0; i < read.msg.count; i++) {
-    read.msg.descriptor[i].SwapBytes();
-  }
-}
-
-void
-ClusterHandler::process_set_data_msgs()
-{
-  uint32_t cluster_function_index;
-  //
-  // Cluster set_data messages must always be processed ahead of all
-  // messages and data.  By convention, set_data messages (highest priority
-  // messages) always reside in the beginning of the descriptor
-  // and small control message structures.
-  //
-
-  /////////////////////////////////////////////
-  // Process small control set_data messages.
-  /////////////////////////////////////////////
-  if (!read.msg.did_small_control_set_data) {
-    char *p    = (char *)&read.msg.descriptor[read.msg.count];
-    char *endp = p + read.msg.control_bytes;
-    while (p < endp) {
-      if (needByteSwap) {
-        ats_swap32((uint32_t *)p);                     // length
-        ats_swap32((uint32_t *)(p + sizeof(int32_t))); // function code
-      }
-      int len                = *(int32_t *)p;
-      cluster_function_index = *(uint32_t *)(p + sizeof(int32_t));
-
-      if ((cluster_function_index < (uint32_t)SIZE_clusterFunction) &&
-          (cluster_function_index == SET_CHANNEL_DATA_CLUSTER_FUNCTION)) {
-        clusterFunction[SET_CHANNEL_DATA_CLUSTER_FUNCTION].pfn(this, p + (2 * sizeof(uint32_t)), len - sizeof(uint32_t));
-        // Mark message as processed.
-        *((uint32_t *)(p + sizeof(uint32_t))) = ~*((uint32_t *)(p + sizeof(uint32_t)));
-        p += (2 * sizeof(uint32_t)) + (len - sizeof(uint32_t));
-        p = (char *)DOUBLE_ALIGN(p);
-      } else {
-        // Reverse swap since this message will be reprocessed.
-
-        if (needByteSwap) {
-          ats_swap32((uint32_t *)p);                     // length
-          ats_swap32((uint32_t *)(p + sizeof(int32_t))); // function code
-        }
-        break; // End of set_data messages
-      }
-    }
-    read.msg.control_data_offset        = p - (char *)&read.msg.descriptor[read.msg.count];
-    read.msg.did_small_control_set_data = 1;
-  }
-  /////////////////////////////////////////////
-  // Process large control set_data messages.
-  /////////////////////////////////////////////
-  if (!read.msg.did_large_control_set_data) {
-    IncomingControl *ic = incoming_control.head;
-
-    while (ic) {
-      if (needByteSwap) {
-        ats_swap32((uint32_t *)ic->data); // function code
-      }
-      cluster_function_index = *((uint32_t *)ic->data);
-
-      if ((cluster_function_index < (uint32_t)SIZE_clusterFunction) &&
-          (cluster_function_index == SET_CHANNEL_DATA_CLUSTER_FUNCTION)) {
-        char *p = ic->data;
-        clusterFunction[SET_CHANNEL_DATA_CLUSTER_FUNCTION].pfn(this, (void *)(p + sizeof(int32_t)), ic->len - sizeof(int32_t));
-
-        // Reverse swap since this will be processed again for deallocation.
-        if (needByteSwap) {
-          ats_swap32((uint32_t *)p);                     // length
-          ats_swap32((uint32_t *)(p + sizeof(int32_t))); // function code
-        }
-        // Mark message as processed.
-        // Defer dellocation until entire read is complete.
-        *((uint32_t *)p) = ~*((uint32_t *)p);
-
-        ic = (IncomingControl *)ic->link.next;
-      } else {
-        // Reverse swap action this message will be reprocessed.
-        if (needByteSwap) {
-          ats_swap32((uint32_t *)ic->data); // function code
-        }
-        break;
-      }
-    }
-    read.msg.did_large_control_set_data = 1;
-  }
-}
-
-void
-ClusterHandler::process_small_control_msgs()
-{
-  if (read.msg.did_small_control_msgs) {
-    return;
-  } else {
-    read.msg.did_small_control_msgs = 1;
-  }
-
-  ink_hrtime now = Thread::get_hrtime();
-  char *p        = (char *)&read.msg.descriptor[read.msg.count] + read.msg.control_data_offset;
-  char *endp     = (char *)&read.msg.descriptor[read.msg.count] + read.msg.control_bytes;
-
-  while (p < endp) {
-    /////////////////////////////////////////////////////////////////
-    // Place non cluster small incoming messages on external
-    // incoming queue for processing by callout threads.
-    /////////////////////////////////////////////////////////////////
-    if (needByteSwap) {
-      ats_swap32((uint32_t *)p);                     // length
-      ats_swap32((uint32_t *)(p + sizeof(int32_t))); // function code
-    }
-    int len = *(int32_t *)p;
-    p += sizeof(int32_t);
-    uint32_t cluster_function_index = *(uint32_t *)p;
-    ink_release_assert(cluster_function_index != SET_CHANNEL_DATA_CLUSTER_FUNCTION);
-
-    if (cluster_function_index >= (uint32_t)SIZE_clusterFunction) {
-      Warning("1Bad cluster function index (small control)");
-      p += len;
-
-    } else if (clusterFunction[cluster_function_index].ClusterFunc) {
-      //////////////////////////////////////////////////////////////////////
-      // Cluster function, can only be processed in ET_CLUSTER thread
-      //////////////////////////////////////////////////////////////////////
-      p += sizeof(uint32_t);
-      clusterFunction[cluster_function_index].pfn(this, p, len - sizeof(int32_t));
-      p += (len - sizeof(int32_t));
-
-    } else {
-      ///////////////////////////////////////////////////////
-      // Non Cluster function, defer to callout threads
-      ///////////////////////////////////////////////////////
-      IncomingControl *ic = IncomingControl::alloc();
-      ic->recognized_time = now;
-      ic->len             = len;
-      ic->alloc_data();
-      memcpy(ic->data, p, ic->len);
-      SetHighBit(&ic->len); // mark as small cntl
-      ink_atomiclist_push(&external_incoming_control, (void *)ic);
-      p += len;
-    }
-    p = (char *)DOUBLE_ALIGN(p);
-  }
-}
-
-void
-ClusterHandler::process_large_control_msgs()
-{
-  if (read.msg.did_large_control_msgs) {
-    return;
-  } else {
-    read.msg.did_large_control_msgs = 1;
-  }
-
-  ////////////////////////////////////////////////////////////////
-  // Place non cluster large incoming messages on external
-  // incoming queue for processing by callout threads.
-  ////////////////////////////////////////////////////////////////
-  IncomingControl *ic = nullptr;
-  uint32_t cluster_function_index;
-
-  while ((ic = incoming_control.dequeue())) {
-    if (needByteSwap) {
-      ats_swap32((uint32_t *)ic->data); // function code
-    }
-    cluster_function_index = *((uint32_t *)ic->data);
-    ink_release_assert(cluster_function_index != SET_CHANNEL_DATA_CLUSTER_FUNCTION);
-
-    if (cluster_function_index == (uint32_t)~SET_CHANNEL_DATA_CLUSTER_FUNCTION) {
-      // SET_CHANNEL_DATA_CLUSTER_FUNCTION already processed.
-      // Just do memory deallocation.
-
-      if (!clusterFunction[SET_CHANNEL_DATA_CLUSTER_FUNCTION].fMalloced)
-        ic->freeall();
-      continue;
-    }
-
-    if (cluster_function_index >= (uint32_t)SIZE_clusterFunction) {
-      Warning("Bad cluster function index (large control)");
-      ic->freeall();
-
-    } else if (clusterFunction[cluster_function_index].ClusterFunc) {
-      // Cluster message, process in ET_CLUSTER thread
-      clusterFunction[cluster_function_index].pfn(this, (void *)(ic->data + sizeof(int32_t)), ic->len - sizeof(int32_t));
-
-      // Deallocate memory
-      if (!clusterFunction[cluster_function_index].fMalloced)
-        ic->freeall();
-
-    } else {
-      // Non Cluster message, process in non ET_CLUSTER thread
-      ink_atomiclist_push(&external_incoming_control, (void *)ic);
-    }
-  }
-}
-
-void
-ClusterHandler::process_freespace_msgs()
-{
-  if (read.msg.did_freespace_msgs) {
-    return;
-  } else {
-    read.msg.did_freespace_msgs = 1;
-  }
-
-  int i;
-  //
-  // unpack CLUSTER_SEND_FREE (VC free space) messages and update
-  // the free space in the target VC(s).
-  //
-  for (i = 0; i < read.msg.count; i++) {
-    if (read.msg.descriptor[i].type == CLUSTER_SEND_FREE && read.msg.descriptor[i].channel != CLUSTER_CONTROL_CHANNEL) {
-      int c = read.msg.descriptor[i].channel;
-      if (c < n_channels && VALID_CHANNEL(channels[c]) &&
-          (CLUSTER_SEQUENCE_NUMBER(channels[c]->token.sequence_number) == read.msg.descriptor[i].sequence_number)) {
-        //
-        // VC received freespace message, move it to the
-        // current bucket, since it may have data to
-        // write (WRITE_VC_PRIORITY).
-        //
-        channels[c]->remote_free = read.msg.descriptor[i].length;
-        vcs_push(channels[c], VC_CLUSTER_WRITE);
-      }
-    }
-  }
-}
-
-void
-ClusterHandler::add_to_byte_bank(ClusterVConnection *vc)
-{
-  ByteBankDescriptor *bb_desc       = ByteBankDescriptor::ByteBankDescriptor_alloc(vc->read_block);
-  bool pending_byte_bank_completion = vc->byte_bank_q.head ? true : false;
-
-  // Put current byte bank descriptor on completion list
-  vc->byte_bank_q.enqueue(bb_desc);
-
-  // Start byte bank completion action if not active
-  if (!pending_byte_bank_completion) {
-    ClusterVC_remove_read(vc);
-    delayed_reads.push(vc);
-    CLUSTER_INCREMENT_DYN_STAT(CLUSTER_LEVEL1_BANK_STAT);
-  } else {
-    CLUSTER_INCREMENT_DYN_STAT(CLUSTER_MULTILEVEL_BANK_STAT);
-  }
-  vc->read_block = nullptr;
-}
-
-void
-ClusterHandler::update_channels_read()
-{
-  //
-  // Update channels from which data has been read.
-  //
-  int i;
-  int len;
-  // This isn't used.
-  // int nread = read.bytes_xfered;
-
-  process_set_data_msgs();
-
-  //
-  // update the ClusterVConnections
-  //
-  for (i = 0; i < read.msg.count; i++) {
-    if (read.msg.descriptor[i].type == CLUSTER_SEND_DATA && read.msg.descriptor[i].channel != CLUSTER_CONTROL_CHANNEL) {
-      ClusterVConnection *vc = channels[read.msg.descriptor[i].channel];
-      if (VALID_CHANNEL(vc) && (read.msg.descriptor[i].sequence_number) == CLUSTER_SEQUENCE_NUMBER(vc->token.sequence_number)) {
-        vc->last_activity_time = current_time; // note activity time
-
-        len = read.msg.descriptor[i].length;
-        if (!len) {
-          continue;
-        }
-
-        if (!vc->pending_remote_fill && vc_ok_read(vc) && (!vc->read_locked || vc->byte_bank_q.head)) {
-          //
-          // Byte bank active or unable to acquire lock on VC.
-          // Move data into the byte bank and attempt delivery
-          // at the next periodic event.
-          //
-          vc->read_block->fill(len); // note bytes received
-          add_to_byte_bank(vc);
-
-        } else {
-          if (vc->pending_remote_fill || (vc->read_locked && vc_ok_read(vc))) {
-            vc->read_block->fill(len); // note bytes received
-            if (!vc->pending_remote_fill) {
-              vc->read.vio.buffer.writer()->append_block(vc->read_block->clone());
-              vc->read_block->consume(len); // note bytes moved to user
-            }
-            complete_channel_read(len, vc);
-          }
-        }
-      }
-    }
-  }
-
-  // Processs control and freespace messages
-  process_small_control_msgs();
-  process_large_control_msgs();
-  process_freespace_msgs();
-}
-
-//
-// This member function is run in a non ET_CLUSTER thread, which
-// performs the input message processing on behalf of ET_CLUSTER.
-// Primary motivation is to allow blocking and unbounded runtime
-// for message processing which cannot be done with a ET_CLUSTER thread.
-//
-int
-ClusterHandler::process_incoming_callouts(ProxyMutex *mutex)
-{
-  ink_hrtime now;
-  //
-  // Atomically dequeue all active requests from the external queue and
-  // move them to the local working queue.  Insertion queue order is
-  // maintained.
-  //
-  Queue<IncomingControl> local_incoming_control;
-  IncomingControl *ic_ext_next;
-  IncomingControl *ic_ext;
-
-  while (true) {
-    ic_ext = (IncomingControl *)ink_atomiclist_popall(&external_incoming_control);
-    if (!ic_ext)
-      break;
-
-    while (ic_ext) {
-      ic_ext_next       = (IncomingControl *)ic_ext->link.next;
-      ic_ext->link.next = nullptr;
-      local_incoming_control.push(ic_ext);
-      ic_ext = ic_ext_next;
-    }
-
-    // Perform callout actions for each message.
-    int small_control_msg;
-    IncomingControl *ic = nullptr;
-
-    while ((ic = local_incoming_control.pop())) {
-      LOG_EVENT_TIME(ic->recognized_time, inmsg_time_dist, inmsg_events);
-
-      // Determine if this a small control message
-      small_control_msg = IsHighBitSet(&ic->len);
-      ClearHighBit(&ic->len); // Clear small msg flag bit
-
-      if (small_control_msg) {
-        int len                         = ic->len;
-        char *p                         = ic->data;
-        uint32_t cluster_function_index = *(uint32_t *)p;
-        p += sizeof(uint32_t);
-
-        if (cluster_function_index < (uint32_t)SIZE_clusterFunction) {
-          ////////////////////////////////
-          // Invoke processing function
-          ////////////////////////////////
-          ink_assert(!clusterFunction[cluster_function_index].ClusterFunc);
-          clusterFunction[cluster_function_index].pfn(this, p, len - sizeof(int32_t));
-          now = Thread::get_hrtime();
-          CLUSTER_SUM_DYN_STAT(CLUSTER_CTRL_MSGS_RECV_TIME_STAT, now - ic->recognized_time);
-        } else {
-          Warning("2Bad cluster function index (small control)");
-        }
-        // Deallocate memory
-        if (!clusterFunction[cluster_function_index].fMalloced)
-          ic->freeall();
-
-      } else {
-        ink_assert(ic->len > 4);
-        uint32_t cluster_function_index = *(uint32_t *)ic->data;
-        bool valid_index;
-
-        if (cluster_function_index < (uint32_t)SIZE_clusterFunction) {
-          valid_index = true;
-          ////////////////////////////////
-          // Invoke processing function
-          ////////////////////////////////
-          ink_assert(!clusterFunction[cluster_function_index].ClusterFunc);
-          clusterFunction[cluster_function_index].pfn(this, (void *)(ic->data + sizeof(int32_t)), ic->len - sizeof(int32_t));
-          now = Thread::get_hrtime();
-          CLUSTER_SUM_DYN_STAT(CLUSTER_CTRL_MSGS_RECV_TIME_STAT, now - ic->recognized_time);
-        } else {
-          valid_index = false;
-          Warning("2Bad cluster function index (large control)");
-        }
-        if (valid_index && !clusterFunction[cluster_function_index].fMalloced)
-          ic->freeall();
-      }
-    }
-  }
-  return EVENT_CONT;
-}
-
-void
-ClusterHandler::update_channels_partial_read()
-{
-  //
-  // We were unable to read the computed amount.  Reflect the partial
-  // amount read in the associated VC read buffer data structures.
-  //
-  int i;
-  int64_t res = read.bytes_xfered;
-
-  if (!res) {
-    return;
-  }
-  ink_assert(res <= read.did);
-
-  // how much of the iov was done
-
-  int64_t iov_done[MAX_TCOUNT];
-  int64_t total        = 0;
-  int64_t already_read = read.did - read.bytes_xfered;
-
-  for (i = 0; i < read.n_iov; i++) {
-    ink_release_assert(already_read >= 0);
-    iov_done[i] = read.iov[i].iov_len;
-
-    // Skip over bytes already processed
-    if (already_read) {
-      already_read -= iov_done[i];
-      if (already_read < 0) {
-        iov_done[i]  = -already_read; // bytes remaining
-        already_read = 0;
-      } else {
-        iov_done[i] = 0;
-        continue;
-      }
-    }
-    // Adjustments for partial read for the current transfer
-    res -= iov_done[i];
-    if (res < 0) {
-      iov_done[i] += res;
-      res = 0;
-    } else {
-      total += iov_done[i];
-    }
-  }
-  ink_assert(total <= read.did);
-
-  int read_all_large_control_msgs = 0;
-  //
-  // update the ClusterVConnections buffer pointers
-  //
-  for (i = 0; i < read.msg.count; i++) {
-    if (read.msg.descriptor[i].type == CLUSTER_SEND_DATA && read.msg.descriptor[i].channel != CLUSTER_CONTROL_CHANNEL) {
-      ClusterVConnection *vc = channels[read.msg.descriptor[i].channel];
-      if (VALID_CHANNEL(vc) && (read.msg.descriptor[i].sequence_number) == CLUSTER_SEQUENCE_NUMBER(vc->token.sequence_number)) {
-        if (vc->pending_remote_fill || (vc_ok_read(vc) && (vc->iov_map != CLUSTER_IOV_NONE))) {
-          vc->last_activity_time = current_time; // note activity time
-          ClusterVConnState *s   = &vc->read;
-          ink_assert(vc->iov_map < read.n_iov);
-          int len = iov_done[vc->iov_map];
-
-          if (len) {
-            if (!read_all_large_control_msgs) {
-              //
-              // Since all large set_data control messages reside at the
-              // beginning, all have been read if the first non-control
-              // descriptor contains > 0 bytes.
-              // Process them ahead of any VC data completion actions
-              // followed by small control and freespace message processing.
-              //
-              process_set_data_msgs();
-              process_small_control_msgs();
-              process_freespace_msgs();
-              read_all_large_control_msgs = 1;
-            }
-            iov_done[vc->iov_map] = 0;
-            vc->read_block->fill(len); // note bytes received
-
-            if (!vc->pending_remote_fill) {
-              if (vc->read_locked) {
-                Debug("cluster_vc_xfer", "Partial read, credit ch %d %p %d bytes", vc->channel, vc, len);
-                s->vio.buffer.writer()->append_block(vc->read_block->clone());
-                if (complete_channel_read(len, vc)) {
-                  vc->read_block->consume(len); // note bytes moved to user
-                }
-
-              } else {
-                // If we have all the data for the VC, move it
-                // into the byte bank.  Otherwise, do nothing since
-                // we will resume the read at this VC.
-
-                if (len == (int)read.msg.descriptor[i].length) {
-                  Debug("cluster_vc_xfer", "Partial read, byte bank move ch %d %p %d bytes", vc->channel, vc, len);
-                  add_to_byte_bank(vc);
-                }
-              }
-            } else {
-              Debug("cluster_vc_xfer", "Partial remote fill read, credit ch %d %p %d bytes", vc->channel, vc, len);
-              complete_channel_read(len, vc);
-            }
-            read.msg.descriptor[i].length -= len;
-            ink_assert(((int)read.msg.descriptor[i].length) >= 0);
-          }
-          Debug(CL_TRACE, "partial_channel_read chan=%d len=%d", vc->channel, len);
-        }
-      }
-    }
-  }
-}
-
-bool
-ClusterHandler::complete_channel_read(int len, ClusterVConnection *vc)
-{
-  //
-  // We have processed a complete VC read request message for a channel,
-  // perform completion actions.
-  //
-  ClusterVConnState *s = &vc->read;
-
-  if (vc->pending_remote_fill) {
-    Debug(CL_TRACE, "complete_channel_read chan=%d len=%d", vc->channel, len);
-    vc->initial_data_bytes += len;
-    ++vc->pending_remote_fill; // Note completion
-    return (vc->closed ? false : true);
-  }
-
-  if (vc->closed)
-    return false; // No action if already closed
-
-  ink_assert(s->vio.mutex == s->vio._cont->mutex);
-
-  Debug("cluster_vc_xfer", "Complete read, credit ch %d %p %d bytes", vc->channel, vc, len);
-  s->vio.ndone += len;
-
-  if (s->vio.ntodo() <= 0) {
-    s->enabled = 0;
-    if (cluster_signal_and_update_locked(VC_EVENT_READ_COMPLETE, vc, s) == EVENT_DONE)
-      return false;
-  } else {
-    if (cluster_signal_and_update_locked(VC_EVENT_READ_READY, vc, s) == EVENT_DONE)
-      return false;
-    if (s->vio.ntodo() <= 0)
-      s->enabled = 0;
-  }
-
-  vcs_push(vc, VC_CLUSTER_READ);
-  return true;
-}
-
-void
-ClusterHandler::finish_delayed_reads()
-{
-  //
-  // Process pending VC delayed reads generated in the last read from
-  // the node to node connection. For explanation of "delayed read" see
-  // comments at the beginning of the member functions for ClusterHandler.
-  //
-  ClusterVConnection *vc = nullptr;
-  DLL<ClusterVConnectionBase> l;
-  while ((vc = (ClusterVConnection *)delayed_reads.pop())) {
-    MUTEX_TRY_LOCK_SPIN(lock, vc->read.vio.mutex, thread, READ_LOCK_SPIN_COUNT);
-    if (lock.is_locked()) {
-      if (vc_ok_read(vc)) {
-        ink_assert(!vc->read.queue);
-        ByteBankDescriptor *d;
-
-        while ((d = vc->byte_bank_q.dequeue())) {
-          if (vc->read.queue) {
-            // Previous complete_channel_read() put us back on the list,
-            //  remove our self to process another byte bank completion
-            ClusterVC_remove_read(vc);
-          }
-          Debug("cluster_vc_xfer", "Delayed read, credit ch %d %p %" PRId64 " bytes", vc->channel, vc,
-                d->get_block()->read_avail());
-          vc->read.vio.buffer.writer()->append_block(d->get_block());
-
-          if (complete_channel_read(d->get_block()->read_avail(), vc)) {
-            ByteBankDescriptor::ByteBankDescriptor_free(d);
-          } else {
-            ByteBankDescriptor::ByteBankDescriptor_free(d);
-            break;
-          }
-        }
-      }
-    } else
-      l.push(vc);
-  }
-  delayed_reads = l;
-}
-
-void
-ClusterHandler::update_channels_written()
-{
-  //
-  // We have sucessfully pushed the write data for the VC(s) described
-  // by the descriptors.
-  // Move the channels in this bucket to a new bucket.
-  // Lower the priority of those with too little data and raise that of
-  // those with too much data.
-  //
-  ink_hrtime now;
-  for (int i = 0; i < write.msg.count; i++) {
-    if (write.msg.descriptor[i].type == CLUSTER_SEND_DATA) {
-      if (write.msg.descriptor[i].channel != CLUSTER_CONTROL_CHANNEL) {
-        ClusterVConnection *vc = channels[write.msg.descriptor[i].channel];
-        if (VALID_CHANNEL(vc) && (write.msg.descriptor[i].sequence_number) == CLUSTER_SEQUENCE_NUMBER(vc->token.sequence_number)) {
-          if (vc->pending_remote_fill) {
-            Debug(CL_TRACE, "update_channels_written chan=%d seqno=%d len=%d", write.msg.descriptor[i].channel,
-                  write.msg.descriptor[i].sequence_number, write.msg.descriptor[i].length);
-            vc->pending_remote_fill = 0;
-            vc->remote_write_block  = nullptr; // free data block
-            continue;                          // ignore remote write fill VC(s)
-          }
-
-          ClusterVConnState *s = &vc->write;
-          int len              = write.msg.descriptor[i].length;
-          vc->write_bytes_in_transit -= len;
-          ink_release_assert(vc->write_bytes_in_transit >= 0);
-          Debug(CL_PROTO, "(%d) data sent %d %" PRId64, write.msg.descriptor[i].channel, len, s->vio.ndone);
-
-          if (vc_ok_write(vc)) {
-            vc->last_activity_time = current_time; // note activity time
-            int64_t ndone          = vc->was_closed() ? 0 : s->vio.ndone;
-
-            if (ndone < vc->remote_free) {
-              vcs_push(vc, VC_CLUSTER_WRITE);
-            }
-          }
-        }
-      } else {
-        //
-        // Free up outgoing control message space
-        //
-        OutgoingControl *oc = write.msg.outgoing_control.dequeue();
-        oc->free_data();
-        oc->mutex = nullptr;
-        now       = Thread::get_hrtime();
-        CLUSTER_SUM_DYN_STAT(CLUSTER_CTRL_MSGS_SEND_TIME_STAT, now - oc->submit_time);
-        LOG_EVENT_TIME(oc->submit_time, cluster_send_time_dist, cluster_send_events);
-        oc->freeall();
-      }
-    }
-  }
-  //
-  // For compound messages, deallocate the data and header descriptors.
-  // The deallocation of the data descriptor will indirectly invoke
-  // the free memory proc described in set_data.
-  //
-  invoke_remote_data_args *args;
-  OutgoingControl *hdr_oc;
-  while ((hdr_oc = write.msg.outgoing_callout.dequeue())) {
-    args = (invoke_remote_data_args *)(hdr_oc->data + sizeof(int32_t));
-    ink_assert(args->magicno == invoke_remote_data_args::MagicNo);
-
-    // Free data descriptor
-    args->data_oc->free_data(); // invoke memory free callback
-    args->data_oc->mutex = nullptr;
-    args->data_oc->freeall();
-
-    // Free descriptor
-    hdr_oc->free_data();
-    hdr_oc->mutex = nullptr;
-    now           = Thread::get_hrtime();
-    CLUSTER_SUM_DYN_STAT(CLUSTER_CTRL_MSGS_SEND_TIME_STAT, now - hdr_oc->submit_time);
-    LOG_EVENT_TIME(hdr_oc->submit_time, cluster_send_time_dist, cluster_send_events);
-    hdr_oc->freeall();
-  }
-}
-
-int
-ClusterHandler::build_write_descriptors()
-{
-  //
-  // Construct the write descriptors for VC write data in the current
-  // write_vcs bucket with considerations for maximum elements per
-  // write (struct iovec system maximum).
-  //
-  int count_bucket            = cur_vcs;
-  int tcount                  = write.msg.count + 2; // count + descriptor
-  int write_descriptors_built = 0;
-  int valid;
-  int list_len = 0;
-  ClusterVConnection *vc, *vc_next;
-
-  //
-  // Build descriptors for connections with stuff to send.
-  //
-  vc = (ClusterVConnection *)ink_atomiclist_popall(&write_vcs_ready);
-  while (vc) {
-    enter_exit(&cls_build_writes_entered, &cls_writes_exited);
-    vc_next              = (ClusterVConnection *)vc->ready_alink.next;
-    vc->ready_alink.next = nullptr;
-    list_len++;
-    if (VC_CLUSTER_CLOSED == vc->type) {
-      vc->in_vcs = false;
-      vc->type   = VC_NULL;
-      clusterVCAllocator.free(vc);
-      vc = vc_next;
-      continue;
-    }
-
-    if (tcount >= MAX_TCOUNT) {
-      vcs_push(vc, VC_CLUSTER_WRITE);
-    } else {
-      vc->in_vcs = false;
-      cluster_reschedule_offset(this, vc, &vc->write, 0);
-      tcount++;
-    }
-    vc = vc_next;
-  }
-  if (list_len) {
-    CLUSTER_SUM_DYN_STAT(CLUSTER_VC_WRITE_LIST_LEN_STAT, list_len);
-  }
-
-  tcount  = write.msg.count + 2;
-  vc_next = (ClusterVConnection *)write_vcs[count_bucket].head;
-  while (vc_next) {
-    vc      = vc_next;
-    vc_next = (ClusterVConnection *)vc->write.link.next;
-
-    if (VC_CLUSTER_CLOSED == vc->type) {
-      vc->type = VC_NULL;
-      clusterVCAllocator.free(vc);
-      continue;
-    }
-
-    if (tcount >= MAX_TCOUNT)
-      break;
-
-    valid = valid_for_data_write(vc);
-    if (-1 == valid) {
-      vcs_push(vc, VC_CLUSTER_WRITE);
-    } else if (valid) {
-      ink_assert(vc->write_locked); // Acquired in valid_for_data_write()
-      if ((vc->remote_free > (vc->write.vio.ndone - vc->write_list_bytes)) && channels[vc->channel] == vc) {
-        ink_assert(vc->write_list && vc->write_list_bytes);
-
-        int d                                   = write.msg.count;
-        write.msg.descriptor[d].type            = CLUSTER_SEND_DATA;
-        write.msg.descriptor[d].channel         = vc->channel;
-        write.msg.descriptor[d].sequence_number = vc->token.sequence_number;
-        int s                                   = vc->write_list_bytes;
-        ink_release_assert(s <= MAX_CLUSTER_SEND_LENGTH);
-
-        // Transfer no more than nbytes
-        if ((vc->write.vio.ndone - s) > vc->write.vio.nbytes)
-          s = vc->write.vio.nbytes - (vc->write.vio.ndone - s);
-
-        if ((vc->write.vio.ndone - s) > vc->remote_free)
-          s                            = vc->remote_free - (vc->write.vio.ndone - s);
-        write.msg.descriptor[d].length = s;
-        write.msg.count++;
-        tcount++;
-        write_descriptors_built++;
-
-#ifdef CLUSTER_STATS
-        _vc_writes++;
-        _vc_write_bytes += s;
-#endif
-
-      } else {
-        MUTEX_UNTAKE_LOCK(vc->write_locked, thread);
-        vc->write_locked = nullptr;
-
-        if (channels[vc->channel] == vc)
-          CLUSTER_INCREMENT_DYN_STAT(CLUSTER_NO_REMOTE_SPACE_STAT);
-      }
-    }
-  }
-  return (write_descriptors_built);
-}
-
-int
-ClusterHandler::build_freespace_descriptors()
-{
-  //
-  // Construct the write descriptors for VC freespace data in the current
-  // read_vcs bucket with considerations for maximum elements per
-  // write (struct iovec system maximum) and for pending elements already
-  // in the list.
-  //
-  int count_bucket                = cur_vcs;
-  int tcount                      = write.msg.count + 2; // count + descriptor require 2 iovec(s)
-  int freespace_descriptors_built = 0;
-  int s                           = 0;
-  int list_len                    = 0;
-  ClusterVConnection *vc, *vc_next;
-
-  //
-  // Build descriptors for available space
-  //
-  vc = (ClusterVConnection *)ink_atomiclist_popall(&read_vcs_ready);
-  while (vc) {
-    enter_exit(&cls_build_reads_entered, &cls_reads_exited);
-    vc_next              = (ClusterVConnection *)vc->ready_alink.next;
-    vc->ready_alink.next = nullptr;
-    list_len++;
-    if (VC_CLUSTER_CLOSED == vc->type) {
-      vc->in_vcs = false;
-      vc->type   = VC_NULL;
-      clusterVCAllocator.free(vc);
-      vc = vc_next;
-      continue;
-    }
-
-    if (tcount >= MAX_TCOUNT) {
-      vcs_push(vc, VC_CLUSTER_READ);
-    } else {
-      vc->in_vcs = false;
-      cluster_reschedule_offset(this, vc, &vc->read, 0);
-      tcount++;
-    }
-    vc = vc_next;
-  }
-  if (list_len) {
-    CLUSTER_SUM_DYN_STAT(CLUSTER_VC_READ_LIST_LEN_STAT, list_len);
-  }
-
-  tcount  = write.msg.count + 2;
-  vc_next = (ClusterVConnection *)read_vcs[count_bucket].head;
-  while (vc_next) {
-    vc      = vc_next;
-    vc_next = (ClusterVConnection *)vc->read.link.next;
-
-    if (VC_CLUSTER_CLOSED == vc->type) {
-      vc->type = VC_NULL;
-      clusterVCAllocator.free(vc);
-      continue;
-    }
-
-    if (tcount >= MAX_TCOUNT)
-      break;
-
-    s = valid_for_freespace_write(vc);
-    if (-1 == s) {
-      vcs_push(vc, VC_CLUSTER_READ);
-    } else if (s) {
-      if (vc_ok_read(vc) && channels[vc->channel] == vc) {
-        // Send free space only if changed
-        int d                                   = write.msg.count;
-        write.msg.descriptor[d].type            = CLUSTER_SEND_FREE;
-        write.msg.descriptor[d].channel         = vc->channel;
-        write.msg.descriptor[d].sequence_number = vc->token.sequence_number;
-
-        ink_assert(s > 0);
-        write.msg.descriptor[d].length = s;
-        vc->last_local_free            = s;
-        Debug(CL_PROTO, "(%d) free space priority %d", vc->channel, vc->read.priority);
-        write.msg.count++;
-        tcount++;
-        freespace_descriptors_built++;
-      }
-    }
-  }
-  return (freespace_descriptors_built);
-}
-
-int
-ClusterHandler::build_controlmsg_descriptors()
-{
-  //
-  // Construct the write descriptors for control message data in the
-  // outgoing_control queue with considerations for maximum elements per
-  // write (struct iovec system maximum) and for elements already
-  // in the list.
-  //
-  int tcount             = write.msg.count + 2; // count + descriptor require 2 iovec(s)
-  int control_msgs_built = 0;
-  bool compound_msg; // msg + chan data
-  //
-  // Build descriptors for control messages
-  //
-  OutgoingControl *c = nullptr;
-  int control_bytes  = 0;
-  int q              = 0;
-
-  while (tcount < (MAX_TCOUNT - 1)) { // -1 to allow for compound messages
-    c = outgoing_control[q].pop();
-    if (!c) {
-      // Move elements from global outgoing_control to local queue
-      OutgoingControl *c_next;
-      c = (OutgoingControl *)ink_atomiclist_popall(&outgoing_control_al[q]);
-      if (c == nullptr) {
-        if (++q >= CLUSTER_CMSG_QUEUES) {
-          break;
-        } else {
-          continue;
-        }
-      }
-      while (c) {
-        c_next       = (OutgoingControl *)c->link.next;
-        c->link.next = nullptr;
-        outgoing_control[q].push(c);
-        c = c_next;
-      }
-      continue;
-
-    } else {
-      compound_msg = (*((int32_t *)c->data) == -1); // (msg+chan data)?
-    }
-    if (!compound_msg && c->len <= SMALL_CONTROL_MESSAGE &&
-        // check if the receiving cluster function will want to malloc'ed data
-        !clusterFunction[*(int32_t *)c->data].fMalloced && control_bytes + c->len + sizeof(int32_t) * 2 + 7 < CONTROL_DATA) {
-      write.msg.outgoing_small_control.enqueue(c);
-      control_bytes += c->len + sizeof(int32_t) * 2 + 7; // safe approximation
-      control_msgs_built++;
-
-      if (clusterFunction[*(int32_t *)c->data].post_pfn) {
-        clusterFunction[*(int32_t *)c->data].post_pfn(this, c->data + sizeof(int32_t), c->len);
-      }
-      continue;
-    }
-    //
-    // Build large control message descriptor
-    //
-    if (compound_msg) {
-      // Extract out components of compound message.
-      invoke_remote_data_args *cmhdr = (invoke_remote_data_args *)(c->data + sizeof(int32_t));
-      OutgoingControl *oc_header     = c;
-      OutgoingControl *oc_msg        = cmhdr->msg_oc;
-      OutgoingControl *oc_data       = cmhdr->data_oc;
-
-      ink_assert(cmhdr->magicno == invoke_remote_data_args::MagicNo);
-      //
-      // Build descriptors and order the data before the reply message.
-      // Reply message processing assumes data completion action performed
-      // prior to processing completion message.
-      // Not an issue today since channel data is always processed first.
-      //
-      int d;
-      d                                       = write.msg.count;
-      write.msg.descriptor[d].type            = CLUSTER_SEND_DATA;
-      write.msg.descriptor[d].channel         = cmhdr->dest_channel;
-      write.msg.descriptor[d].length          = oc_data->len;
-      write.msg.descriptor[d].sequence_number = cmhdr->token.sequence_number;
-
-#ifdef CLUSTER_STATS
-      _vc_write_bytes += oc_data->len;
-#endif
-
-      // Setup remote write fill iovec.  Remote write fills have no VIO.
-      ClusterVConnection *vc = channels[cmhdr->dest_channel];
-
-      if (VALID_CHANNEL(vc) && vc->pending_remote_fill) {
-        ink_release_assert(!vc->remote_write_block);
-        vc->remote_write_block = oc_data->get_block();
-
-        // Note: No array overrun since we are bounded by (MAX_TCOUNT-1).
-        write.msg.count++;
-        tcount++;
-        control_msgs_built++;
-        d = write.msg.count;
-        write.msg.outgoing_control.enqueue(oc_msg);
-        write.msg.descriptor[d].type    = CLUSTER_SEND_DATA;
-        write.msg.descriptor[d].channel = CLUSTER_CONTROL_CHANNEL;
-        write.msg.descriptor[d].length  = oc_msg->len;
-
-#ifdef CLUSTER_STATS
-        _control_write_bytes += oc_msg->len;
-#endif
-
-        write.msg.count++;
-        tcount++;
-        control_msgs_built++;
-
-        // Queue header to process buffer free memory callbacks after send.
-        write.msg.outgoing_callout.enqueue(oc_header);
-
-      } else {
-        // Operation cancelled free memory.
-        Warning("Pending remote read fill aborted chan=%d len=%d", cmhdr->dest_channel, oc_data->len);
-
-        // Free compound message
-        oc_header->free_data();
-        oc_header->mutex = nullptr;
-        oc_header->freeall();
-
-        // Free response message
-        oc_msg->free_data();
-        oc_msg->mutex = nullptr;
-        oc_msg->freeall();
-
-        // Free data descriptor
-        oc_data->free_data(); // invoke memory free callback
-        oc_data->mutex = nullptr;
-        oc_data->freeall();
-      }
-
-    } else {
-      write.msg.outgoing_control.enqueue(c);
-
-      int d                           = write.msg.count;
-      write.msg.descriptor[d].type    = CLUSTER_SEND_DATA;
-      write.msg.descriptor[d].channel = CLUSTER_CONTROL_CHANNEL;
-      write.msg.descriptor[d].length  = c->len;
-
-#ifdef CLUSTER_STATS
-      _control_write_bytes += c->len;
-#endif
-
-      write.msg.count++;
-      tcount++;
-      control_msgs_built++;
-
-      if (clusterFunction[*(int32_t *)c->data].post_pfn) {
-        clusterFunction[*(int32_t *)c->data].post_pfn(this, c->data + sizeof(int32_t), c->len);
-      }
-    }
-  }
-  return control_msgs_built;
-}
-
-int
-ClusterHandler::add_small_controlmsg_descriptors()
-{
-  //
-  // Move small control message data to free space after descriptors
-  //
-  char *p            = (char *)&write.msg.descriptor[write.msg.count];
-  OutgoingControl *c = nullptr;
-
-  while ((c = write.msg.outgoing_small_control.dequeue())) {
-    *(int32_t *)p = c->len;
-    p += sizeof(int32_t);
-    memcpy(p, c->data, c->len);
-    c->free_data();
-    c->mutex = nullptr;
-    p += c->len;
-    ink_hrtime now = Thread::get_hrtime();
-    CLUSTER_SUM_DYN_STAT(CLUSTER_CTRL_MSGS_SEND_TIME_STAT, now - c->submit_time);
-    LOG_EVENT_TIME(c->submit_time, cluster_send_time_dist, cluster_send_events);
-    c->freeall();
-    p = (char *)DOUBLE_ALIGN(p);
-  }
-  write.msg.control_bytes = p - (char *)&write.msg.descriptor[write.msg.count];
-
-#ifdef CLUSTER_STATS
-  _control_write_bytes += write.msg.control_bytes;
-#endif
-
-  return 1;
-}
-
-struct DestructorLock {
-  DestructorLock(EThread *thread)
-  {
-    have_lock = false;
-    t         = thread;
-  }
-  ~DestructorLock()
-  {
-    if (have_lock && m) {
-      Mutex_unlock(m, t);
-    }
-    m = nullptr;
-  }
-  EThread *t;
-  Ptr<ProxyMutex> m;
-  bool have_lock;
-};
-
-int
-ClusterHandler::valid_for_data_write(ClusterVConnection *vc)
-{
-  //
-  // Determine if writes are allowed on this VC
-  //
-  ClusterVConnState *s = &vc->write;
-
-  ink_assert(!on_stolen_thread);
-  ink_assert((ProxyMutex *)!vc->write_locked);
-
-  //
-  // Attempt to get the lock, if we miss, push vc into the future
-  //
-  DestructorLock lock(thread);
-
-retry:
-  if ((lock.m = s->vio.mutex)) {
-    lock.have_lock = MUTEX_TAKE_TRY_LOCK_FOR_SPIN(lock.m, thread, s->vio._cont, WRITE_LOCK_SPIN_COUNT);
-    if (!lock.have_lock) {
-      CLUSTER_INCREMENT_DYN_STAT(CLUSTER_CONNECTIONS_WRITE_LOCKED_STAT);
-
-#ifdef CLUSTER_STATS
-      _dw_missed_lock++;
-#endif
-      return -1;
-    }
-  }
-
-  if (vc->was_closed()) {
-    if (vc->schedule_write()) {
-#ifdef CLUSTER_TOMCAT
-      ink_assert(lock.m);
-#endif
-      vc->write_locked = lock.m;
-      lock.m           = nullptr;
-      lock.have_lock   = false;
-      return 1;
-    } else {
-      if (!vc->write_bytes_in_transit) {
-        close_ClusterVConnection(vc);
-      }
-      return 0;
-    }
-  }
-
-  if (!s->enabled && !vc->was_remote_closed()) {
-#ifdef CLUSTER_STATS
-    _dw_not_enabled++;
-#endif
-    return 0;
-  }
-
-  if (vc->pending_remote_fill) {
-    if (vc->was_remote_closed())
-      close_ClusterVConnection(vc);
-
-#ifdef CLUSTER_STATS
-    _dw_wait_remote_fill++;
-#endif
-    return 0;
-  }
-
-  if (!lock.have_lock || !s->vio.mutex || !s->vio._cont) {
-    if (!lock.have_lock && s->vio.mutex && s->vio._cont) {
-      goto retry;
-    } else {
-// No active VIO
-#ifdef CLUSTER_STATS
-      _dw_no_active_vio++;
-#endif
-      return 0;
-    }
-  }
-  //
-  // If this connection has been closed remotely, send EOS
-  //
-  if (vc->was_remote_closed()) {
-    if (!vc->write_bytes_in_transit && !vc->schedule_write()) {
-      remote_close(vc, s);
-    }
-    return 0;
-  }
-  //
-  // If not enabled or not WRITE
-  //
-  if (!s->enabled || s->vio.op != VIO::WRITE) {
-    s->enabled = 0;
-#ifdef CLUSTER_STATS
-    _dw_not_enabled_or_no_write++;
-#endif
-    return 0;
-  }
-  //
-  // If no room on the remote side or set_data() messages pending
-  //
-  int set_data_msgs_pending = vc->n_set_data_msgs;
-  if (set_data_msgs_pending || (vc->remote_free <= (s->vio.ndone - vc->write_list_bytes))) {
-    if (set_data_msgs_pending) {
-      CLUSTER_INCREMENT_DYN_STAT(CLUSTER_VC_WRITE_STALL_STAT);
-
-#ifdef CLUSTER_STATS
-      _dw_set_data_pending++;
-#endif
-
-    } else {
-#ifdef CLUSTER_STATS
-      _dw_no_free_space++;
-#endif
-    }
-    return 0;
-  }
-  //
-  // Calculate amount writable
-  //
-  MIOBufferAccessor &buf = s->vio.buffer;
-
-  int64_t towrite      = buf.reader()->read_avail();
-  int64_t ntodo        = s->vio.ntodo();
-  bool write_vc_signal = false;
-
-  if (towrite > ntodo)
-    towrite = ntodo;
-
-  ink_assert(ntodo >= 0);
-  if (ntodo <= 0) {
-    cluster_signal_and_update(VC_EVENT_WRITE_COMPLETE, vc, s);
-    return 0;
-  }
-  if (buf.writer()->write_avail() && towrite != ntodo) {
-    write_vc_signal = true;
-    if (cluster_signal_and_update(VC_EVENT_WRITE_READY, vc, s) == EVENT_DONE)
-      return 0;
-    ink_assert(s->vio.ntodo() >= 0);
-    if (s->vio.ntodo() <= 0) {
-      cluster_signal_and_update(VC_EVENT_WRITE_COMPLETE, vc, s);
-      return 0;
-    }
-  }
-  // Clone nbytes of vio.buffer.reader IOBufferBlock list allowing
-  // write_list to contain no more than DEFAULT_MAX_BUFFER_SIZE bytes.
-
-  Ptr<IOBufferBlock> b_list;
-  IOBufferBlock *b_tail;
-  int bytes_to_fill;
-  int consume_bytes;
-
-  bytes_to_fill = DEFAULT_MAX_BUFFER_SIZE - vc->write_list_bytes;
-
-  if (towrite && bytes_to_fill) {
-    consume_bytes = (towrite > bytes_to_fill) ? bytes_to_fill : towrite;
-    b_list = clone_IOBufferBlockList(s->vio.buffer.reader()->block, s->vio.buffer.reader()->start_offset, consume_bytes, &b_tail);
-    ink_assert(b_tail);
-
-    // Append cloned IOBufferBlock list to VC write_list.
-
-    if (vc->write_list_tail) {
-      vc->write_list_tail->next = b_list;
-    } else {
-      vc->write_list = b_list;
-    }
-    vc->write_list_tail = b_tail;
-    vc->write_list_bytes += consume_bytes;
-    ink_assert(bytes_IOBufferBlockList(vc->write_list, 1) == vc->write_list_bytes);
-
-    // We may defer the write, but tell the user we have consumed the data.
-
-    (s->vio.buffer.reader())->consume(consume_bytes);
-    s->vio.ndone += consume_bytes;
-    if (s->vio.ntodo() <= 0) {
-      cluster_signal_and_update_locked(VC_EVENT_WRITE_COMPLETE, vc, s);
-    }
-  }
-
-  if (vc->schedule_write()) {
-#ifdef CLUSTER_TOMCAT
-    ink_assert(s->vio.mutex);
-#endif
-    vc->write_locked = lock.m;
-    lock.m           = nullptr;
-    lock.have_lock   = false;
-    return 1;
-  } else {
-    if (!write_vc_signal && buf.writer()->write_avail() && towrite != ntodo)
-      cluster_signal_and_update(VC_EVENT_WRITE_READY, vc, s);
-    return 0;
-  }
-}
-
-int
-ClusterHandler::valid_for_freespace_write(ClusterVConnection *vc)
-{
-  //
-  // Determine if freespace messages are allowed on this VC
-  //
-  ClusterVConnState *s = &vc->read;
-
-  ink_assert(!on_stolen_thread);
-
-  //
-  // Attempt to get the lock, if we miss, push vc into the future
-  //
-  DestructorLock lock(thread);
-
-retry:
-  if ((lock.m = s->vio.mutex)) {
-    lock.have_lock = MUTEX_TAKE_TRY_LOCK_FOR_SPIN(lock.m, thread, s->vio._cont, READ_LOCK_SPIN_COUNT);
-
-    if (!lock.have_lock) {
-      CLUSTER_INCREMENT_DYN_STAT(CLUSTER_CONNECTIONS_READ_LOCKED_STAT);
-
-#ifdef CLUSTER_STATS
-      _fw_missed_lock++;
-#endif
-      return -1;
-    }
-  }
-  if (vc->was_closed()) {
-    if (!vc->write_bytes_in_transit && !vc->schedule_write()) {
-      close_ClusterVConnection(vc);
-    }
-    return 0;
-  }
-
-  if (!s->enabled && !vc->was_remote_closed()) {
-#ifdef CLUSTER_STATS
-    _fw_not_enabled++;
-#endif
-    return 0;
-  }
-
-  if (vc->pending_remote_fill) {
-    if (vc->was_remote_closed())
-      close_ClusterVConnection(vc);
-
-#ifdef CLUSTER_STATS
-    _fw_wait_remote_fill++;
-#endif
-    return 0;
-  }
-
-  if (!lock.have_lock || !s->vio.mutex || !s->vio._cont) {
-    if (!lock.have_lock && s->vio.mutex && s->vio._cont) {
-      goto retry;
-    } else {
-// No active VIO
-#ifdef CLUSTER_STATS
-      _fw_no_active_vio++;
-#endif
-      return 0;
-    }
-  }
-  //
-  // If this connection has been closed remotely, send EOS
-  //
-  if (vc->was_remote_closed()) {
-    if (vc->write_bytes_in_transit || vc->schedule_write()) {
-      // Defer close until write data is pushed
-      return 0;
-    }
-    remote_close(vc, s);
-    return 0;
-  }
-  //
-  // If not enabled or not WRITE
-  //
-  if (!s->enabled || s->vio.op != VIO::READ) {
-#ifdef CLUSTER_STATS
-    _fw_not_enabled_or_no_read++;
-#endif
-    return 0;
-  }
-
-  int64_t ntodo = s->vio.ntodo();
-  ink_assert(ntodo >= 0);
-
-  if (ntodo <= 0) {
-    cluster_signal_and_update(VC_EVENT_READ_COMPLETE, vc, s);
-    return 0;
-  }
-
-  int64_t bytes_to_move = vc->initial_data_bytes;
-  if (vc->read_block && bytes_to_move) {
-    // Push initial read data into VC
-
-    if (ntodo >= bytes_to_move) {
-      Debug("cluster_vc_xfer", "finish initial data push ch %d bytes %" PRId64, vc->channel, vc->read_block->read_avail());
-
-      s->vio.buffer.writer()->append_block(vc->read_block->clone());
-      vc->read_block = nullptr;
-
-    } else {
-      bytes_to_move = ntodo;
-
-      Debug("cluster_vc_xfer", "initial data push ch %d bytes %" PRId64, vc->channel, bytes_to_move);
-
-      // Clone a portion of the data
-
-      IOBufferBlock *b, *btail;
-      b = clone_IOBufferBlockList(vc->read_block, 0, bytes_to_move, &btail);
-      s->vio.buffer.writer()->append_block(b);
-      vc->read_block->consume(bytes_to_move);
-    }
-    s->vio.ndone += bytes_to_move;
-    vc->initial_data_bytes -= bytes_to_move;
-
-    if (s->vio.ntodo() <= 0) {
-      s->enabled = 0;
-      cluster_signal_and_update_locked(VC_EVENT_READ_COMPLETE, vc, s);
-      return 0;
-
-    } else {
-      if (vc->have_all_data) {
-        if (!vc->read_block) {
-          s->enabled = 0;
-          cluster_signal_and_update(VC_EVENT_EOS, vc, s);
-          return 0;
-        }
-      }
-      if (cluster_signal_and_update_locked(VC_EVENT_READ_READY, vc, s) == EVENT_DONE)
-        return false;
-
-      if (s->vio.ntodo() <= 0)
-        s->enabled = 0;
-
-      if (vc->initial_data_bytes)
-        return 0;
-    }
-  }
-  // At this point, all initial read data passed in the open_read reply
-  // has been moved into the user VC.
-  // Now allow send of freespace to receive additional data.
-
-  int64_t nextfree = vc->read.vio.ndone;
-
-  nextfree = (nextfree + DEFAULT_MAX_BUFFER_SIZE - 1) / DEFAULT_MAX_BUFFER_SIZE;
-  nextfree *= DEFAULT_MAX_BUFFER_SIZE;
-
-  if (nextfree >= (vc->last_local_free / 2)) {
-    nextfree = vc->last_local_free + (8 * DEFAULT_MAX_BUFFER_SIZE);
-  }
-
-  if ((vc->last_local_free == 0) || (nextfree >= vc->last_local_free)) {
-    Debug(CL_PROTO, "(%d) update freespace %" PRId64, vc->channel, nextfree);
-    //
-    // Have good VC candidate locked for freespace write
-    //
-    return nextfree;
-
-  } else {
-    // No free space update required
-    return 0;
-  }
-}
-
-void
-ClusterHandler::vcs_push(ClusterVConnection *vc, int type)
-{
-  if (vc->type <= VC_CLUSTER)
-    vc->type = type;
-
-  while ((vc->type > VC_CLUSTER) && !vc->in_vcs && ink_atomic_cas(pvint32(&vc->in_vcs), 0, 1)) {
-    if (vc->type == VC_CLUSTER_READ)
-      ink_atomiclist_push(&vc->ch->read_vcs_ready, (void *)vc);
-    else
-      ink_atomiclist_push(&vc->ch->write_vcs_ready, (void *)vc);
-    return;
-  }
-}
-
-int
-ClusterHandler::remote_close(ClusterVConnection *vc, ClusterVConnState *ns)
-{
-  if (ns->vio.op != VIO::NONE && !vc->closed) {
-    ns->enabled = 0;
-    if (vc->remote_closed > 0) {
-      if (ns->vio.op == VIO::READ) {
-        if (ns->vio.nbytes == ns->vio.ndone) {
-          return cluster_signal_and_update(VC_EVENT_READ_COMPLETE, vc, ns);
-        } else {
-          return cluster_signal_and_update(VC_EVENT_EOS, vc, ns);
-        }
-      } else {
-        return cluster_signal_and_update(VC_EVENT_EOS, vc, ns);
-      }
-    } else {
-      return cluster_signal_error_and_update(vc, ns, vc->remote_lerrno);
-    }
-  }
-  return EVENT_CONT;
-}
-
-void
-ClusterHandler::steal_thread(EThread *t)
-{
-  //
-  // Attempt to push the control message now instead of waiting
-  // for the periodic event to process it.
-  //
-  if (t != thread &&      // different thread to steal
-      write.to_do <= 0 && // currently not trying to send data
-      // nothing big outstanding
-      !write.msg.count) {
-    mainClusterEvent(CLUSTER_EVENT_STEAL_THREAD, (Event *)t);
-  }
-}
-
-void
-ClusterHandler::free_locks(bool read_flag, int i)
-{
-  //
-  // Free VC locks.  Handle partial acquires up to i
-  //
-  if (i == CLUSTER_FREE_ALL_LOCKS) {
-    if (read_flag) {
-      i = (read.msg.state >= 2 ? read.msg.count : 0);
-    } else {
-      i = write.msg.count;
-    }
-  }
-  ClusterState &s = (read_flag ? read : write);
-  for (int j = 0; j < i; j++) {
-    if (s.msg.descriptor[j].type == CLUSTER_SEND_DATA && s.msg.descriptor[j].channel != CLUSTER_CONTROL_CHANNEL) {
-      ClusterVConnection *vc = channels[s.msg.descriptor[j].channel];
-      if (VALID_CHANNEL(vc)) {
-        if (read_flag) {
-          if (vc->read_locked) {
-            MUTEX_UNTAKE_LOCK(vc->read.vio.mutex, thread);
-            vc->read_locked = nullptr;
-          }
-        } else {
-          if (vc->write_locked) {
-            MUTEX_UNTAKE_LOCK(vc->write_locked, thread);
-            vc->write_locked = nullptr;
-          }
-        }
-      }
-    } else if (!read_flag && s.msg.descriptor[j].type == CLUSTER_SEND_FREE &&
-               s.msg.descriptor[j].channel != CLUSTER_CONTROL_CHANNEL) {
-      ClusterVConnection *vc = channels[s.msg.descriptor[j].channel];
-      if (VALID_CHANNEL(vc)) {
-        if (vc->read_locked) {
-          MUTEX_UNTAKE_LOCK(vc->read_locked, thread);
-          vc->read_locked = nullptr;
-        }
-      }
-    }
-  }
-}
-
-#ifdef CLUSTER_IMMEDIATE_NETIO
-void
-ClusterHandler::build_poll(bool next)
-{
-  Pollfd *pfd;
-  if (next) {
-    pfd     = thread->nextPollDescriptor->alloc();
-    pfd->fd = net_vc->get_socket();
-    ifd     = pfd - thread->nextPollDescriptor->pfd;
-  } else {
-    pfd     = thread->pollDescriptor->alloc();
-    pfd->fd = net_vc->get_socket();
-    ifd     = pfd - thread->pollDescriptor->pfd;
-  }
-  pfd->events = POLLHUP;
-  if (next) {
-    if (read.to_do)
-      pfd->events |= POLLIN;
-    if (write.to_do)
-      pfd->events |= POLLOUT;
-  } else {
-    // we have to lie since we are in the same cycle
-    pfd->events = POLLIN | POLLOUT;
-    // reads/writes are non-blocking anyway
-    pfd->revents = POLLIN | POLLOUT;
-  }
-}
-#endif // CLUSTER_IMMEDIATE_NETIO
-
-extern int CacheClusterMonitorEnabled;
-extern int CacheClusterMonitorIntervalSecs;
-
-//
-// The main event for machine-machine link
-//
-int
-ClusterHandler::mainClusterEvent(int event, Event *e)
-{
-  // Set global time
-  current_time = Thread::get_hrtime();
-
-  if (CacheClusterMonitorEnabled) {
-    if ((current_time - last_trace_dump) > HRTIME_SECONDS(CacheClusterMonitorIntervalSecs)) {
-      last_trace_dump = current_time;
-      dump_internal_data();
-    }
-  }
-//
-// Note: The caller always acquires the ClusterHandler mutex prior
-//       to the call.  This guarantees single threaded access in
-//       mainClusterEvent()
-//
-
-/////////////////////////////////////////////////////////////////////////
-// If cluster interconnect is overloaded, disable remote cluster ops.
-/////////////////////////////////////////////////////////////////////////
-#ifndef DEBUG
-  if (clm && ClusterLoadMonitor::cf_monitor_enabled > 0) {
-#else
-  if (0) {
-#endif
-    bool last_state = disable_remote_cluster_ops;
-    if (clm->is_cluster_overloaded()) {
-      disable_remote_cluster_ops = true;
-    } else {
-      disable_remote_cluster_ops = false;
-    }
-    if (last_state != disable_remote_cluster_ops) {
-      if (disable_remote_cluster_ops) {
-        Note("Network congestion to [%u.%u.%u.%u] encountered, reverting to proxy only mode", DOT_SEPARATED(ip));
-      } else {
-        Note("Network congestion to [%u.%u.%u.%u] cleared, reverting to cache mode", DOT_SEPARATED(ip));
-        last_cluster_op_enable = current_time;
-      }
-    }
-  }
-
-  on_stolen_thread = (event == CLUSTER_EVENT_STEAL_THREAD);
-  bool io_callback = (event == EVENT_IMMEDIATE);
-
-  if (on_stolen_thread) {
-    thread = (EThread *)e;
-  } else {
-    if (io_callback) {
-      thread = this_ethread();
-    } else {
-      thread = e->ethread;
-    }
-  }
-
-  int io_activity = 1;
-  bool only_write_control_msgs;
-  int res;
-
-  while (io_activity) {
-    io_activity             = 0;
-    only_write_control_msgs = false;
-
-    if (downing) {
-      machine_down();
-      break;
-    }
-
-    //////////////////////////
-    // Read Processing
-    //////////////////////////
-    if (!on_stolen_thread) {
-      if (delayed_reads.head) {
-        CLUSTER_INCREMENT_DYN_STAT(CLUSTER_DELAYED_READS_STAT);
-        finish_delayed_reads();
-      }
-      if ((res = process_read(current_time)) < 0) {
-        break;
-      }
-      io_activity += res;
-
-      if (delayed_reads.head) {
-        CLUSTER_INCREMENT_DYN_STAT(CLUSTER_DELAYED_READS_STAT);
-        finish_delayed_reads();
-      }
-    }
-    /////////////////////////
-    // Write Processing
-    /////////////////////////
-    if ((res = process_write(current_time, only_write_control_msgs)) < 0) {
-      break;
-    }
-    io_activity += res;
-
-    /////////////////////////////////////////
-    // Process deferred open_local requests
-    /////////////////////////////////////////
-    if (!on_stolen_thread) {
-      if (do_open_local_requests())
-        thread->signal_hook(thread);
-    }
-  }
-
-#ifdef CLUSTER_IMMEDIATE_NETIO
-  if (!dead && ((event == EVENT_POLL) || (event == EVENT_INTERVAL))) {
-    if (res >= 0) {
-      build_poll(true);
-    }
-  }
-#endif
-  return EVENT_CONT;
-}
-
-int ClusterHandler::process_read(ink_hrtime /* now ATS_UNUSED */)
-{
-#ifdef CLUSTER_STATS
-  _process_read_calls++;
-#endif
-  if (dead) {
-    // Node is down
-    return 0;
-  }
-  ///////////////////////////////
-  // Cluster read state machine
-  ///////////////////////////////
-
-  for (;;) {
-    switch (read.state) {
-    ///////////////////////////////////////////////
-    case ClusterState::READ_START:
-      ///////////////////////////////////////////////
-      {
-#ifdef CLUSTER_STATS
-        _n_read_start++;
-#endif
-        read.msg.clear();
-        read.start_time = Thread::get_hrtime();
-        if (build_initial_vector(CLUSTER_READ)) {
-          read.state = ClusterState::READ_HEADER;
-        } else {
-          return 0;
-        }
-        break;
-      }
-    ///////////////////////////////////////////////
-    case ClusterState::READ_HEADER:
-      ///////////////////////////////////////////////
-      {
-#ifdef CLUSTER_STATS
-        _n_read_header++;
-#endif
-        read.state = ClusterState::READ_AWAIT_HEADER;
-        if (!read.doIO()) {
-          // i/o not initiated, retry later
-          read.state = ClusterState::READ_HEADER;
-          return 0;
-        }
-        break;
-      }
-    ///////////////////////////////////////////////
-    case ClusterState::READ_AWAIT_HEADER:
-      ///////////////////////////////////////////////
-      {
-#ifdef CLUSTER_STATS
-        _n_read_await_header++;
-#endif
-        if (!read.io_complete) {
-          return 0;
-        } else {
-          if (read.io_complete < 0) {
-            // read error, declare node down
-            machine_down();
-            return -1;
-          }
-        }
-        if (read.to_do) {
-          if (read.bytes_xfered) {
-            CLUSTER_INCREMENT_DYN_STAT(CLUSTER_PARTIAL_READS_STAT);
-            read.state = ClusterState::READ_HEADER;
-            break;
-          } else {
-            // Zero byte read
-            read.state = ClusterState::READ_HEADER;
-            return 0;
-          }
-        } else {
-#ifdef MSG_TRACE
-          fprintf(t_fd, "[R] seqno=%d count=%d control_bytes=%d count_check=%d dsum=%d csum=%d\n", read.sequence_number,
-                  read.msg.hdr()->count, read.msg.hdr()->control_bytes, read.msg.hdr()->count_check,
-                  read.msg.hdr()->descriptor_cksum, read.msg.hdr()->control_bytes_cksum);
-          fflush(t_fd);
-#endif
-          CLUSTER_SUM_DYN_STAT(CLUSTER_READ_BYTES_STAT, read.did);
-          if (needByteSwap) {
-            read.msg.hdr()->SwapBytes();
-          }
-          read.msg.count               = read.msg.hdr()->count;
-          read.msg.control_bytes       = read.msg.hdr()->control_bytes;
-          read.msg.descriptor_cksum    = read.msg.hdr()->descriptor_cksum;
-          read.msg.control_bytes_cksum = read.msg.hdr()->control_bytes_cksum;
-          read.msg.unused              = read.msg.hdr()->unused;
-
-          if (MAGIC_COUNT(read) != read.msg.hdr()->count_check) {
-            ink_assert(!"Read bad ClusterMsgHeader data");
-            Warning("Bad ClusterMsgHeader read on [%d.%d.%d.%d], restarting", DOT_SEPARATED(ip));
-            Note("Cluster read from [%u.%u.%u.%u] failed, declaring down", DOT_SEPARATED(ip));
-            machine_down();
-            return -1;
-          }
-
-          if (read.msg.count || read.msg.control_bytes) {
-            read.msg.state++;
-            read.state = ClusterState::READ_SETUP_DESCRIPTOR;
-          } else {
-            read.state = ClusterState::READ_COMPLETE;
-          }
-          break;
-        }
-      }
-    ///////////////////////////////////////////////
-    case ClusterState::READ_SETUP_DESCRIPTOR:
-      ///////////////////////////////////////////////
-      {
-#ifdef CLUSTER_STATS
-        _n_read_setup_descriptor++;
-#endif
-        if (build_initial_vector(CLUSTER_READ)) {
-          read.state = ClusterState::READ_DESCRIPTOR;
-        } else {
-          return 0;
-        }
-        break;
-      }
-    ///////////////////////////////////////////////
-    case ClusterState::READ_DESCRIPTOR:
-      ///////////////////////////////////////////////
-      {
-#ifdef CLUSTER_STATS
-        _n_read_descriptor++;
-#endif
-        read.state = ClusterState::READ_AWAIT_DESCRIPTOR;
-        if (!read.doIO()) {
-          // i/o not initiated, retry later
-          read.state = ClusterState::READ_DESCRIPTOR;
-          return 0;
-        }
-        break;
-      }
-    ///////////////////////////////////////////////
-    case ClusterState::READ_AWAIT_DESCRIPTOR:
-      ///////////////////////////////////////////////
-      {
-#ifdef CLUSTER_STATS
-        _n_read_await_descriptor++;
-#endif
-        if (!read.io_complete) {
-          return 0;
-        } else {
-          if (read.io_complete < 0) {
-            // read error, declare node down
-            machine_down();
-            return -1;
-          }
-        }
-        if (read.to_do) {
-          if (read.bytes_xfered) {
-            CLUSTER_INCREMENT_DYN_STAT(CLUSTER_PARTIAL_READS_STAT);
-            read.state = ClusterState::READ_DESCRIPTOR;
-            break;
-          } else {
-            // Zero byte read
-            read.state = ClusterState::READ_DESCRIPTOR;
-            return 0;
-          }
-        } else {
-#ifdef CLUSTER_MESSAGE_CKSUM
-          ink_release_assert(read.msg.calc_descriptor_cksum() == read.msg.descriptor_cksum);
-          ink_release_assert(read.msg.calc_control_bytes_cksum() == read.msg.control_bytes_cksum);
-#endif
-          CLUSTER_SUM_DYN_STAT(CLUSTER_READ_BYTES_STAT, read.did);
-          if (needByteSwap) {
-            // Descriptors need byte swap
-            swap_descriptor_bytes();
-          }
-          if (read.msg.count == 0) {
-            read.bytes_xfered = 0;
-            read.state        = ClusterState::READ_COMPLETE;
-          } else {
-            read.msg.state++;
-            read.state = ClusterState::READ_SETUP_DATA;
-          }
-          break;
-        }
-      }
-    ///////////////////////////////////////////////
-    case ClusterState::READ_SETUP_DATA:
-      ///////////////////////////////////////////////
-      {
-#ifdef CLUSTER_STATS
-        _n_read_setup_data++;
-#endif
-        if (build_initial_vector(CLUSTER_READ)) {
-          free_locks(CLUSTER_READ);
-          if (read.to_do) {
-            read.state = ClusterState::READ_DATA;
-          } else {
-            // Descriptor contains no VC data
-            read.state = ClusterState::READ_COMPLETE;
-          }
-        } else {
-          return 0;
-        }
-        break;
-      }
-    ///////////////////////////////////////////////
-    case ClusterState::READ_DATA:
-      ///////////////////////////////////////////////
-      {
-#ifdef CLUSTER_STATS
-        _n_read_data++;
-#endif
-        ink_release_assert(read.to_do);
-        read.state = ClusterState::READ_AWAIT_DATA;
-        if (!read.doIO()) {
-          // i/o not initiated, retry later
-          read.state = ClusterState::READ_DATA;
-          return 0;
-        }
-        break;
-      }
-    ///////////////////////////////////////////////
-    case ClusterState::READ_AWAIT_DATA:
-      ///////////////////////////////////////////////
-      {
-#ifdef CLUSTER_STATS
-        _n_read_await_data++;
-#endif
-        if (!read.io_complete) {
-          return 0; // awaiting i/o complete
-        } else {
-          if (read.io_complete > 0) {
-            read.state = ClusterState::READ_POST_COMPLETE;
-          } else {
-            // read error, declare node down
-            machine_down();
-            return -1;
-          }
-        }
-        break;
-      }
-    ///////////////////////////////////////////////
-    case ClusterState::READ_POST_COMPLETE:
-      ///////////////////////////////////////////////
-      {
-#ifdef CLUSTER_STATS
-        _n_read_post_complete++;
-#endif
-        if (!get_read_locks()) {
-          return 0;
-        }
-        if (read.to_do) {
-          if (read.bytes_xfered) {
-            update_channels_partial_read();
-            free_locks(CLUSTER_READ);
-            CLUSTER_SUM_DYN_STAT(CLUSTER_READ_BYTES_STAT, read.bytes_xfered);
-            CLUSTER_INCREMENT_DYN_STAT(CLUSTER_PARTIAL_READS_STAT);
-            read.state = ClusterState::READ_DATA;
-            return 1;
-          } else {
-            // Zero byte read
-            free_locks(CLUSTER_READ);
-            read.state = ClusterState::READ_DATA;
-            return 0;
-          }
-        } else {
-          CLUSTER_SUM_DYN_STAT(CLUSTER_READ_BYTES_STAT, read.bytes_xfered);
-          read.state = ClusterState::READ_COMPLETE;
-          break;
-        }
-      }
-    ///////////////////////////////////////////////
-    case ClusterState::READ_COMPLETE:
-      ///////////////////////////////////////////////
-      {
-#ifdef CLUSTER_STATS
-        _n_read_complete++;
-#endif
-        ink_hrtime rdmsg_end_time = Thread::get_hrtime();
-        CLUSTER_SUM_DYN_STAT(CLUSTER_RDMSG_ASSEMBLE_TIME_STAT, rdmsg_end_time - read.start_time);
-        read.start_time = HRTIME_MSECONDS(0);
-        if (dump_msgs)
-          dump_read_msg();
-        read.sequence_number++;
-        update_channels_read();
-        free_locks(CLUSTER_READ);
-
-        read.state = ClusterState::READ_START;
-        break; // setup next read
-      }
-    //////////////////
-    default:
-      //////////////////
-      {
-        ink_release_assert(!"ClusterHandler::process_read invalid state");
-      }
-
-    } // end of switch
-  }   // end of for
-}
-
-int
-ClusterHandler::process_write(ink_hrtime now, bool only_write_control_msgs)
-{
-#ifdef CLUSTER_STATS
-  _process_write_calls++;
-#endif
-  /////////////////////////////////
-  // Cluster write state machine
-  /////////////////////////////////
-  for (;;) {
-    switch (write.state) {
-    ///////////////////////////////////////////////
-    case ClusterState::WRITE_START:
-      ///////////////////////////////////////////////
-      {
-#ifdef CLUSTER_STATS
-        _n_write_start++;
-#endif
-        write.msg.clear();
-        write.last_time                  = Thread::get_hrtime();
-        pw_write_descriptors_built       = -1;
-        pw_freespace_descriptors_built   = -1;
-        pw_controldata_descriptors_built = -1;
-        pw_time_expired                  = 0;
-        write.state                      = ClusterState::WRITE_SETUP;
-        break;
-      }
-    ///////////////////////////////////////////////
-    case ClusterState::WRITE_SETUP:
-      ///////////////////////////////////////////////
-      {
-#ifdef CLUSTER_STATS
-        _n_write_setup++;
-#endif
-        if (!on_stolen_thread && !only_write_control_msgs) {
-          /////////////////////////////////////////////////////////////
-          // Build a complete write descriptor containing control,
-          // data and freespace message data.
-          /////////////////////////////////////////////////////////////
-
-          // Control message descriptors
-          if (pw_controldata_descriptors_built) {
-            pw_controldata_descriptors_built = build_controlmsg_descriptors();
-          }
-          // Write data descriptors
-          if (pw_write_descriptors_built) {
-            pw_write_descriptors_built = build_write_descriptors();
-          }
-          // Free space descriptors
-          if (pw_freespace_descriptors_built) {
-            pw_freespace_descriptors_built = build_freespace_descriptors();
-          }
-          add_small_controlmsg_descriptors(); // always last
-        } else {
-          /////////////////////////////////////////////////////////////
-          // Build a write descriptor only containing control data.
-          /////////////////////////////////////////////////////////////
-          pw_write_descriptors_built       = 0;
-          pw_freespace_descriptors_built   = 0;
-          pw_controldata_descriptors_built = build_controlmsg_descriptors();
-          add_small_controlmsg_descriptors(); // always last
-        }
-
-        // If nothing to write, post write completion
-        if (!pw_controldata_descriptors_built && !pw_write_descriptors_built && !pw_freespace_descriptors_built) {
-          write.state = ClusterState::WRITE_COMPLETE;
-          break;
-        } else {
-          started_on_stolen_thread = on_stolen_thread;
-          control_message_write    = only_write_control_msgs;
-        }
-
-// Move required data into the message header
-#ifdef CLUSTER_MESSAGE_CKSUM
-        write.msg.descriptor_cksum        = write.msg.calc_descriptor_cksum();
-        write.msg.hdr()->descriptor_cksum = write.msg.descriptor_cksum;
-
-        write.msg.control_bytes_cksum        = write.msg.calc_control_bytes_cksum();
-        write.msg.hdr()->control_bytes_cksum = write.msg.control_bytes_cksum;
-        write.msg.unused                     = 0;
-#endif
-        write.msg.hdr()->count         = write.msg.count;
-        write.msg.hdr()->control_bytes = write.msg.control_bytes;
-        write.msg.hdr()->count_check   = MAGIC_COUNT(write);
-
-        ink_release_assert(build_initial_vector(CLUSTER_WRITE));
-        free_locks(CLUSTER_WRITE);
-        write.state = ClusterState::WRITE_INITIATE;
-        break;
-      }
-    ///////////////////////////////////////////////
-    case ClusterState::WRITE_INITIATE:
-      ///////////////////////////////////////////////
-      {
-#ifdef CLUSTER_STATS
-        _n_write_initiate++;
-#endif
-        write.state = ClusterState::WRITE_AWAIT_COMPLETION;
-        if (!write.doIO()) {
-          // i/o not initiated, retry later
-          write.state = ClusterState::WRITE_INITIATE;
-          return 0;
-        }
-        break;
-      }
-    ///////////////////////////////////////////////
-    case ClusterState::WRITE_AWAIT_COMPLETION:
-      ///////////////////////////////////////////////
-      {
-#ifdef CLUSTER_STATS
-        _n_write_await_completion++;
-#endif
-        if (!write.io_complete) {
-          // Still waiting for write i/o completion
-          return 0;
-        } else {
-          if (write.io_complete < 0) {
-            // write error, declare node down
-            machine_down();
-            write.state = ClusterState::WRITE_INITIATE;
-            break;
-          }
-          if (write.to_do) {
-            if (write.bytes_xfered) {
-              CLUSTER_INCREMENT_DYN_STAT(CLUSTER_PARTIAL_WRITES_STAT);
-              write.state = ClusterState::WRITE_INITIATE;
-              break;
-            } else {
-              // Zero byte write
-              write.state = ClusterState::WRITE_INITIATE;
-              return 0;
-            }
-          }
-          CLUSTER_SUM_DYN_STAT(CLUSTER_WRITE_BYTES_STAT, write.bytes_xfered);
-          write.sequence_number++;
-          write.state = ClusterState::WRITE_POST_COMPLETE;
-        }
-        break;
-      }
-    ///////////////////////////////////////////////
-    case ClusterState::WRITE_POST_COMPLETE:
-      ///////////////////////////////////////////////
-      {
-#ifdef CLUSTER_STATS
-        _n_write_post_complete++;
-#endif
-        if (!get_write_locks()) {
-          CLUSTER_INCREMENT_DYN_STAT(CLUSTER_WRITE_LOCK_MISSES_STAT);
-          return 0;
-        }
-        //
-        // Move the channels into their new buckets based on how much
-        // was written
-        //
-        update_channels_written();
-        free_locks(CLUSTER_WRITE);
-        write.state = ClusterState::WRITE_COMPLETE;
-        break;
-      }
-    ///////////////////////////////////////////////
-    case ClusterState::WRITE_COMPLETE:
-      ///////////////////////////////////////////////
-      {
-#ifdef CLUSTER_STATS
-        _n_write_complete++;
-#endif
-        write.state        = ClusterState::WRITE_START;
-        ink_hrtime curtime = Thread::get_hrtime();
-
-        if (!on_stolen_thread) {
-          //
-          // Complete all work in the current bucket before moving to next
-          //
-          pw_time_expired = (curtime - now) > CLUSTER_MAX_RUN_TIME;
-
-          if (!control_message_write && !pw_write_descriptors_built && !pw_freespace_descriptors_built &&
-              !pw_controldata_descriptors_built) {
-            // skip to the next bucket
-            cur_vcs = (cur_vcs + 1) % CLUSTER_BUCKETS;
-          }
-        } else {
-          //
-          // Place an upper bound on thread stealing
-          //
-          pw_time_expired = (curtime - now) > CLUSTER_MAX_THREAD_STEAL_TIME;
-          if (pw_time_expired) {
-            CLUSTER_INCREMENT_DYN_STAT(CLUSTER_THREAD_STEAL_EXPIRES_STAT);
-          }
-        }
-        //
-        // periodic activities
-        //
-        if (!on_stolen_thread && !cur_vcs && !dead) {
-          //
-          // check if this machine is supposed to be in the cluster
-          //
-          MachineList *mc = the_cluster_machines_config();
-          if (mc && !mc->find(ip, port)) {
-            Note("Cluster [%u.%u.%u.%u:%d] not in config, declaring down", DOT_SEPARATED(ip), port);
-            machine_down();
-          }
-        }
-        if (pw_time_expired) {
-          return -1; // thread run time expired
-        } else {
-          if (pw_write_descriptors_built || pw_freespace_descriptors_built || pw_controldata_descriptors_built) {
-            break; // start another write
-          } else {
-            return 0; // no more data to write
-          }
-        }
-      }
-    //////////////////
-    default:
-      //////////////////
-      {
-        ink_release_assert(!"ClusterHandler::process_write invalid state");
-      }
-
-    } // End of switch
-  }   // End of for
-}
-
-int
-ClusterHandler::do_open_local_requests()
-{
-  //
-  // open_local requests which are unable to obtain the ClusterHandler
-  // mutex are deferred and placed onto external_incoming_open_local queue.
-  // It is here where we process the open_local requests within the
-  // ET_CLUSTER thread.
-  //
-  int pending_request = 0;
-  ClusterVConnection *cvc;
-  ClusterVConnection *cvc_ext;
-  ClusterVConnection *cvc_ext_next;
-  EThread *tt = this_ethread();
-  Queue<ClusterVConnection> local_incoming_open_local;
-
-  //
-  // Atomically dequeue all requests from the external queue and
-  // move them to the local working queue while maintaining insertion order.
-  //
-  while (true) {
-    cvc_ext = (ClusterVConnection *)ink_atomiclist_popall(&external_incoming_open_local);
-    if (cvc_ext == nullptr)
-      break;
-
-    while (cvc_ext) {
-      cvc_ext_next       = (ClusterVConnection *)cvc_ext->link.next;
-      cvc_ext->link.next = nullptr;
-      local_incoming_open_local.push(cvc_ext);
-      cvc_ext = cvc_ext_next;
-    }
-
-    // Process deferred open_local requests.
-
-    while ((cvc = local_incoming_open_local.pop())) {
-      MUTEX_TRY_LOCK(lock, cvc->action_.mutex, tt);
-      if (lock.is_locked()) {
-        if (cvc->start(tt) < 0) {
-          cvc->token.clear();
-          if (cvc->action_.continuation) {
-            cvc->action_.continuation->handleEvent(CLUSTER_EVENT_OPEN_FAILED, nullptr);
-            clusterVCAllocator.free(cvc);
-          }
-        }
-        MUTEX_RELEASE(lock);
-
-      } else {
-        // unable to get mutex, insert request back onto global queue.
-        Debug(CL_TRACE, "do_open_local_requests() unable to acquire mutex (cvc=%p)", cvc);
-        pending_request = 1;
-        ink_atomiclist_push(&external_incoming_open_local, (void *)cvc);
-      }
-    }
-  }
-  return pending_request;
-}
-
-// End of ClusterHandler.cc
diff --git a/iocore/cluster/ClusterHandlerBase.cc b/iocore/cluster/ClusterHandlerBase.cc
deleted file mode 100644
index 6202a41..0000000
--- a/iocore/cluster/ClusterHandlerBase.cc
+++ /dev/null
@@ -1,1438 +0,0 @@
-/** @file
-
-  A brief file description
-
-  @section license License
-
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
- */
-
-/****************************************************************************
-
-  ClusterHandlerBase.cc
-****************************************************************************/
-
-#include "P_Cluster.h"
-
-extern int cluster_receive_buffer_size;
-extern int cluster_send_buffer_size;
-extern uint32_t cluster_sockopt_flags;
-extern uint32_t cluster_packet_mark;
-extern uint32_t cluster_packet_tos;
-extern int num_of_cluster_threads;
-
-///////////////////////////////////////////////////////////////
-// Incoming message continuation for periodic callout threads
-///////////////////////////////////////////////////////////////
-
-ClusterCalloutContinuation::ClusterCalloutContinuation(struct ClusterHandler *ch) : Continuation(nullptr), _ch(ch)
-{
-  mutex = new_ProxyMutex();
-  SET_HANDLER((ClstCoutContHandler)&ClusterCalloutContinuation::CalloutHandler);
-}
-
-ClusterCalloutContinuation::~ClusterCalloutContinuation()
-{
-  mutex = nullptr;
-}
-
-int
-ClusterCalloutContinuation::CalloutHandler(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */)
-{
-  return _ch->process_incoming_callouts(this->mutex.get());
-}
-
-/*************************************************************************/
-// ClusterControl member functions (Internal Class)
-/*************************************************************************/
-ClusterControl::ClusterControl()
-  : Continuation(nullptr),
-    len(0),
-    size_index(-1),
-    real_data(nullptr),
-    data(nullptr),
-    free_proc(nullptr),
-    free_proc_arg(nullptr),
-    iob_block(nullptr)
-{
-}
-
-void
-ClusterControl::real_alloc_data(int read_access, bool align_int32_on_non_int64_boundary)
-{
-  ProxyMutex *mutex = this_ethread()->mutex.get();
-
-  ink_assert(!data);
-  if ((len + DATA_HDR + sizeof(int32_t)) <= DEFAULT_MAX_BUFFER_SIZE) {
-    size_index = buffer_size_to_index(len + DATA_HDR + sizeof(int32_t), MAX_BUFFER_SIZE_INDEX);
-    iob_block  = new_IOBufferBlock();
-    iob_block->alloc(size_index); // aligns on 8 byte boundary
-    real_data = (int64_t *)iob_block->buf();
-
-    if (align_int32_on_non_int64_boundary) {
-      data = ((char *)real_data) + sizeof(int32_t) + DATA_HDR;
-    } else {
-      data = ((char *)real_data) + DATA_HDR;
-    }
-  } else {
-    int size   = sizeof(int64_t) * (((len + DATA_HDR + sizeof(int32_t) + sizeof(int64_t) - 1) / sizeof(int64_t)) + 1);
-    size_index = -1;
-    iob_block  = new_IOBufferBlock();
-    iob_block->alloc(BUFFER_SIZE_FOR_XMALLOC(size));
-    real_data = (int64_t *)iob_block->buf();
-
-    if (align_int32_on_non_int64_boundary) {
-      data = (char *)DOUBLE_ALIGN(real_data) + sizeof(int32_t) + DATA_HDR;
-    } else {
-      data = (char *)DOUBLE_ALIGN(real_data) + DATA_HDR;
-    }
-    CLUSTER_INCREMENT_DYN_STAT(CLUSTER_ALLOC_DATA_NEWS_STAT);
-  }
-
-  // IOBufferBlock adjustments
-  if (read_access) {
-    // Make iob_block->read_avail() == len
-    iob_block->fill((char *)data - (char *)real_data);    // skip header
-    iob_block->consume((char *)data - (char *)real_data); // skip header
-    iob_block->fill(len);
-  } else {
-    // Make iob_block->write_avail() == len
-    iob_block->fill((char *)data - (char *)real_data);    // skip header
-    iob_block->consume((char *)data - (char *)real_data); // skip header
-    iob_block->_buf_end = iob_block->end() + len;
-  }
-
-  // Write size_index, magic number and 'this' in leading bytes
-  char *size_index_ptr = (char *)data - DATA_HDR;
-  *size_index_ptr      = size_index;
-  ++size_index_ptr;
-
-  *size_index_ptr = (char)ALLOC_DATA_MAGIC;
-  ++size_index_ptr;
-
-  void *val = (void *)this;
-  memcpy(size_index_ptr, (char *)&val, sizeof(void *));
-}
-
-void
-ClusterControl::free_data()
-{
-  if (data && iob_block) {
-    if (free_proc) {
-      // Free memory via callback proc
-      (*free_proc)(free_proc_arg);
-      iob_block = nullptr; // really free memory
-      return;
-    }
-    if (real_data) {
-      ink_release_assert(*(((uint8_t *)data) - DATA_HDR + 1) == (uint8_t)ALLOC_DATA_MAGIC);
-      *(((uint8_t *)data) - DATA_HDR + 1) = (uint8_t)~ALLOC_DATA_MAGIC;
-
-      ink_release_assert(*(((char *)data) - DATA_HDR) == size_index);
-    } else {
-      // malloc'ed memory, not alloced via real_alloc_data().
-      // Data will be ats_free()'ed when IOBufferBlock is freed
-    }
-    iob_block = nullptr; // free memory
-  }
-}
-
-/*************************************************************************/
-// IncomingControl member functions (Internal Class)
-/*************************************************************************/
-IncomingControl *
-IncomingControl::alloc()
-{
-  return inControlAllocator.alloc();
-}
-
-IncomingControl::IncomingControl() : recognized_time(0)
-{
-}
-
-void
-IncomingControl::freeall()
-{
-  free_data();
-  inControlAllocator.free(this);
-}
-
-/*************************************************************************/
-// OutgoingControl member functions (Internal Class)
-/*************************************************************************/
-OutgoingControl *
-OutgoingControl::alloc()
-{
-  return outControlAllocator.alloc();
-}
-
-OutgoingControl::OutgoingControl() : ch(nullptr), submit_time(0)
-{
-}
-
-int
-OutgoingControl::startEvent(int event, Event *e)
-{
-  //
-  // This event handler is used by ClusterProcessor::invoke_remote()
-  // to delay (CLUSTER_OPT_DELAY) the enqueuing of the control message.
-  //
-  (void)event;
-  (void)e;
-  // verify that the machine has not gone down
-  if (!ch || !ch->thread)
-    return EVENT_DONE;
-
-  int32_t cluster_fn = *(int32_t *)this->data;
-  int32_t pri        = ClusterFuncToQpri(cluster_fn);
-  ink_atomiclist_push(&ch->outgoing_control_al[pri], (void *)this);
-
-  return EVENT_DONE;
-}
-
-void
-OutgoingControl::freeall()
-{
-  free_data();
-  outControlAllocator.free(this);
-}
-
-/*************************************************************************/
-// ClusterState member functions (Internal Class)
-/*************************************************************************/
-ClusterState::ClusterState(ClusterHandler *c, bool read_chan)
-  : Continuation(nullptr),
-    ch(c),
-    read_channel(read_chan),
-    do_iodone_event(false),
-    n_descriptors(0),
-    sequence_number(0),
-    to_do(0),
-    did(0),
-    n_iov(0),
-    io_complete(1),
-    io_complete_event(0),
-    v(nullptr),
-    bytes_xfered(0),
-    last_ndone(0),
-    total_bytes_xfered(0),
-    iov(nullptr),
-    iob_iov(nullptr),
-    byte_bank(nullptr),
-    n_byte_bank(0),
-    byte_bank_size(0),
-    missed(0),
-    missed_msg(false),
-    read_state_t(READ_START),
-    write_state_t(WRITE_START)
-{
-  mutex = new_ProxyMutex();
-  if (read_channel) {
-    state = ClusterState::READ_START;
-    SET_HANDLER(&ClusterState::doIO_read_event);
-  } else {
-    state = ClusterState::WRITE_START;
-    SET_HANDLER(&ClusterState::doIO_write_event);
-  }
-  last_time  = HRTIME_SECONDS(0);
-  start_time = HRTIME_SECONDS(0);
-  int size;
-  //
-  // Note: we allocate space for maximum iovec(s), descriptor(s)
-  //       and small control message data.
-  //
-
-  //////////////////////////////////////////////////
-  // Place an invalid page in front of iovec data.
-  //////////////////////////////////////////////////
-  size_t pagesize = ats_pagesize();
-  size            = ((MAX_TCOUNT + 1) * sizeof(IOVec)) + (2 * pagesize);
-  iob_iov         = new_IOBufferData(BUFFER_SIZE_FOR_XMALLOC(size));
-  char *addr      = (char *)align_pointer_forward(iob_iov->data(), pagesize);
-
-  iov = (IOVec *)(addr + pagesize);
-
-  ///////////////////////////////////////////////////
-  // Place an invalid page in front of message data.
-  ///////////////////////////////////////////////////
-  size                     = sizeof(ClusterMsgHeader) + (MAX_TCOUNT + 1) * sizeof(Descriptor) + CONTROL_DATA + (2 * pagesize);
-  msg.iob_descriptor_block = new_IOBufferBlock();
-  msg.iob_descriptor_block->alloc(BUFFER_SIZE_FOR_XMALLOC(size));
-
-  addr = (char *)align_pointer_forward(msg.iob_descriptor_block->data->data(), pagesize);
-
-  addr = addr + pagesize;
-  memset(addr, 0, size - (2 * pagesize));
-  msg.descriptor = (Descriptor *)(addr + sizeof(ClusterMsgHeader));
-
-  mbuf = new_empty_MIOBuffer();
-}
-
-ClusterState::~ClusterState()
-{
-  mutex = nullptr;
-  if (iov) {
-    iob_iov = nullptr; // Free memory
-  }
-
-  if (msg.descriptor) {
-    msg.iob_descriptor_block = nullptr; // Free memory
-  }
-  // Deallocate IO Core structures
-  int n;
-  for (n = 0; n < MAX_TCOUNT; ++n) {
-    block[n] = nullptr;
-  }
-  free_empty_MIOBuffer(mbuf);
-  mbuf = nullptr;
-}
-
-void
-ClusterState::build_do_io_vector()
-{
-  //
-  // Construct the do_io_xxx data structures allowing transfer
-  // of the data described by the iovec structure.
-  //
-  int bytes_to_xfer = 0;
-  int n;
-  IOBufferBlock *last_block = nullptr;
-
-  mbuf->clear();
-
-  // Build the IOBufferBlock chain.
-
-  for (n = 0; n < n_iov; ++n) {
-    bytes_to_xfer += iov[n].iov_len;
-
-    if (last_block) {
-      last_block->next = block[n];
-    }
-    last_block = block[n].get();
-    while (last_block->next) {
-      last_block = last_block->next.get();
-    }
-  }
-
-  mbuf->_writer = block[0];
-  ink_release_assert(bytes_to_xfer == to_do);
-  ink_assert(bytes_to_xfer == bytes_IOBufferBlockList(mbuf->_writer, !read_channel));
-}
-
-#ifdef CLUSTER_TOMCAT
-#define REENABLE_IO()                          \
-  if (!ch->on_stolen_thread && !io_complete) { \
-    v->reenable_re();                          \
-  }
-
-#else // !CLUSTER_TOMCAT
-
-#ifdef CLUSTER_IMMEDIATE_NETIO
-#define REENABLE_IO()                                     \
-  if (!io_complete) {                                     \
-    ((NetVConnection *)v->vc_server)->reenable_re_now(v); \
-  }
-
-#else // !CLUSTER_IMMEDIATE_NETIO
-
-#define REENABLE_IO() \
-  if (!io_complete) { \
-    v->reenable_re(); \
-  }
-#endif // !CLUSTER_IMMEDIATE_NETIO
-
-#endif // !CLUSTER_TOMCAT
-
-int
-ClusterState::doIO()
-{
-  ink_release_assert(io_complete);
-#if !defined(CLUSTER_IMMEDIATE_NETIO)
-  MUTEX_TRY_LOCK(lock, this->mutex, this_ethread());
-  if (!lock.is_locked()) {
-    return 0; // unable to initiate operation
-  }
-#endif
-
-  if (!ch->net_vc) {
-    // Node has gone down, simulate successful transfer
-    io_complete = 1;
-    bytes_xfered += to_do;
-    to_do = 0;
-    return 1;
-  }
-  //
-  // Setup and initiate or resume Cluster i/o request to the NetProcessor.
-  //
-  if ((to_do && (io_complete_event == VC_EVENT_READ_READY)) || (io_complete_event == VC_EVENT_WRITE_READY)) {
-    if (read_channel) {
-      // Partial read case
-      ink_assert(v->buffer.writer()->current_write_avail() == to_do);
-
-    } else {
-      // Partial write case
-      ink_assert(v->buffer.reader()->read_avail() == to_do);
-    }
-
-    // Resume operation
-    v->nbytes = to_do + did;
-    ink_release_assert(v->nbytes > v->ndone);
-
-    io_complete       = false;
-    io_complete_event = 0;
-    REENABLE_IO();
-
-  } else {
-    // Start new do_io_xxx operation.
-    // Initialize globals
-
-    io_complete       = false;
-    io_complete_event = 0;
-    bytes_xfered      = 0;
-    last_ndone        = 0;
-
-    build_do_io_vector();
-
-    if (read_channel) {
-      ink_assert(mbuf->current_write_avail() == to_do);
-#ifdef CLUSTER_IMMEDIATE_NETIO
-      v = ch->net_vc->do_io_read_now(this, to_do, mbuf);
-#else
-      v = ch->net_vc->do_io_read(this, to_do, mbuf);
-#endif
-      REENABLE_IO();
-
-    } else {
-      IOBufferReader *r = mbuf->alloc_reader();
-      r->block          = mbuf->_writer;
-      ink_assert(r->read_avail() == to_do);
-#ifdef CLUSTER_IMMEDIATE_NETIO
-      v = ch->net_vc->do_io_write_now(this, to_do, r);
-#else
-      v = ch->net_vc->do_io_write(this, to_do, r);
-#endif
-      REENABLE_IO();
-    }
-  }
-  return 1; // operation initiated
-}
-
-int
-ClusterState::doIO_read_event(int event, void *d)
-{
-  ink_release_assert(!io_complete);
-  if (!v) {
-    v = (VIO *)d; // Immediate callback on first NetVC read
-  }
-  ink_assert((VIO *)d == v);
-
-  switch (event) {
-  case VC_EVENT_READ_READY: {
-    // Disable read processing
-    v->nbytes = v->ndone;
-    // fall through
-  }
-  case VC_EVENT_READ_COMPLETE: {
-    bytes_xfered = v->ndone - last_ndone;
-    if (bytes_xfered) {
-      total_bytes_xfered += bytes_xfered;
-      did += bytes_xfered;
-      to_do -= bytes_xfered;
-    }
-    last_ndone        = v->ndone;
-    io_complete_event = event;
-    INK_WRITE_MEMORY_BARRIER;
-
-    io_complete = 1;
-    IOComplete();
-
-    break;
-  }
-  case VC_EVENT_EOS:
-  case VC_EVENT_ERROR:
-  case VC_EVENT_INACTIVITY_TIMEOUT:
-  case VC_EVENT_ACTIVE_TIMEOUT:
-  default: {
-    io_complete_event = event;
-    INK_WRITE_MEMORY_BARRIER;
-
-    io_complete = -1;
-    IOComplete();
-    break;
-  }
-  } // End of switch
-
-  return EVENT_DONE;
-}
-
-int
-ClusterState::doIO_write_event(int event, void *d)
-{
-  ink_release_assert(!io_complete);
-  if (!v) {
-    v = (VIO *)d; // Immediate callback on first NetVC write
-  }
-  ink_assert((VIO *)d == v);
-
-  switch (event) {
-  case VC_EVENT_WRITE_READY:
-#ifdef CLUSTER_IMMEDIATE_NETIO
-  {
-    // Disable write processing
-    v->nbytes = v->ndone;
-    // fall through
-  }
-#endif
-  case VC_EVENT_WRITE_COMPLETE: {
-    bytes_xfered = v->ndone - last_ndone;
-    if (bytes_xfered) {
-      total_bytes_xfered += bytes_xfered;
-      did += bytes_xfered;
-      to_do -= bytes_xfered;
-    }
-    last_ndone = v->ndone;
-#ifdef CLUSTER_IMMEDIATE_NETIO
-    io_complete_event = event;
-    INK_WRITE_MEMORY_BARRIER;
-
-    io_complete = 1;
-    IOComplete();
-#else
-    if (event == VC_EVENT_WRITE_COMPLETE) {
-      io_complete_event = event;
-      INK_WRITE_MEMORY_BARRIER;
-
-      io_complete = 1;
-      IOComplete();
-    } else {
-      if (bytes_xfered) {
-        v->reenable_re(); // Immediate action
-      } else {
-        v->reenable();
-      }
-      return EVENT_DONE;
-    }
-#endif
-    break;
-  }
-  case VC_EVENT_EOS:
-  case VC_EVENT_ERROR:
-  case VC_EVENT_INACTIVITY_TIMEOUT:
-  case VC_EVENT_ACTIVE_TIMEOUT:
-  default: {
-    io_complete_event = event;
-    INK_WRITE_MEMORY_BARRIER;
-
-    io_complete = -1;
-    IOComplete();
-    break;
-  }
-  } // End of switch
-
-  return EVENT_DONE;
-}
-
-void
-ClusterState::IOComplete()
-{
-  // If no thread appears (approximate check) to be holding
-  // the ClusterHandler mutex (no cluster processing in progress)
-  // and immediate i/o completion events are allowed,
-  // start i/o completion processing.
-
-  if (do_iodone_event && !ch->mutex->thread_holding) {
-    MUTEX_TRY_LOCK(lock, ch->mutex, this_ethread());
-    if (lock.is_locked()) {
-      ch->handleEvent(EVENT_IMMEDIATE, (void *)nullptr);
-    } else {
-      eventProcessor.schedule_imm_signal(ch, ET_CLUSTER);
-    }
-  }
-}
-
-int
-ClusterHandler::cluster_signal_and_update(int event, ClusterVConnection *vc, ClusterVConnState *s)
-{
-  s->vio._cont->handleEvent(event, &s->vio);
-
-  if (vc->closed) {
-    if (!vc->write_list && !vc->write_bytes_in_transit) {
-      close_ClusterVConnection(vc);
-    }
-    return EVENT_DONE;
-  } else {
-    ink_assert((event != VC_EVENT_ERROR) || ((event == VC_EVENT_ERROR) && vc->closed));
-    return EVENT_CONT;
-  }
-}
-
-int
-ClusterHandler::cluster_signal_and_update_locked(int event, ClusterVConnection *vc, ClusterVConnState *s)
-{
-  // should assert we have s->vio.mutex
-  s->vio._cont->handleEvent(event, &s->vio);
-
-  if (vc->closed) {
-    if (!vc->write_list && !vc->write_bytes_in_transit) {
-      close_free_lock(vc, s);
-    }
-    return EVENT_DONE;
-  } else
-    return EVENT_CONT;
-}
-
-int
-ClusterHandler::cluster_signal_error_and_update(ClusterVConnection *vc, ClusterVConnState *s, int lerrno)
-{
-  s->enabled = 0;
-  vc->lerrno = lerrno;
-  return cluster_signal_and_update(VC_EVENT_ERROR, vc, s);
-}
-
-bool
-ClusterHandler::check_channel(int c)
-{
-  //
-  // Check to see that there is enough room to store channel c
-  //
-  while (n_channels <= c) {
-    int old_channels = n_channels;
-    if (!n_channels) {
-      n_channels = MIN_CHANNELS;
-    } else {
-      if ((n_channels * 2) <= MAX_CHANNELS) {
-        n_channels = n_channels * 2;
-      } else {
-        return false; // Limit exceeded
-      }
-    }
-    // Allocate ClusterVConnection table entries
-    channels = (ClusterVConnection **)ats_realloc(channels, n_channels * sizeof(ClusterVConnection *));
-
-    // Allocate ChannelData table entries
-    channel_data = (struct ChannelData **)ats_realloc(channel_data, n_channels * sizeof(struct ChannelData *));
-
-    for (int i = old_channels; i < n_channels; i++) {
-      if (local_channel(i)) {
-        if (i > LAST_DEDICATED_CHANNEL) {
-          channels[i]     = (ClusterVConnection *)1; // mark as invalid
-          channel_data[i] = (struct ChannelData *)ats_malloc(sizeof(struct ChannelData));
-          memset(channel_data[i], 0, sizeof(struct ChannelData));
-          channel_data[i]->channel_number = i;
-          free_local_channels.enqueue(channel_data[i]);
-        } else {
-          channels[i]     = nullptr;
-          channel_data[i] = nullptr;
-        }
-      } else {
-        channels[i]     = nullptr;
-        channel_data[i] = nullptr;
-      }
-    }
-  }
-  return true; // OK
-}
-
-int
-ClusterHandler::alloc_channel(ClusterVConnection *vc, int requested)
-{
-  //
-  // Allocate a channel
-  //
-  struct ChannelData *cdp = nullptr;
-  int i                   = requested;
-
-  if (!i) {
-    int loops = 1;
-    do {
-      cdp = free_local_channels.dequeue();
-      if (!cdp) {
-        if (!check_channel(n_channels)) {
-          return -2; // Limit exceeded
-        }
-      } else {
-        ink_assert(cdp == channel_data[cdp->channel_number]);
-        i = cdp->channel_number;
-        break;
-      }
-    } while (loops--);
-
-    ink_release_assert(i != 0);                                 // required
-    ink_release_assert(channels[i] == (ClusterVConnection *)1); // required
-    Debug(CL_TRACE, "alloc_channel local chan=%d VC=%p", i, vc);
-
-  } else {
-    if (!check_channel(i)) {
-      return -2; // Limit exceeded
-    }
-    if (channels[i]) {
-      Debug(CL_TRACE, "alloc_channel remote inuse chan=%d VC=%p", i, vc);
-      return -1; // channel in use
-    } else {
-      Debug(CL_TRACE, "alloc_channel remote chan=%d VC=%p", i, vc);
-    }
-  }
-  channels[i] = vc;
-  vc->channel = i;
-  return i;
-}
-
-void
-ClusterHandler::free_channel(ClusterVConnection *vc)
-{
-  //
-  // Free a channel
-  //
-  int i = vc->channel;
-  if (i > LAST_DEDICATED_CHANNEL && channels[i] == vc) {
-    if (local_channel(i)) {
-      channels[i] = (ClusterVConnection *)1;
-      free_local_channels.enqueue(channel_data[i]);
-      Debug(CL_TRACE, "free_channel local chan=%d VC=%p", i, vc);
-    } else {
-      channels[i] = nullptr;
-      Debug(CL_TRACE, "free_channel remote chan=%d VC=%p", i, vc);
-    }
-  }
-  vc->channel = 0;
-}
-
-int
-ClusterHandler::machine_down()
-{
-  char textbuf[sizeof("255.255.255.255:65535")];
-
-  if (dead) {
-    return EVENT_DONE;
-  }
-//
-// Looks like this machine dropped out of the cluster.
-// Deal with it.
-// Fatal read/write errors on the node to node connection along
-// with failure of the cluster membership check in the periodic event
-// result in machine_down().
-//
-#ifdef LOCAL_CLUSTER_TEST_MODE
-  Note("machine down %u.%u.%u.%u:%d", DOT_SEPARATED(ip), port);
-#else
-  Note("machine down %u.%u.%u.%u:%d", DOT_SEPARATED(ip), id);
-#endif
-  machine_offline_APIcallout(ip);
-  snprintf(textbuf, sizeof(textbuf), "%hhu.%hhu.%hhu.%hhu:%d", DOT_SEPARATED(ip), port);
-  RecSignalManager(REC_SIGNAL_MACHINE_DOWN, textbuf);
-  if (net_vc) {
-    net_vc->do_io(VIO::CLOSE);
-    net_vc = nullptr;
-  }
-  // Cancel pending cluster reads and writes
-  read.io_complete  = -1;
-  write.io_complete = -1;
-
-  MUTEX_TAKE_LOCK(the_cluster_config_mutex, this_ethread());
-  ClusterConfiguration *c      = this_cluster()->current_configuration();
-  machine->clusterHandlers[id] = nullptr;
-  if ((--machine->now_connections == 0) && c->find(ip, port)) {
-    ClusterConfiguration *cc = configuration_remove_machine(c, machine);
-    CLUSTER_DECREMENT_DYN_STAT(CLUSTER_NODES_STAT);
-    this_cluster()->configurations.push(cc);
-    machine->dead = true;
-  }
-  MUTEX_UNTAKE_LOCK(the_cluster_config_mutex, this_ethread());
-  MachineList *cc = the_cluster_config();
-  if (cc && cc->find(ip, port) && connector) {
-    Debug(CL_NOTE, "cluster connect retry for %hhu.%hhu.%hhu.%hhu", DOT_SEPARATED(ip));
-    clusterProcessor.connect(ip, port, id);
-  }
-  return zombify(); // defer deletion of *this
-}
-
-int
-ClusterHandler::zombify(Event * /* e ATS_UNUSED */)
-{
-  //
-  // Node associated with *this is declared down, setup the event to cleanup
-  // and defer deletion of *this
-  //
-  dead = true;
-  if (cluster_periodic_event) {
-    cluster_periodic_event->cancel(this);
-    cluster_periodic_event = nullptr;
-  }
-  clm->cancel_monitor();
-
-  SET_HANDLER((ClusterContHandler)&ClusterHandler::protoZombieEvent);
-  //
-  // At this point, allow the caller (either process_read/write to complete)
-  // prior to performing node down actions.
-  //
-  eventProcessor.schedule_in(this, HRTIME_SECONDS(1), ET_CLUSTER);
-  return EVENT_DONE;
-}
-
-int
-ClusterHandler::connectClusterEvent(int event, Event *e)
-{
-  if ((event == EVENT_IMMEDIATE) || (event == EVENT_INTERVAL)) {
-    //
-    // Attempt connect to target node and if successful, setup the event
-    // to initiate the node to node connection protocol.
-    // Initiated via ClusterProcessor::connect().
-    //
-    MachineList *cc = the_cluster_config();
-    if (!machine)
-      machine = new ClusterMachine(hostname, ip, port);
-#ifdef LOCAL_CLUSTER_TEST_MODE
-    if (!(cc && cc->find(ip, port))) {
-#else
-    if (this_cluster_machine()->ip == machine->ip || !(cc && cc->find(ip, port))) {
-#endif
-      if (this_cluster_machine()->ip != machine->ip)
-        Debug(CL_NOTE, "cluster connect aborted, machine %u.%u.%u.%u not in cluster", DOT_SEPARATED(machine->ip));
-      delete machine;
-      machine = nullptr;
-      delete this;
-      return EVENT_DONE;
-    }
-    // Connect to cluster member
-    Debug(CL_NOTE, "connect_re from %u.%u.%u.%u to %u.%u.%u.%u", DOT_SEPARATED(this_cluster_machine()->ip),
-          DOT_SEPARATED(machine->ip));
-    ip = machine->ip;
-
-    NetVCOptions opt;
-    opt.socket_send_bufsize = cluster_send_buffer_size;
-    opt.socket_recv_bufsize = cluster_receive_buffer_size;
-    opt.sockopt_flags       = cluster_sockopt_flags;
-    opt.packet_mark         = cluster_packet_mark;
-    opt.packet_tos          = cluster_packet_tos;
-    opt.etype               = ET_CLUSTER;
-    opt.addr_binding        = NetVCOptions::INTF_ADDR;
-    opt.local_ip            = this_cluster_machine()->ip;
-
-    struct sockaddr_in addr;
-    ats_ip4_set(&addr, machine->ip, htons(machine->cluster_port ? machine->cluster_port : cluster_port));
-
-    // TODO: Should we check the Action* returned here?
-    netProcessor.connect_re(this, ats_ip_sa_cast(&addr), &opt);
-    return EVENT_DONE;
-  } else {
-    if (event == NET_EVENT_OPEN) {
-      net_vc = (NetVConnection *)e;
-      SET_HANDLER((ClusterContHandler)&ClusterHandler::startClusterEvent);
-      eventProcessor.schedule_imm(this, ET_CLUSTER);
-      return EVENT_DONE;
-
-    } else {
-      eventProcessor.schedule_in(this, CLUSTER_MEMBER_DELAY);
-      return EVENT_CONT;
-    }
-  }
-}
-
-int
-ClusterHandler::startClusterEvent(int event, Event *e)
-{
-  char textbuf[sizeof("255.255.255.255:65535")];
-
-  // Perform the node to node connection establish protocol.
-
-  (void)event;
-  ink_assert(!read_vcs);
-  ink_assert(!write_vcs);
-
-  if (event == EVENT_IMMEDIATE) {
-    if (cluster_connect_state == ClusterHandler::CLCON_INITIAL) {
-      cluster_connect_state = ClusterHandler::CLCON_SEND_MSG;
-    } else {
-      ink_release_assert(!"startClusterEvent, EVENT_IMMEDIATE not expected");
-    }
-  } else {
-    ink_release_assert(event == EVENT_INTERVAL);
-  }
-
-  for (;;) {
-    switch (cluster_connect_state) {
-    ////////////////////////////////////////////////////////////////////////////
-    case ClusterHandler::CLCON_INITIAL:
-      ////////////////////////////////////////////////////////////////////////////
-      {
-        ink_release_assert(!"Invalid state [CLCON_INITIAL]");
-      }
-    ////////////////////////////////////////////////////////////////////////////
-    case ClusterHandler::CLCON_SEND_MSG:
-      ////////////////////////////////////////////////////////////////////////////
-      {
-// Send initial message.
-#ifdef LOCAL_CLUSTER_TEST_MODE
-        nodeClusteringVersion._port = cluster_port;
-#endif
-        cluster_connect_state = ClusterHandler::CLCON_SEND_MSG_COMPLETE;
-        if (connector)
-          nodeClusteringVersion._id = id;
-        build_data_vector((char *)&nodeClusteringVersion, sizeof(nodeClusteringVersion), false);
-        if (!write.doIO()) {
-          // i/o not initiated, delay and retry
-          cluster_connect_state = ClusterHandler::CLCON_SEND_MSG;
-          eventProcessor.schedule_in(this, CLUSTER_PERIOD, ET_CLUSTER);
-          return EVENT_DONE;
-        }
-        break;
-      }
-    ////////////////////////////////////////////////////////////////////////////
-    case ClusterHandler::CLCON_SEND_MSG_COMPLETE:
-      ////////////////////////////////////////////////////////////////////////////
-      {
-        if (write.io_complete) {
-          if ((write.io_complete < 0) || ((size_t)write.did < sizeof(nodeClusteringVersion))) {
-            Debug(CL_NOTE, "unable to write to cluster node %u.%u.%u.%u: %d", DOT_SEPARATED(ip), write.io_complete_event);
-            cluster_connect_state = ClusterHandler::CLCON_ABORT_CONNECT;
-            break; // goto next state
-          }
-          // Write OK, await message from peer node.
-          build_data_vector((char *)&clusteringVersion, sizeof(clusteringVersion), true);
-          cluster_connect_state = ClusterHandler::CLCON_READ_MSG;
-          break;
-        } else {
-          // Delay and check for i/o completion
-          eventProcessor.schedule_in(this, CLUSTER_PERIOD, ET_CLUSTER);
-          return EVENT_DONE;
-        }
-      }
-    ////////////////////////////////////////////////////////////////////////////
-    case ClusterHandler::CLCON_READ_MSG:
-      ////////////////////////////////////////////////////////////////////////////
-      {
-        cluster_connect_state = ClusterHandler::CLCON_READ_MSG_COMPLETE;
-        if (!read.doIO()) {
-          // i/o not initiated, delay and retry
-          cluster_connect_state = ClusterHandler::CLCON_READ_MSG;
-          eventProcessor.schedule_in(this, CLUSTER_PERIOD, ET_CLUSTER);
-          return EVENT_DONE;
-        }
-        break;
-      }
-    ////////////////////////////////////////////////////////////////////////////
-    case ClusterHandler::CLCON_READ_MSG_COMPLETE:
-      ////////////////////////////////////////////////////////////////////////////
-      {
-        if (read.io_complete) {
-          if (read.io_complete < 0) {
-            // Read error, abort connect
-            cluster_connect_state = ClusterHandler::CLCON_ABORT_CONNECT;
-            break; // goto next state
-          }
-          if ((size_t)read.did < sizeof(clusteringVersion)) {
-            // Partial read, resume read.
-            cluster_connect_state = ClusterHandler::CLCON_READ_MSG;
-            break;
-          }
-          cluster_connect_state = ClusterHandler::CLCON_VALIDATE_MSG;
-          break;
-        } else {
-          // Delay and check for i/o completion
-          eventProcessor.schedule_in(this, CLUSTER_PERIOD, ET_CLUSTER);
-          return EVENT_DONE;
-        }
-      }
-    ////////////////////////////////////////////////////////////////////////////
-    case ClusterHandler::CLCON_VALIDATE_MSG:
-      ////////////////////////////////////////////////////////////////////////////
-      {
-        int proto_major = -1;
-        int proto_minor = -1;
-
-        clusteringVersion.AdjustByteOrder();
-        /////////////////////////////////////////////////////////////////////////
-        // Determine the message protocol major version to use, by stepping down
-        // from current to the minimium level until a match is found.
-        // Derive the minor number as follows, if the current (major, minor)
-        // is the current node (major, minor) use the given minor number.
-        // Otherwise, minor number is zero.
-        /////////////////////////////////////////////////////////////////////////
-        for (int major = clusteringVersion._major; major >= clusteringVersion._min_major; --major) {
-          if ((major >= nodeClusteringVersion._min_major) && (major <= nodeClusteringVersion._major)) {
-            proto_major = major;
-          }
-        }
-        if (proto_major > 0) {
-          ///////////////////////////
-          // Compute minor version
-          ///////////////////////////
-          if (proto_major == clusteringVersion._major) {
-            proto_minor = clusteringVersion._minor;
-
-            if (proto_minor != nodeClusteringVersion._minor)
-              Warning("Different clustering minor versions (%d,%d) for node %u.%u.%u.%u, continuing", proto_minor,
-                      nodeClusteringVersion._minor, DOT_SEPARATED(ip));
-          } else {
-            proto_minor = 0;
-          }
-
-        } else {
-          Warning("Bad cluster major version range (%d-%d) for node %u.%u.%u.%u connect failed", clusteringVersion._min_major,
-                  clusteringVersion._major, DOT_SEPARATED(ip));
-          cluster_connect_state = ClusterHandler::CLCON_ABORT_CONNECT;
-          break; // goto next state
-        }
-
-#ifdef LOCAL_CLUSTER_TEST_MODE
-        port = clusteringVersion._port & 0xffff;
-#endif
-        if (!connector)
-          id = clusteringVersion._id & 0xffff;
-
-        machine->msg_proto_major = proto_major;
-        machine->msg_proto_minor = proto_minor;
-
-        if (eventProcessor.n_threads_for_type[ET_CLUSTER] != num_of_cluster_threads) {
-          cluster_connect_state = ClusterHandler::CLCON_ABORT_CONNECT;
-          break;
-        }
-
-        thread = eventProcessor.eventthread[ET_CLUSTER][id % num_of_cluster_threads];
-        if (net_vc->thread == thread) {
-          cluster_connect_state = CLCON_CONN_BIND_OK;
-          break;
-        } else {
-          cluster_connect_state = ClusterHandler::CLCON_CONN_BIND_CLEAR;
-        }
-      }
-
-    case ClusterHandler::CLCON_CONN_BIND_CLEAR: {
-      UnixNetVConnection *vc = (UnixNetVConnection *)net_vc;
-      MUTEX_TRY_LOCK(lock, vc->nh->mutex, e->ethread);
-      MUTEX_TRY_LOCK(lock1, vc->mutex, e->ethread);
-      if (lock.is_locked() && lock1.is_locked()) {
-        vc->ep.stop();
-        vc->nh->open_list.remove(vc);
-        vc->nh->cop_list.remove(vc);
-        vc->thread = nullptr;
-        if (vc->nh->read_ready_list.in(vc))
-          vc->nh->read_ready_list.remove(vc);
-        if (vc->nh->write_ready_list.in(vc))
-          vc->nh->write_ready_list.remove(vc);
-        if (vc->read.in_enabled_list)
-          vc->nh->read_enable_list.remove(vc);
-        if (vc->write.in_enabled_list)
-          vc->nh->write_enable_list.remove(vc);
-
-        // CLCON_CONN_BIND handle in bind vc->thread (bind thread nh)
-        cluster_connect_state = ClusterHandler::CLCON_CONN_BIND;
-        thread->schedule_in(this, CLUSTER_PERIOD);
-        return EVENT_DONE;
-      } else {
-        // CLCON_CONN_BIND_CLEAR handle in origin vc->thread (origin thread nh)
-        vc->thread->schedule_in(this, CLUSTER_PERIOD);
-        return EVENT_DONE;
-      }
-    }
-
-    case ClusterHandler::CLCON_CONN_BIND: {
-      //
-      NetHandler *nh         = get_NetHandler(e->ethread);
-      UnixNetVConnection *vc = (UnixNetVConnection *)net_vc;
-      MUTEX_TRY_LOCK(lock, nh->mutex, e->ethread);
-      MUTEX_TRY_LOCK(lock1, vc->mutex, e->ethread);
-      if (lock.is_locked() && lock1.is_locked()) {
-        if (vc->read.in_enabled_list)
-          nh->read_enable_list.push(vc);
-        if (vc->write.in_enabled_list)
-          nh->write_enable_list.push(vc);
-
-        vc->nh             = nh;
-        vc->thread         = e->ethread;
-        PollDescriptor *pd = get_PollDescriptor(e->ethread);
-        if (vc->ep.start(pd, vc, EVENTIO_READ | EVENTIO_WRITE) < 0) {
-          cluster_connect_state = ClusterHandler::CLCON_DELETE_CONNECT;
-          break; // goto next state
-        }
-
-        nh->open_list.enqueue(vc);
-        cluster_connect_state = ClusterHandler::CLCON_CONN_BIND_OK;
-      } else {
-        thread->schedule_in(this, CLUSTER_PERIOD);
-        return EVENT_DONE;
-      }
-    }
-
-    case ClusterHandler::CLCON_CONN_BIND_OK: {
-      int failed = 0;
-
-      // include this node into the cluster configuration
-      MUTEX_TAKE_LOCK(the_cluster_config_mutex, this_ethread());
-      MachineList *cc = the_cluster_config();
-      if (cc && cc->find(ip, port)) {
-        ClusterConfiguration *c = this_cluster()->current_configuration();
-        ClusterMachine *m       = c->find(ip, port);
-
-        if (!m) { // this first connection
-          ClusterConfiguration *cconf = configuration_add_machine(c, machine);
-          CLUSTER_INCREMENT_DYN_STAT(CLUSTER_NODES_STAT);
-          this_cluster()->configurations.push(cconf);
-        } else {
-          // close new connection if old connections is exist
-          if (id >= m->num_connections || m->clusterHandlers[id]) {
-            failed = -2;
-            MUTEX_UNTAKE_LOCK(the_cluster_config_mutex, this_ethread());
-            goto failed;
-          }
-          machine = m;
-        }
-        machine->now_connections++;
-        machine->clusterHandlers[id] = this;
-        machine->dead                = false;
-        dead                         = false;
-      } else {
-        Debug(CL_NOTE, "cluster connect aborted, machine %u.%u.%u.%u:%d not in cluster", DOT_SEPARATED(ip), port);
-        failed = -1;
-      }
-      MUTEX_UNTAKE_LOCK(the_cluster_config_mutex, this_ethread());
-    failed:
-      if (failed) {
-        if (failed == -1) {
-          if (++configLookupFails <= CONFIG_LOOKUP_RETRIES) {
-            thread->schedule_in(this, CLUSTER_PERIOD);
-            return EVENT_DONE;
-          }
-        }
-        cluster_connect_state = ClusterHandler::CLCON_DELETE_CONNECT;
-        break; // goto next state
-      }
-
-      this->needByteSwap = !clusteringVersion.NativeByteOrder();
-      machine_online_APIcallout(ip);
-
-      // Signal the manager
-      snprintf(textbuf, sizeof(textbuf), "%hhu.%hhu.%hhu.%hhu:%d", DOT_SEPARATED(ip), port);
-      RecSignalManager(REC_SIGNAL_MACHINE_UP, textbuf);
-#ifdef LOCAL_CLUSTER_TEST_MODE
-      Note("machine up %hhu.%hhu.%hhu.%hhu:%d, protocol version=%d.%d", DOT_SEPARATED(ip), port, clusteringVersion._major,
-           clusteringVersion._minor);
-#else
-      Note("machine up %hhu.%hhu.%hhu.%hhu:%d, protocol version=%d.%d", DOT_SEPARATED(ip), id, clusteringVersion._major,
-           clusteringVersion._minor);
-#endif
-
-      read_vcs  = new Queue<ClusterVConnectionBase, ClusterVConnectionBase::Link_read_link>[CLUSTER_BUCKETS];
-      write_vcs = new Queue<ClusterVConnectionBase, ClusterVConnectionBase::Link_write_link>[CLUSTER_BUCKETS];
-      SET_HANDLER((ClusterContHandler)&ClusterHandler::beginClusterEvent);
-
-      // enable schedule_imm() on i/o completion (optimization)
-      read.do_iodone_event  = true;
-      write.do_iodone_event = true;
-
-      cluster_periodic_event = thread->schedule_every(this, -CLUSTER_PERIOD);
-
-      // Startup the periodic events to process entries in
-      //  external_incoming_control.
-
-      int procs_online    = ink_number_of_processors();
-      int total_callbacks = min(procs_online, MAX_COMPLETION_CALLBACK_EVENTS);
-      for (int n = 0; n < total_callbacks; ++n) {
-        callout_cont[n]   = new ClusterCalloutContinuation(this);
-        callout_events[n] = eventProcessor.schedule_every(callout_cont[n], COMPLETION_CALLBACK_PERIOD, ET_NET);
-      }
-
-      // Start cluster interconnect load monitoring
-
-      if (!clm) {
-        clm = new ClusterLoadMonitor(this);
-        clm->init();
-      }
-      return EVENT_DONE;
-    }
-    ////////////////////////////////////////////////////////////////////////////
-    case ClusterHandler::CLCON_ABORT_CONNECT:
-      ////////////////////////////////////////////////////////////////////////////
-      {
-        if (connector) {
-          Debug(CL_NOTE, "cluster connect retry for %u.%u.%u.%u", DOT_SEPARATED(ip));
-          // check for duplicate cluster connect
-          clusterProcessor.connect(ip, port, id, true);
-        }
-        cluster_connect_state = ClusterHandler::CLCON_DELETE_CONNECT;
-        break; // goto next state
-      }
-    ////////////////////////////////////////////////////////////////////////////
-    case ClusterHandler::CLCON_DELETE_CONNECT:
-      ////////////////////////////////////////////////////////////////////////////
-      {
-        // No references possible, so just delete it.
-        delete machine;
-        machine = nullptr;
-        delete this;
-        Debug(CL_NOTE, "Failed cluster connect, deleting");
-        return EVENT_DONE;
-      }
-    ////////////////////////////////////////////////////////////////////////////
-    default:
-      ////////////////////////////////////////////////////////////////////////////
-      {
-        Warning("startClusterEvent invalid state %d", cluster_connect_state);
-        ink_release_assert(!"ClusterHandler::startClusterEvent invalid state");
-        return EVENT_DONE;
-      }
-
-    } // End of switch
-  }   // End of for
-  return EVENT_DONE;
-}
-
-int
-ClusterHandler::beginClusterEvent(int /* event ATS_UNUSED */, Event *e)
-{
-// Establish the main periodic Cluster event
-#ifdef CLUSTER_IMMEDIATE_NETIO
-  build_poll(false);
-#endif
-  SET_HANDLER((ClusterContHandler)&ClusterHandler::mainClusterEvent);
-  return handleEvent(EVENT_INTERVAL, e);
-}
-
-int
-ClusterHandler::zombieClusterEvent(int event, Event *e)
-{
-  //
-  // The ZOMBIE state is entered when the handler may still be referenced
-  // by short running tasks (one scheduling quanta).  The object is delayed
-  // after some unreasonably long (in comparison) time.
-  //
-  (void)event;
-  (void)e;
-  delete this; // I am out of here
-  return EVENT_DONE;
-}
-
-int
-ClusterHandler::protoZombieEvent(int /* event ATS_UNUSED */, Event *e)
-{
-  //
-  // Node associated with *this is declared down.
-  // After cleanup is complete, setup handler to delete *this
-  // after NO_RACE_DELAY
-  //
-  bool failed      = false;
-  ink_hrtime delay = CLUSTER_MEMBER_DELAY * 5;
-  EThread *t       = e ? e->ethread : this_ethread();
-  head_p item;
-
-  /////////////////////////////////////////////////////////////////
-  // Complete pending i/o operations
-  /////////////////////////////////////////////////////////////////
-  mainClusterEvent(EVENT_INTERVAL, e);
-
-  item.data = external_incoming_open_local.head.data;
-  if (TO_PTR(FREELIST_POINTER(item)) || delayed_reads.head || pw_write_descriptors_built || pw_freespace_descriptors_built ||
-      pw_controldata_descriptors_built) {
-    // Operations still pending, retry later
-    if (e) {
-      e->schedule_in(delay);
-      return EVENT_CONT;
-    } else {
-      eventProcessor.schedule_in(this, delay, ET_CLUSTER);
-      return EVENT_DONE;
-    }
-  }
-  ///////////////////////////////////////////////////////////////
-  // Deallocate current read control data
-  ///////////////////////////////////////////////////////////////
-  IncomingControl *ic;
-  while ((ic = incoming_control.dequeue())) {
-    failed    = true;
-    ic->mutex = nullptr;
-    ic->freeall();
-  }
-
-  /////////////////////////////////////////////////////////////////
-  // Post error completion on all active read/write VC(s) and
-  // deallocate closed VC(s).
-  /////////////////////////////////////////////////////////////////
-  for (int i = 0; i < n_channels; i++) {
-    ClusterVConnection *vc = channels[i];
-    if (VALID_CHANNEL(vc)) {
-      if (!vc->closed && vc->read.vio.op == VIO::READ) {
-        MUTEX_TRY_LOCK(lock, vc->read.vio.mutex, t);
-        if (lock.is_locked()) {
-          cluster_signal_error_and_update(vc, &vc->read, 0);
-        } else {
-          failed = true;
-        }
-      }
-      vc = channels[i];
-      if (VALID_CHANNEL(vc) && !vc->closed && vc->write.vio.op == VIO::WRITE) {
-        MUTEX_TRY_LOCK(lock, vc->write.vio.mutex, t);
-        if (lock.is_locked()) {
-          cluster_signal_error_and_update(vc, &vc->write, 0);
-        } else {
-          failed = true;
-        }
-      }
-      vc = channels[i];
-      if (VALID_CHANNEL(vc)) {
-        if (vc->closed) {
-          vc->ch                     = nullptr;
-          vc->write_list             = nullptr;
-          vc->write_list_tail        = nullptr;
-          vc->write_list_bytes       = 0;
-          vc->write_bytes_in_transit = 0;
-          close_ClusterVConnection(vc);
-        } else {
-          failed = true;
-        }
-      }
-    }
-  }
-
-  ///////////////////////////////////////////////////////////////
-  // Empty the external_incoming_control queue before aborting
-  //   the completion callbacks.
-  ///////////////////////////////////////////////////////////////
-  item.data = external_incoming_control.head.data;
-  if (TO_PTR(FREELIST_POINTER(item)) == nullptr) {
-    for (int n = 0; n < MAX_COMPLETION_CALLBACK_EVENTS; ++n) {
-      if (callout_cont[n]) {
-        MUTEX_TRY_LOCK(lock, callout_cont[n]->mutex, t);
-        if (lock.is_locked()) {
-          callout_events[n]->cancel(callout_cont[n]);
-          callout_events[n] = nullptr;
-          delete callout_cont[n];
-          callout_cont[n] = nullptr;
-        } else {
-          failed = true;
-        }
-      }
-    }
-  } else {
-    failed = true;
-  }
-
-  if (!failed) {
-    Debug("cluster_down", "ClusterHandler zombie [%u.%u.%u.%u]", DOT_SEPARATED(ip));
-    SET_HANDLER((ClusterContHandler)&ClusterHandler::zombieClusterEvent);
-    delay = NO_RACE_DELAY;
-  }
-  if (e) {
-    e->schedule_in(delay);
-    return EVENT_CONT;
-  } else {
-    eventProcessor.schedule_in(this, delay, ET_CLUSTER);
-    return EVENT_DONE;
-  }
-}
-
-int dump_verbose = 0;
-
-int
-ClusterHandler::compute_active_channels()
-{
-  ClusterHandler *ch = this;
-  int active_chans   = 0;
-
-  for (int i = LAST_DEDICATED_CHANNEL + 1; i < ch->n_channels; i++) {
-    ClusterVConnection *vc = ch->channels[i];
-    if (VALID_CHANNEL(vc) && (vc->iov_map != CLUSTER_IOV_NOT_OPEN)) {
-      ++active_chans;
-      if (dump_verbose) {
-        printf("ch[%d] vc=0x%p remote_free=%d last_local_free=%d\n", i, vc, vc->remote_free, vc->last_local_free);
-        printf("  r_bytes=%d r_done=%d w_bytes=%d w_done=%d\n", (int)vc->read.vio.nbytes, (int)vc->read.vio.ndone,
-               (int)vc->write.vio.nbytes, (int)vc->write.vio.ndone);
-      }
-    }
-  }
-  return active_chans;
-}
-
-void
-ClusterHandler::dump_internal_data()
-{
-  if (!message_blk) {
-    message_blk = new_IOBufferBlock();
-    message_blk->alloc(MAX_IOBUFFER_SIZE);
-  }
-  int r;
-  int n               = 0;
-  char *b             = message_blk->data->data();
-  unsigned int b_size = message_blk->data->block_size();
-
-  r = snprintf(&b[n], b_size - n, "Host: %hhu.%hhu.%hhu.%hhu\n", DOT_SEPARATED(ip));
-  n += r;
-
-  r =
-    snprintf(&b[n], b_size - n, "chans: %d vc_writes: %" PRId64 " write_bytes: %" PRId64 "(d)+%" PRId64 "(c)=%" PRId64 "\n",
-             compute_active_channels(), _vc_writes, _vc_write_bytes, _control_write_bytes, _vc_write_bytes + _control_write_bytes);
-
-  n += r;
-  r = snprintf(&b[n], b_size - n, "dw: missed_lock: %d not_enabled: %d wait_remote_fill: %d no_active_vio: %d\n", _dw_missed_lock,
-               _dw_not_enabled, _dw_wait_remote_fill, _dw_no_active_vio);
-
-  n += r;
-  r = snprintf(&b[n], b_size - n, "dw: not_enabled_or_no_write: %d set_data_pending: %d no_free_space: %d\n",
-               _dw_not_enabled_or_no_write, _dw_set_data_pending, _dw_no_free_space);
-
-  n += r;
-  r = snprintf(&b[n], b_size - n, "fw: missed_lock: %d not_enabled: %d wait_remote_fill: %d no_active_vio: %d\n", _fw_missed_lock,
-               _fw_not_enabled, _fw_wait_remote_fill, _fw_no_active_vio);
-
-  n += r;
-  r = snprintf(&b[n], b_size - n, "fw: not_enabled_or_no_read: %d\n", _fw_not_enabled_or_no_read);
-
-  n += r;
-  r = snprintf(&b[n], b_size - n, "rd(%d): st:%d rh:%d ahd:%d sd:%d rd:%d ad:%d sda:%d rda:%d awd:%d p:%d c:%d\n",
-               _process_read_calls, _n_read_start, _n_read_header, _n_read_await_header, _n_read_setup_descriptor,
-               _n_read_descriptor, _n_read_await_descriptor, _n_read_setup_data, _n_read_data, _n_read_await_data,
-               _n_read_post_complete, _n_read_complete);
-
-  n += r;
-  r = snprintf(&b[n], b_size - n, "wr(%d): st:%d set:%d ini:%d wait:%d post:%d comp:%d\n", _process_write_calls, _n_write_start,
-               _n_write_setup, _n_write_initiate, _n_write_await_completion, _n_write_post_complete, _n_write_complete);
-
-  n += r;
-  ink_release_assert((n + 1) <= BUFFER_SIZE_FOR_INDEX(MAX_IOBUFFER_SIZE));
-  Note("%s", b);
-  clear_cluster_stats();
-}
-
-void
-ClusterHandler::dump_write_msg(int res)
-{
-  // Debug support for inter cluster message trace
-  Alias32 x;
-  x.u32 = (uint32_t)((struct sockaddr_in *)(net_vc->get_remote_addr()))->sin_addr.s_addr;
-
-  fprintf(stderr, "[W] %hhu.%hhu.%hhu.%hhu SeqNo=%u, Cnt=%d, CntlCnt=%d Todo=%d, Res=%d\n", x.byte[0], x.byte[1], x.byte[2],
-          x.byte[3], write.sequence_number, write.msg.count, write.msg.control_bytes, write.to_do, res);
-  for (int i = 0; i < write.msg.count; ++i) {
-    fprintf(stderr, "   d[%i] Type=%d, Chan=%d, SeqNo=%d, Len=%u\n", i, (write.msg.descriptor[i].type ? 1 : 0),
-            (int)write.msg.descriptor[i].channel, (int)write.msg.descriptor[i].sequence_number, write.msg.descriptor[i].length);
-  }
-}
-
-void
-ClusterHandler::dump_read_msg()
-{
-  // Debug support for inter cluster message trace
-  Alias32 x;
-  x.u32 = (uint32_t)((struct sockaddr_in *)(net_vc->get_remote_addr()))->sin_addr.s_addr;
-
-  fprintf(stderr, "[R] %hhu.%hhu.%hhu.%hhu  SeqNo=%u, Cnt=%d, CntlCnt=%d\n", x.byte[0], x.byte[1], x.byte[2], x.byte[3],
-          read.sequence_number, read.msg.count, read.msg.control_bytes);
-  for (int i = 0; i < read.msg.count; ++i) {
-    fprintf(stderr, "   d[%i] Type=%d, Chan=%d, SeqNo=%d, Len=%u\n", i, (read.msg.descriptor[i].type ? 1 : 0),
-            (int)read.msg.descriptor[i].channel, (int)read.msg.descriptor[i].sequence_number, read.msg.descriptor[i].length);
-  }
-}
-
-// End of  ClusterHandlerBase.cc
diff --git a/iocore/cluster/ClusterHash.cc b/iocore/cluster/ClusterHash.cc
deleted file mode 100644
index 0498512..0000000
--- a/iocore/cluster/ClusterHash.cc
+++ /dev/null
@@ -1,171 +0,0 @@
-/** @file
-
-  A brief file description
-
-  @section license License
-
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
- */
-
-/****************************************************************************
-
-  ClusterHash.cc
- ****************************************************************************/
-#include "P_Cluster.h"
-
-//
-// Configuration of the cluster hash function
-//
-// machineClusterHash  - whether or not the random number generators
-//                       are based on the machines or on the buckets
-// boundClusterHash    - whether or not we force a fixed number of buckets
-//                       to map to each machine
-// randClusterHash     - whether or not to use system rand(3C)
-//                       or a simple linear congruence random number
-//                       generator
-//
-// This produces very stable results (computation time ~.6 seconds) on
-// a UltraSparc at 143Mz.
-// These are only global for testing purposes.
-//
-bool machineClusterHash = true;
-bool boundClusterHash   = false;
-bool randClusterHash    = false;
-
-// This produces better speed for large numbers of machines > 18
-//
-// bool machineClusterHash = false;
-// bool boundClusterHash = true;
-// bool randClusterHash = true;
-
-//
-// Cluster Hash Table
-//
-// see Memo.ClusterHash for details
-//
-
-//
-// Linear Congruence Random number generator
-
-// Not very random, but it generates all the numbers
-// within 1 period which is all we need.
-//
-inline unsigned short
-next_rnd15(unsigned int *p)
-{
-  unsigned int seed = *p;
-  seed              = 1103515145 * seed + 12345;
-  seed              = seed & 0x7FFF;
-  *p                = seed;
-  return seed;
-}
-
-//
-// Build the hash table
-// This function is relatively expensive.
-// It costs: (g++ at -02)
-// ~.04 CPU seconds on a 143MHz Ultra 1 at 1 node
-// ~.3 CPU seconds on a 143MHz Ultra 1 at 31 nodes
-// Overall it is roughly linear in the number of nodes.
-//
-void
-build_hash_table_machine(ClusterConfiguration *c)
-{
-  int left = CLUSTER_HASH_TABLE_SIZE;
-  int m    = 0;
-  int i    = 0;
-  unsigned int rnd[CLUSTER_MAX_MACHINES];
-  unsigned int mach[CLUSTER_MAX_MACHINES];
-  int total = CLUSTER_HASH_TABLE_SIZE;
-
-  for (i = 0; i < c->n_machines; i++) {
-    int mine = total / (c->n_machines - i);
-    mach[i]  = mine;
-    total -= mine;
-  }
-
-  // seed the random number generator with the ip address
-  // do a little xor folding to get it into 15 bits
-  //
-  for (m   = 0; m < c->n_machines; m++)
-    rnd[m] = (((c->machines[m]->ip >> 15) & 0x7FFF) ^ (c->machines[m]->ip & 0x7FFF)) ^ (c->machines[m]->ip >> 30);
-
-  // Initialize the table to "empty"
-  //
-  for (i             = 0; i < CLUSTER_HASH_TABLE_SIZE; i++)
-    c->hash_table[i] = 255;
-
-  // Until we have hit every element of the table, give each
-  // machine a chance to select it's favorites.
-  //
-  m = 0;
-  while (left) {
-    if (!mach[m] && boundClusterHash) {
-      m = (m + 1) % c->n_machines;
-      continue;
-    }
-    do {
-      if (randClusterHash) {
-        i = ink_rand_r(&rnd[m]) % CLUSTER_HASH_TABLE_SIZE;
-      } else
-        i = next_rand(&rnd[m]) % CLUSTER_HASH_TABLE_SIZE;
-    } while (c->hash_table[i] != 255);
-    mach[m]--;
-    c->hash_table[i] = m;
-    left--;
-    m = (m + 1) % c->n_machines;
-  }
-}
-
-static void
-build_hash_table_bucket(ClusterConfiguration *c)
-{
-  int i = 0;
-  unsigned int rnd[CLUSTER_HASH_TABLE_SIZE];
-  unsigned int mach[CLUSTER_MAX_MACHINES];
-  int total = CLUSTER_HASH_TABLE_SIZE;
-
-  for (i = 0; i < c->n_machines; i++) {
-    int mine = total / (c->n_machines - i);
-    mach[i]  = mine;
-    total -= mine;
-  }
-
-  for (i   = 0; i < CLUSTER_HASH_TABLE_SIZE; i++)
-    rnd[i] = i;
-
-  for (i = 0; i < CLUSTER_HASH_TABLE_SIZE; i++) {
-    unsigned char x = 0;
-    do {
-      if (randClusterHash) {
-        x = ink_rand_r(&rnd[i]) % CLUSTER_MAX_MACHINES;
-      } else
-        x = next_rand(&rnd[i]) % CLUSTER_MAX_MACHINES;
-    } while (x >= c->n_machines || (!mach[x] && boundClusterHash));
-    mach[x]--;
-    c->hash_table[i] = x;
-  }
-}
-
-void
-build_cluster_hash_table(ClusterConfiguration *c)
-{
-  if (machineClusterHash)
-    build_hash_table_machine(c);
-  else
-    build_hash_table_bucket(c);
-}
diff --git a/iocore/cluster/ClusterLib.cc b/iocore/cluster/ClusterLib.cc
deleted file mode 100644
index ff87d22..0000000
--- a/iocore/cluster/ClusterLib.cc
+++ /dev/null
@@ -1,346 +0,0 @@
-/** @file
-
-  A brief file description
-
-  @section license License
-
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
- */
-
-/****************************************************************************
-
-  ClusterLib.cc
-****************************************************************************/
-
-#include "P_Cluster.h"
-//
-// cluster_xxx() functions dealing with scheduling of Virtual Connections
-// in the read and write data buckets (read_vcs, write_vcs).
-//
-// In contrast to the net versions, these versions simply change the priority
-// scheduling only occurs after they move into the data_bucket.
-//
-void
-cluster_schedule(ClusterHandler *ch, ClusterVConnection *vc, ClusterVConnState *ns)
-{
-  //
-  // actually schedule into new bucket
-  //
-  int new_bucket = ch->cur_vcs;
-
-  if (vc->type == VC_NULL)
-    vc->type = VC_CLUSTER;
-  if (ns == &vc->read) {
-    ClusterVC_enqueue_read(ch->read_vcs[new_bucket], vc);
-  } else {
-    ClusterVC_enqueue_write(ch->write_vcs[new_bucket], vc);
-  }
-}
-
-void
-cluster_reschedule_offset(ClusterHandler *ch, ClusterVConnection *vc, ClusterVConnState *ns, int offset)
-{
-  if (ns == &vc->read) {
-    if (vc->read.queue)
-      ClusterVC_remove_read(vc);
-    ClusterVC_enqueue_read(ch->read_vcs[(ch->cur_vcs + offset) % CLUSTER_BUCKETS], vc);
-  } else {
-    if (vc->write.queue)
-      ClusterVC_remove_write(vc);
-    ClusterVC_enqueue_write(ch->write_vcs[(ch->cur_vcs + offset) % CLUSTER_BUCKETS], vc);
-  }
-}
-/*************************************************************************/
-// ClusterVCToken member functions (Public Class)
-/*************************************************************************/
-
-// global sequence number for building tokens
-unsigned int cluster_sequence_number = 0;
-
-void
-ClusterVCToken::alloc()
-{
-#ifdef LOCAL_CLUSTER_TEST_MODE
-  ip_created = this_cluster_machine()->cluster_port;
-#else
-  ip_created = this_cluster_machine()->ip;
-#endif
-  sequence_number = ink_atomic_increment((int *)&cluster_sequence_number, 1);
-}
-
-///////////////////////////////////////////
-// IOBufferBlock manipulation routines
-///////////////////////////////////////////
-
-IOBufferBlock *
-clone_IOBufferBlockList(IOBufferBlock *b, int start_off, int n, IOBufferBlock **b_tail)
-{
-  ////////////////////////////////////////////////////////////////
-  // Create a clone list of IOBufferBlock(s) where the sum
-  // of all block read_avail is 'n'.  The given source list
-  // must contain at least 'n' read avail bytes.
-  ////////////////////////////////////////////////////////////////
-  int64_t nbytes = n;
-  int64_t block_read_avail;
-  int64_t bytes_to_skip      = start_off;
-  IOBufferBlock *bsrc        = b;
-  IOBufferBlock *bclone      = nullptr;
-  IOBufferBlock *bclone_head = nullptr;
-
-  while (bsrc && nbytes) {
-    // Skip zero length blocks
-    if (!bsrc->read_avail()) {
-      bsrc = bsrc->next.get();
-      continue;
-    }
-
-    if (bclone_head) {
-      bclone->next = bsrc->clone();
-      bclone       = bclone->next.get();
-    } else {
-      // Skip bytes already processed
-      if (bytes_to_skip) {
-        bytes_to_skip -= bsrc->read_avail();
-
-        if (bytes_to_skip < 0) {
-          // Skip bytes in current block
-          bclone_head = bsrc->clone();
-          bclone_head->consume(bsrc->read_avail() + bytes_to_skip);
-          bclone        = bclone_head;
-          bytes_to_skip = 0;
-
-        } else {
-          // Skip entire block
-          bsrc = bsrc->next.get();
-          continue;
-        }
-      } else {
-        bclone_head = bsrc->clone();
-        bclone      = bclone_head;
-      }
-    }
-    block_read_avail = bclone->read_avail();
-    nbytes -= block_read_avail;
-    if (nbytes < 0) {
-      // Adjust read_avail in clone to match nbytes
-      bclone->fill(nbytes);
-      nbytes = 0;
-    }
-    bsrc = bsrc->next.get();
-  }
-  ink_release_assert(!nbytes);
-  *b_tail = bclone;
-  return bclone_head;
-}
-
-IOBufferBlock *
-consume_IOBufferBlockList(IOBufferBlock *b, int64_t n)
-{
-  IOBufferBlock *b_remainder = nullptr;
-  int64_t nbytes             = n;
-
-  while (b) {
-    nbytes -= b->read_avail();
-    if (nbytes <= 0) {
-      if (nbytes < 0) {
-        // Consumed a partial block, clone remainder
-        b_remainder = b->clone();
-        b->fill(nbytes);                       // make read_avail match nbytes
-        b_remainder->consume(b->read_avail()); // clone for remaining bytes
-        b_remainder->next = b->next;
-        b->next           = nullptr;
-        nbytes            = 0;
-
-      } else {
-        // Consumed entire block
-        b_remainder = b->next.get();
-      }
-      break;
-
-    } else {
-      b = b->next.get();
-    }
-  }
-
-  ink_release_assert(nbytes == 0);
-  return b_remainder; // return remaining blocks
-}
-
-int64_t
-bytes_IOBufferBlockList(IOBufferBlock *b, int64_t read_avail_bytes)
-{
-  int64_t n = 0;
-  ;
-
-  while (b) {
-    if (read_avail_bytes) {
-      n += b->read_avail();
-    } else {
-      n += b->write_avail();
-    }
-    b = b->next.get();
-  }
-
-  return n;
-}
-
-//////////////////////////////////////////////////////
-// Miscellaneous test code
-//////////////////////////////////////////////////////
-#if TEST_PARTIAL_READS
-//
-// Test code which mimic the network slowdown
-//
-int
-partial_readv(int fd, IOVec *iov, int n_iov, int seq)
-{
-  IOVec tiov[16];
-  for (int i  = 0; i < n_iov; i++)
-    tiov[i]   = iov[i];
-  int tn_iov  = n_iov;
-  int rnd     = seq;
-  int element = rand_r((unsigned int *)&rnd);
-  element     = element % n_iov;
-  int byte    = rand_r((unsigned int *)&rnd);
-  byte        = byte % iov[element].iov_len;
-  int stop    = rand_r((unsigned int *)&rnd);
-  if (!(stop % 3)) { // 33% chance
-    tn_iov                = element + 1;
-    tiov[element].iov_len = byte;
-    if (!byte)
-      tn_iov--;
-    if (!tn_iov) {
-      tiov[element].iov_len = 1;
-      tn_iov++;
-    }
-    // printf("partitial read %d [%d]\n",tn_iov,tiov[element].iov_len);
-  }
-  return socketManager.read_vector(fd, &tiov[0], tn_iov);
-}
-#endif // TEST_PARTIAL_READS
-
-#if TEST_PARTIAL_WRITES
-//
-// Test code which mimic the network backing up (too little buffering)
-//
-int
-partial_writev(int fd, IOVec *iov, int n_iov, int seq)
-{
-  int rnd = seq;
-  int sum = 0;
-  int i   = 0;
-  for (i = 0; i < n_iov; i++) {
-    int l = iov[i].iov_len;
-    int r = rand_r((unsigned int *)&rnd);
-    if ((r >> 4) & 1) {
-      l = ((unsigned int)rand_r((unsigned int *)&rnd)) % iov[i].iov_len;
-      if (!l) {
-        l = iov[i].iov_len;
-      }
-    }
-    ink_assert(l <= iov[i].iov_len);
-    fprintf(stderr, "writing %d: [%d] &%X %d of %d\n", seq, i, iov[i].iov_base, l, iov[i].iov_len);
-    int res = socketManager.write(fd, iov[i].iov_base, l);
-    if (res < 0) {
-      return res;
-    }
-    sum += res;
-    if (res != iov[i].iov_len) {
-      return sum;
-    }
-  }
-  return sum;
-}
-#endif // TEST_PARTIAL_WRITES
-
-////////////////////////////////////////////////////////////////////////
-// Global periodic system dump functions
-////////////////////////////////////////////////////////////////////////
-#ifdef ENABLE_TIME_TRACE
-int inmsg_time_dist[TIME_DIST_BUCKETS_SIZE];
-int inmsg_events = 0;
-
-int cluster_send_time_dist[TIME_DIST_BUCKETS_SIZE];
-int cluster_send_events = 0;
-#endif // ENABLE_TIME_TRACE
-
-int time_trace = 0;
-
-void
-dump_time_buckets()
-{
-#ifdef ENABLE_TIME_TRACE
-  printf("\nremote ops:\n");
-  for (int i = 0; i < TIME_DIST_BUCKETS_SIZE; ++i) {
-    printf("%d ", rmt_callback_time_dist[i]);
-    rmt_callback_time_dist[i] = 0;
-  }
-  printf("\nremote lookup ops:\n");
-  for (int j = 0; j < TIME_DIST_BUCKETS_SIZE; ++j) {
-    printf("%d ", lkrmt_callback_time_dist[j]);
-    lkrmt_callback_time_dist[j] = 0;
-  }
-  printf("\nlocal cache ops:\n");
-  for (int k = 0; k < TIME_DIST_BUCKETS_SIZE; ++k) {
-    printf("%d ", callback_time_dist[k]);
-    callback_time_dist[k] = 0;
-  }
-  printf("\nphysical cache ops:\n");
-  for (int l = 0; l < TIME_DIST_BUCKETS_SIZE; ++l) {
-    printf("%d ", cdb_callback_time_dist[l]);
-    cdb_callback_time_dist[l] = 0;
-  }
-  printf("\nin message ops:\n");
-  for (int m = 0; m < TIME_DIST_BUCKETS_SIZE; ++m) {
-    printf("%d ", inmsg_time_dist[m]);
-    inmsg_time_dist[m] = 0;
-  }
-  printf("\ncluster send time:\n");
-  for (int n = 0; n < TIME_DIST_BUCKETS_SIZE; ++n) {
-    printf("%d ", cluster_send_time_dist[n]);
-    cluster_send_time_dist[n] = 0;
-  }
-#endif // ENABLE_TIME_TRACE
-}
-
-GlobalClusterPeriodicEvent::GlobalClusterPeriodicEvent() : Continuation(new_ProxyMutex())
-{
-  SET_HANDLER((GClusterPEHandler)&GlobalClusterPeriodicEvent::calloutEvent);
-}
-
-GlobalClusterPeriodicEvent::~GlobalClusterPeriodicEvent()
-{
-  _thisCallout->cancel(this);
-}
-
-void
-GlobalClusterPeriodicEvent::init()
-{
-  _thisCallout = eventProcessor.schedule_every(this, HRTIME_SECONDS(10), ET_CALL);
-}
-
-int
-GlobalClusterPeriodicEvent::calloutEvent(Event * /* e ATS_UNUSED */, void * /* data ATS_UNUSED */)
-{
-  if (time_trace) {
-    dump_time_buckets();
-  }
-  clusterProcessor.compute_cluster_mode();
-  return EVENT_CONT;
-}
-
-// End of ClusterLib.cc
diff --git a/iocore/cluster/ClusterLoadMonitor.cc b/iocore/cluster/ClusterLoadMonitor.cc
deleted file mode 100644
index 92575b3..0000000
--- a/iocore/cluster/ClusterLoadMonitor.cc
+++ /dev/null
@@ -1,307 +0,0 @@
-/** @file
-
-  A brief file description
-
-  @section license License
-
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
- */
-
-/****************************************************************************
-
-  ClusterLoadMonitor.cc
-****************************************************************************/
-
-#include "P_Cluster.h"
-int ClusterLoadMonitor::cf_monitor_enabled;
-int ClusterLoadMonitor::cf_ping_message_send_msec_interval;
-int ClusterLoadMonitor::cf_num_ping_response_buckets;
-int ClusterLoadMonitor::cf_msecs_per_ping_response_bucket;
-int ClusterLoadMonitor::cf_ping_latency_threshold_msecs;
-int ClusterLoadMonitor::cf_cluster_load_compute_msec_interval;
-int ClusterLoadMonitor::cf_cluster_periodic_msec_interval;
-int ClusterLoadMonitor::cf_ping_history_buf_length;
-int ClusterLoadMonitor::cf_cluster_load_clear_duration;
-int ClusterLoadMonitor::cf_cluster_load_exceed_duration;
-
-ClusterLoadMonitor::ClusterLoadMonitor(ClusterHandler *ch)
-  : Continuation(nullptr),
-    ch(ch),
-    ping_history_buf_head(0),
-    periodic_action(nullptr),
-    cluster_overloaded(0),
-    cancel_periodic(0),
-    cluster_load_msg_sequence_number(0),
-    cluster_load_msg_start_sequence_number(0)
-{
-  mutex = this->ch->mutex;
-  SET_HANDLER(&ClusterLoadMonitor::cluster_load_periodic);
-
-  ping_message_send_msec_interval = cf_ping_message_send_msec_interval ? cf_ping_message_send_msec_interval : 100;
-  Debug("cluster_monitor", "ping_message_send_msec_interval=%d", ping_message_send_msec_interval);
-
-  num_ping_response_buckets = cf_num_ping_response_buckets ? cf_num_ping_response_buckets : 100;
-  Debug("cluster_monitor", "num_ping_response_buckets=%d", num_ping_response_buckets);
-
-  msecs_per_ping_response_bucket = cf_msecs_per_ping_response_bucket ? cf_msecs_per_ping_response_bucket : 50;
-  Debug("cluster_monitor", "msecs_per_ping_response_bucket=%d", msecs_per_ping_response_bucket);
-
-  ping_latency_threshold_msecs = cf_ping_latency_threshold_msecs ? cf_ping_latency_threshold_msecs : 500;
-  Debug("cluster_monitor", "ping_latency_threshold_msecs=%d", ping_latency_threshold_msecs);
-
-  cluster_load_compute_msec_interval = cf_cluster_load_compute_msec_interval ? cf_cluster_load_compute_msec_interval : 5000;
-  Debug("cluster_monitor", "cluster_load_compute_msec_interval=%d", cluster_load_compute_msec_interval);
-
-  cluster_periodic_msec_interval = cf_cluster_periodic_msec_interval ? cf_cluster_periodic_msec_interval : 100;
-  Debug("cluster_monitor", "cluster_periodic_msec_interval=%d", cluster_periodic_msec_interval);
-
-  ping_history_buf_length = cf_ping_history_buf_length ? cf_ping_history_buf_length : 120;
-  Debug("cluster_monitor", "ping_history_buf_length=%d", ping_history_buf_length);
-
-  cluster_load_clear_duration = cf_cluster_load_clear_duration ? cf_cluster_load_clear_duration : 24;
-  Debug("cluster_monitor", "cluster_load_clear_duration=%d", cluster_load_clear_duration);
-
-  cluster_load_exceed_duration = cf_cluster_load_exceed_duration ? cf_cluster_load_exceed_duration : 4;
-  Debug("cluster_monitor", "cluster_load_exceed_duration=%d", cluster_load_exceed_duration);
-
-  int nbytes            = sizeof(int) * num_ping_response_buckets;
-  ping_response_buckets = (int *)ats_malloc(nbytes);
-  memset((char *)ping_response_buckets, 0, nbytes);
-
-  nbytes                    = sizeof(ink_hrtime) * ping_history_buf_length;
-  ping_response_history_buf = (ink_hrtime *)ats_malloc(nbytes);
-  memset((char *)ping_response_history_buf, 0, nbytes);
-
-  last_ping_message_sent    = HRTIME_SECONDS(0);
-  last_cluster_load_compute = HRTIME_SECONDS(0);
-}
-
-void
-ClusterLoadMonitor::init()
-{
-  periodic_action = eventProcessor.schedule_every(this, HRTIME_MSECONDS(cluster_periodic_msec_interval), ET_CALL);
-}
-
-ClusterLoadMonitor::~ClusterLoadMonitor()
-{
-  //
-  // Note: Since the ClusterLoadMonitor is only associated
-  //       with the ClusterHandler, a periodic callback operating
-  //       on a freed ClusterLoadMonitor is not possible, since the
-  //       ClusterHandler is only deleted after several minutes.  Allowing
-  //       plenty of time for the periodic to cancel itself via the
-  //       "cancel_periodic" flag.
-  //
-  ink_release_assert(!periodic_action);
-  if (ping_response_buckets) {
-    ats_free(ping_response_buckets);
-    ping_response_buckets = nullptr;
-  }
-  if (ping_response_history_buf) {
-    ats_free(ping_response_history_buf);
-    ping_response_history_buf = nullptr;
-  }
-}
-
-void
-ClusterLoadMonitor::cancel_monitor()
-{
-  if (!cancel_periodic)
-    cancel_periodic = 1;
-}
-
-bool
-ClusterLoadMonitor::is_cluster_overloaded()
-{
-  return (cluster_overloaded ? true : false);
-}
-
-void
-ClusterLoadMonitor::compute_cluster_load()
-{
-  // Compute ping message latency by scanning the response time
-  // buckets and averaging the results.
-
-  int n;
-  int sum      = 0;
-  int entries  = 0;
-  int n_bucket = 0;
-
-  for (n = 0; n < num_ping_response_buckets; ++n) {
-    if (ping_response_buckets[n]) {
-      entries += ping_response_buckets[n];
-      sum += (ping_response_buckets[n] * (n + 1));
-    }
-    ping_response_buckets[n] = 0;
-  }
-  if (entries) {
-    n_bucket = sum / entries;
-  } else {
-    n_bucket = 1;
-  }
-  ink_hrtime current_ping_latency = HRTIME_MSECONDS(n_bucket * msecs_per_ping_response_bucket);
-
-  // Invalidate messages associated with this sample interval
-  cluster_load_msg_start_sequence_number = cluster_load_msg_sequence_number;
-
-  // Log ping latency in history buffer.
-
-  ping_response_history_buf[ping_history_buf_head++] = current_ping_latency;
-  ping_history_buf_head                              = ping_history_buf_head % ping_history_buf_length;
-
-  // Determine the current state of the cluster interconnect using
-  // the configured limits.  We determine the state as follows.
-  //   if (cluster overloaded)
-  //     Determine if it is still in the overload state by examining
-  //     the last 'cluster_load_clear_duration' entries in the history
-  //     buffer and declaring it not overloaded if none of the entries
-  //     exceed the threshold.
-  //   else
-  //     Determine if it is now in the overload state by examining
-  //     the last 'cluster_load_exceed_duration' entries in the history
-  //     buffer and declaring it overloaded if all of the entries
-  //     exceed the threshold.
-
-  int start, end;
-  ink_hrtime ping_latency_threshold = HRTIME_MSECONDS(ping_latency_threshold_msecs);
-
-  start = ping_history_buf_head - 1;
-  if (start < 0)
-    start += ping_history_buf_length;
-  end = start;
-
-  if (cluster_overloaded) {
-    end -= (cluster_load_clear_duration <= ping_history_buf_length ? cluster_load_clear_duration : ping_history_buf_length);
-  } else {
-    end -= (cluster_load_exceed_duration <= ping_history_buf_length ? cluster_load_exceed_duration : ping_history_buf_length);
-  }
-  if (end < 0)
-    end += ping_history_buf_length;
-
-  int threshold_clear    = 0;
-  int threshold_exceeded = 0;
-  do {
-    if (ping_response_history_buf[start] >= ping_latency_threshold)
-      ++threshold_exceeded;
-    else
-      ++threshold_clear;
-    if (--start < 0)
-      start = start + ping_history_buf_length;
-  } while (start != end);
-
-  if (cluster_overloaded) {
-    if (threshold_exceeded == 0)
-      cluster_overloaded = 0;
-  } else {
-    if (threshold_exceeded && (threshold_clear == 0))
-      cluster_overloaded = 1;
-  }
-  Debug("cluster_monitor", "[%u.%u.%u.%u] overload=%d, clear=%d, exceed=%d, latency=%d", DOT_SEPARATED(this->ch->machine->ip),
-        cluster_overloaded, threshold_clear, threshold_exceeded, n_bucket);
-}
-
-void
-ClusterLoadMonitor::note_ping_response_time(ink_hrtime response_time, int sequence_number)
-{
-#ifdef CLUSTER_TOMCAT
-  ProxyMutex *mutex = this->ch->mutex.get(); // hack for stats
-#endif
-
-  CLUSTER_SUM_DYN_STAT(CLUSTER_PING_TIME_STAT, response_time);
-  int bucket = (int)(response_time / HRTIME_MSECONDS(msecs_per_ping_response_bucket));
-  Debug("cluster_monitor_ping", "[%u.%u.%u.%u] ping: %d %d", DOT_SEPARATED(this->ch->machine->ip), bucket, sequence_number);
-
-  if (bucket >= num_ping_response_buckets)
-    bucket = num_ping_response_buckets - 1;
-  ink_atomic_increment(&ping_response_buckets[bucket], 1);
-}
-
-void
-ClusterLoadMonitor::recv_cluster_load_msg(cluster_load_ping_msg *m)
-{
-  // We have received back our ping message.
-  ink_hrtime now = Thread::get_hrtime();
-
-  if ((now >= m->send_time) &&
-      ((m->sequence_number >= cluster_load_msg_start_sequence_number) && (m->sequence_number < cluster_load_msg_sequence_number))) {
-    // Valid message, note response time.
-    note_ping_response_time(now - m->send_time, m->sequence_number);
-  }
-}
-
-void
-ClusterLoadMonitor::cluster_load_ping_rethandler(ClusterHandler *ch, void *data, int len)
-{
-  // Global cluster load ping message return handler which
-  // dispatches the result to the class specific handler.
-
-  if (ch) {
-    if (len == sizeof(struct cluster_load_ping_msg)) {
-      struct cluster_load_ping_msg m;
-      memcpy((void *)&m, data, len); // unmarshal
-
-      if (m.monitor && (m.magicno == cluster_load_ping_msg::CL_MSG_MAGICNO) &&
-          (m.version == cluster_load_ping_msg::CL_MSG_VERSION)) {
-        m.monitor->recv_cluster_load_msg(&m);
-      }
-    }
-  }
-}
-
-void
-ClusterLoadMonitor::send_cluster_load_msg(ink_hrtime current_time)
-{
-  // Build and send cluster load ping message.
-
-  struct cluster_load_ping_msg m(this);
-
-  m.sequence_number = cluster_load_msg_sequence_number++;
-  m.send_time       = current_time;
-  cluster_ping(ch, cluster_load_ping_rethandler, (void *)&m, sizeof(m));
-}
-
-int
-ClusterLoadMonitor::cluster_load_periodic(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */)
-{
-  // Perform periodic cluster load computation actions.
-
-  if (cancel_periodic) {
-    periodic_action->cancel();
-    periodic_action = nullptr;
-    return EVENT_DONE;
-  }
-
-  if (!cf_monitor_enabled) {
-    return EVENT_CONT;
-  }
-  // Generate periodic ping messages.
-
-  ink_hrtime current_time = Thread::get_hrtime();
-  if ((current_time - last_ping_message_sent) > HRTIME_MSECONDS(ping_message_send_msec_interval)) {
-    send_cluster_load_msg(current_time);
-    last_ping_message_sent = current_time;
-  }
-  // Compute cluster load.
-
-  if ((current_time - last_cluster_load_compute) > HRTIME_MSECONDS(cluster_load_compute_msec_interval)) {
-    compute_cluster_load();
-    last_cluster_load_compute = current_time;
-  }
-  return EVENT_CONT;
-}
-
-// End of ClusterLoadMonitor.cc
diff --git a/iocore/cluster/ClusterMachine.cc b/iocore/cluster/ClusterMachine.cc
deleted file mode 100644
index c1b3de5..0000000
--- a/iocore/cluster/ClusterMachine.cc
+++ /dev/null
@@ -1,292 +0,0 @@
-/** @file
-
-  A brief file description
-
-  @section license License
-
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
- */
-
-/****************************************************************************
-
-  Machine.cc
- ****************************************************************************/
-
-#include "ts/ink_platform.h"
-#include "ts/ink_file.h"
-#include <unistd.h>
-#include "P_Cluster.h"
-#include "ts/I_Layout.h"
-extern int num_of_cluster_threads;
-
-MachineList *machines_config = nullptr;
-MachineList *cluster_config  = nullptr;
-
-ProxyMutex *the_cluster_config_mutex;
-
-static ClusterMachine *cluster_machine;
-
-MachineList *
-the_cluster_machines_config()
-{
-  return machines_config;
-}
-
-MachineList *
-the_cluster_config()
-{
-  return cluster_config;
-}
-
-ClusterMachine *
-this_cluster_machine()
-{
-  return cluster_machine;
-}
-
-void
-create_this_cluster_machine()
-{
-  the_cluster_config_mutex = new_ProxyMutex();
-  cluster_machine          = new ClusterMachine;
-}
-
-ClusterMachine::ClusterMachine(char *ahostname, unsigned int aip, int aport)
-  : dead(false),
-    hostname(ahostname),
-    ip(aip),
-    cluster_port(aport),
-    num_connections(0),
-    now_connections(0),
-    free_connections(0),
-    rr_count(0),
-    msg_proto_major(0),
-    msg_proto_minor(0),
-    clusterHandlers(nullptr)
-{
-  ProxyMutex *mutex = this_ethread()->mutex.get();
-  CLUSTER_INCREMENT_DYN_STAT(CLUSTER_MACHINES_ALLOCATED_STAT);
-  if (!aip) {
-    char localhost[1024];
-    if (!ahostname) {
-      ink_release_assert(!gethostname(localhost, 1023));
-      ahostname = localhost;
-    }
-    hostname = ats_strdup(ahostname);
-
-// If we are running if the manager, it the our ip address for
-//   clustering from the manager, so the manager can control what
-//   interface we cluster over.  Otherwise figure it out ourselves
-#ifdef LOCAL_CLUSTER_TEST_MODE
-    ip = inet_addr("127.0.0.1");
-#else
-#ifdef CLUSTER_TEST
-    int clustering_enabled = true;
-#else
-    int clustering_enabled = !!getenv("PROXY_CLUSTER_ADDR");
-#endif
-    if (clustering_enabled) {
-      char *clusterIP = getenv("PROXY_CLUSTER_ADDR");
-      Debug("cluster_note", "[Machine::Machine] Cluster IP addr: %s", clusterIP);
-      ip = inet_addr(clusterIP);
-    } else {
-      struct addrinfo hints;
-      memset(&hints, 0, sizeof(hints));
-      hints.ai_family = AF_INET;
-      struct addrinfo *result, *rp;
-      int err = getaddrinfo(ahostname, nullptr, &hints, &result);
-      if (err != 0) {
-        Warning("unable to DNS %s: %s", ahostname, gai_strerror(err));
-        ip = 0;
-      } else {
-        // lowest IP address
-
-        ip = (unsigned int)-1; // 0xFFFFFFFF
-        for (rp = result; rp != nullptr; rp = rp->ai_next)
-          if (ip > reinterpret_cast<sockaddr_in *>(rp->ai_addr)->sin_addr.s_addr)
-            ip = reinterpret_cast<sockaddr_in *>(rp->ai_addr)->sin_addr.s_addr;
-        if (ip == (unsigned int)-1)
-          ip = 0;
-        freeaddrinfo(result);
-      }
-      // ip = htonl(ip); for the alpha!
-    }
-#endif // LOCAL_CLUSTER_TEST_MODE
-  } else {
-    ip = aip;
-
-    struct sockaddr_in addr;
-    memset(&addr, 0, sizeof(addr));
-    addr.sin_addr.s_addr = ip;
-    char hbuf[NI_MAXHOST];
-    int err = getnameinfo(reinterpret_cast<sockaddr *>(&addr), sizeof(addr), hbuf, sizeof(hbuf), nullptr, 0, 0);
-    if (err != 0) {
-      Alias32 x;
-      memcpy(&x.u32, &ip, sizeof(x.u32));
-      Debug("machine_debug", "unable to reverse DNS %u.%u.%u.%u: %s", x.byte[0], x.byte[1], x.byte[2], x.byte[3],
-            gai_strerror(err));
-    } else
-      hostname = ats_strdup(hbuf);
-  }
-  if (hostname)
-    hostname_len = strlen(hostname);
-  else
-    hostname_len = 0;
-
-  num_connections = num_of_cluster_threads;
-  clusterHandlers = (ClusterHandler **)ats_calloc(num_connections, sizeof(ClusterHandler *));
-}
-
-ClusterHandler *
-ClusterMachine::pop_ClusterHandler(int no_rr)
-{
-  int find    = 0;
-  int64_t now = rr_count;
-  if (no_rr == 0) {
-    ink_atomic_increment(&rr_count, 1);
-  }
-
-  /* will happen when ts start (cluster connection is not established) */
-  while (!clusterHandlers[now % this->num_connections] && (find < this->num_connections)) {
-    now++;
-    find++;
-  }
-  return this->clusterHandlers[now % this->num_connections];
-}
-
-ClusterMachine::~ClusterMachine()
-{
-  ats_free(hostname);
-  ats_free(clusterHandlers);
-}
-
-struct MachineTimeoutContinuation;
-typedef int (MachineTimeoutContinuation::*McTimeoutContHandler)(int, void *);
-struct MachineTimeoutContinuation : public Continuation {
-  ClusterMachine *m;
-  int
-  dieEvent(int event, Event *e)
-  {
-    (void)event;
-    (void)e;
-    delete m;
-    delete this;
-    return EVENT_DONE;
-  }
-
-  MachineTimeoutContinuation(ClusterMachine *am) : Continuation(nullptr), m(am)
-  {
-    SET_HANDLER((McTimeoutContHandler)&MachineTimeoutContinuation::dieEvent);
-  }
-};
-
-void
-free_ClusterMachine(ClusterMachine *m)
-{
-  ProxyMutex *mutex = this_ethread()->mutex.get();
-  // delay before the final free
-  CLUSTER_INCREMENT_DYN_STAT(CLUSTER_MACHINES_FREED_STAT);
-  m->dead = true;
-  eventProcessor.schedule_in(new MachineTimeoutContinuation(m), MACHINE_TIMEOUT, ET_CALL);
-}
-
-void
-free_MachineList(MachineList *l)
-{
-  new_Freer(l, MACHINE_TIMEOUT);
-}
-
-MachineList *
-read_MachineList(const char *filename, int afd)
-{
-  char line[256];
-  int n = -1, i = 0, ln = 0;
-  ats_scoped_obj<MachineList> l;
-  ink_assert(filename || (afd != -1));
-  ats_scoped_str path(RecConfigReadConfigPath(nullptr, filename));
-  ats_scoped_fd sfd;
-
-  int fd = ((afd != -1) ? afd : sfd = open(path, O_RDONLY));
-  if (fd >= 0) {
-    while (ink_file_fd_readline(fd, sizeof(line) - 1, line) > 0) {
-      ln++;
-      if (*line == '#')
-        continue;
-      if (n == -1 && ParseRules::is_digit(*line)) {
-        n = atoi(line);
-        if (n > 0) {
-          l    = (MachineList *)operator new(sizeof(MachineList) + (n - 1) * sizeof(MachineListElement));
-          l->n = 0;
-        } else {
-          l = nullptr;
-        }
-        continue;
-      }
-      if (l && ParseRules::is_digit(*line) && i < n) {
-        char *port = strchr(line, ':');
-        if (!port)
-          goto Lfail;
-        *port++          = 0;
-        l->machine[i].ip = inet_addr(line);
-        if (-1 == (int)l->machine[i].ip) {
-          if (afd == -1) {
-            Warning("read machine list failure, bad ip, line %d", ln);
-            return nullptr;
-          } else {
-            char s[256];
-            snprintf(s, sizeof s, "bad ip, line %d", ln);
-            return (MachineList *)ats_strdup(s);
-          }
-        }
-        l->machine[i].port = atoi(port);
-        if (!l->machine[i].port)
-          goto Lfail;
-        i++;
-        l->n++;
-        continue;
-      Lfail:
-        if (afd == -1) {
-          Warning("read machine list failure, bad port, line %d", ln);
-          return nullptr;
-        } else {
-          char s[256];
-          snprintf(s, sizeof s, "bad port, line %d", ln);
-          return (MachineList *)ats_strdup(s);
-        }
-      }
-    }
-  } else {
-    Warning("read machine list failure, open failed");
-    return nullptr;
-  }
-
-  if (n >= 0) {
-    if (i != n) {
-      if (afd == -1) {
-        Warning("read machine list failure, length mismatch");
-        return nullptr;
-      } else
-        return (MachineList *)ats_strdup("number of machines does not match length of list\n");
-    }
-  }
-
-  if (afd != -1) {
-    return (MachineList *)nullptr;
-  }
-
-  return l.release();
-}
diff --git a/iocore/cluster/ClusterProcessor.cc b/iocore/cluster/ClusterProcessor.cc
deleted file mode 100644
index 64e4282..0000000
--- a/iocore/cluster/ClusterProcessor.cc
+++ /dev/null
@@ -1,764 +0,0 @@
-/** @file
-
-  A brief file description
-
-  @section license License
-
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
- */
-
-/****************************************************************************
-
-  ClusterProcessor.cc
-****************************************************************************/
-
-#include "P_Cluster.h"
-/*************************************************************************/
-// ClusterProcessor member functions (Public class)
-/*************************************************************************/
-int cluster_port_number      = DEFAULT_CLUSTER_PORT_NUMBER;
-int cache_clustering_enabled = 0;
-int num_of_cluster_threads   = DEFAULT_NUMBER_OF_CLUSTER_THREADS;
-
-ClusterProcessor clusterProcessor;
-RecRawStatBlock *cluster_rsb = nullptr;
-int ET_CLUSTER;
-
-ClusterProcessor::ClusterProcessor() : accept_handler(nullptr), this_cluster(nullptr)
-{
-}
-
-ClusterProcessor::~ClusterProcessor()
-{
-  if (accept_handler) {
-    accept_handler->ShutdownDelete();
-    accept_handler = nullptr;
-  }
-}
-
-int
-ClusterProcessor::internal_invoke_remote(ClusterHandler *ch, int cluster_fn, void *data, int len, int options, void *cmsg)
-{
-  ProxyMutex *mutex = this_ethread()->mutex.get();
-  //
-  // RPC facility for intercluster communication available to other
-  //  subsystems.
-  //
-  bool steal         = (options & CLUSTER_OPT_STEAL ? true : false);
-  bool delay         = (options & CLUSTER_OPT_DELAY ? true : false);
-  bool data_in_ocntl = (options & CLUSTER_OPT_DATA_IS_OCONTROL ? true : false);
-  bool malloced      = (cluster_fn == CLUSTER_FUNCTION_MALLOCED);
-  OutgoingControl *c;
-
-  if (!ch || (!malloced && !((unsigned int)cluster_fn < (uint32_t)SIZE_clusterFunction))) {
-    // Invalid message or node is down, free message data
-    if (cmsg) {
-      invoke_remote_data_args *args = (invoke_remote_data_args *)(((OutgoingControl *)cmsg)->data + sizeof(int32_t));
-      ink_assert(args->magicno == invoke_remote_data_args::MagicNo);
-
-      args->data_oc->freeall();
-      ((OutgoingControl *)cmsg)->freeall();
-    }
-    if (data_in_ocntl) {
-      c = *((OutgoingControl **)((char *)data - sizeof(OutgoingControl *)));
-      c->freeall();
-    }
-    if (malloced) {
-      ats_free(data);
-    }
-    return -1;
-  }
-
-  if (data_in_ocntl) {
-    c = *((OutgoingControl **)((char *)data - sizeof(OutgoingControl *)));
-  } else {
-    c = OutgoingControl::alloc();
-  }
-  CLUSTER_INCREMENT_DYN_STAT(CLUSTER_CTRL_MSGS_SENT_STAT);
-  c->submit_time = Thread::get_hrtime();
-
-  if (malloced) {
-    c->set_data((char *)data, len);
-  } else {
-    if (!data_in_ocntl) {
-      c->len = len + sizeof(int32_t);
-      c->alloc_data();
-    }
-    if (!c->fast_data()) {
-      CLUSTER_INCREMENT_DYN_STAT(CLUSTER_SLOW_CTRL_MSGS_SENT_STAT);
-    }
-    *(int32_t *)c->data = cluster_fn;
-    if (!data_in_ocntl) {
-      memcpy(c->data + sizeof(int32_t), data, len);
-    }
-  }
-
-  SET_CONTINUATION_HANDLER(c, (OutgoingCtrlHandler)&OutgoingControl::startEvent);
-
-  /////////////////////////////////////
-  // Compound message adjustments
-  /////////////////////////////////////
-  if (cmsg) {
-    invoke_remote_data_args *args = (invoke_remote_data_args *)(((OutgoingControl *)cmsg)->data + sizeof(int32_t));
-    ink_assert(args->magicno == invoke_remote_data_args::MagicNo);
-    args->msg_oc = c;
-    c            = (OutgoingControl *)cmsg;
-  }
-#ifndef CLUSTER_THREAD_STEALING
-  delay = true;
-#endif
-  if (!delay) {
-    EThread *tt = this_ethread();
-    {
-      int q = ClusterFuncToQpri(cluster_fn);
-      ink_atomiclist_push(&ch->outgoing_control_al[q], (void *)c);
-
-      MUTEX_TRY_LOCK(lock, ch->mutex, tt);
-      if (!lock.is_locked()) {
-        if (ch->thread && ch->thread->signal_hook)
-          ch->thread->signal_hook(ch->thread);
-        return 1;
-      }
-      if (steal)
-        ch->steal_thread(tt);
-      return 1;
-    }
-  } else {
-    c->mutex = ch->mutex;
-    eventProcessor.schedule_imm_signal(c);
-    return 0;
-  }
-}
-
-int
-ClusterProcessor::invoke_remote(ClusterHandler *ch, int cluster_fn, void *data, int len, int options)
-{
-  return internal_invoke_remote(ch, cluster_fn, data, len, options, (void *)nullptr);
-}
-
-int
-ClusterProcessor::invoke_remote_data(ClusterHandler *ch, int cluster_fn, void *data, int data_len, IOBufferBlock *buf,
-                                     int dest_channel, ClusterVCToken *token, void (*bufdata_free_proc)(void *),
-                                     void *bufdata_free_proc_arg, int options)
-{
-  if (!buf) {
-    // No buffer data, translate this into a invoke_remote() request
-    return internal_invoke_remote(ch, cluster_fn, data, data_len, options, (void *)nullptr);
-  }
-  ink_assert(data);
-  ink_assert(data_len);
-  ink_assert(dest_channel);
-  ink_assert(token);
-  ink_assert(bufdata_free_proc);
-  ink_assert(bufdata_free_proc_arg);
-
-  /////////////////////////////////////////////////////////////////////////
-  // Build the compound message as described by invoke_remote_data_args.
-  /////////////////////////////////////////////////////////////////////////
-
-  // Build OutgoingControl for buffer data
-  OutgoingControl *bufdata_oc = OutgoingControl::alloc();
-  bufdata_oc->set_data(buf, bufdata_free_proc, bufdata_free_proc_arg);
-
-  // Build OutgoingControl for compound message header
-  invoke_remote_data_args mh;
-  mh.msg_oc       = nullptr;
-  mh.data_oc      = bufdata_oc;
-  mh.dest_channel = dest_channel;
-  mh.token        = *token;
-
-  OutgoingControl *chdr = OutgoingControl::alloc();
-  chdr->submit_time     = Thread::get_hrtime();
-  chdr->len             = sizeof(int32_t) + sizeof(mh);
-  chdr->alloc_data();
-  *(int32_t *)chdr->data = -1; // always -1 for compound message
-  memcpy(chdr->data + sizeof(int32_t), (char *)&mh, sizeof(mh));
-
-  return internal_invoke_remote(ch, cluster_fn, data, data_len, options, (void *)chdr);
-}
-
-// TODO: Why pass in the length here if not used ?
-void
-ClusterProcessor::free_remote_data(char *p, int /* l ATS_UNUSED */)
-{
-  char *d      = p - sizeof(int32_t); // reset to ptr to function code
-  int data_hdr = ClusterControl::DATA_HDR;
-
-  ink_release_assert(*((uint8_t *)(d - data_hdr + 1)) == (uint8_t)ALLOC_DATA_MAGIC);
-  unsigned char size_index = *(d - data_hdr);
-  if (!(size_index & 0x80)) {
-    ink_release_assert(size_index <= (DEFAULT_BUFFER_SIZES - 1));
-  } else {
-    ink_release_assert(size_index == 0xff);
-  }
-
-  // Extract 'this' pointer
-
-  ClusterControl *ccl;
-  memcpy((char *)&ccl, (d - data_hdr + 2), sizeof(void *));
-  ink_assert(ccl->valid_alloc_data());
-
-  // Deallocate control structure and data
-
-  ccl->freeall();
-}
-
-ClusterVConnection *
-ClusterProcessor::open_local(Continuation *cont, ClusterMachine * /* m ATS_UNUSED */, ClusterVCToken &token, int options)
-{
-  //
-  //  New connect protocol.
-  //  As a VC initiator, establish the VC connection to the remote node
-  //  by allocating the VC locally and requiring the caller to pass the
-  //  token and channel id in the remote request.  The remote handler calls
-  //  connect_local to establish the remote side of the connection.
-  //
-  bool immediate       = ((options & CLUSTER_OPT_IMMEDIATE) ? true : false);
-  bool allow_immediate = ((options & CLUSTER_OPT_ALLOW_IMMEDIATE) ? true : false);
-
-  ClusterHandler *ch = ((CacheContinuation *)cont)->ch;
-  if (!ch)
-    return nullptr;
-  EThread *t = ch->thread;
-  if (!t)
-    return nullptr;
-
-  EThread *thread        = this_ethread();
-  ProxyMutex *mutex      = thread->mutex.get();
-  ClusterVConnection *vc = clusterVCAllocator.alloc();
-  vc->new_connect_read   = (options & CLUSTER_OPT_CONN_READ ? 1 : 0);
-  vc->start_time         = Thread::get_hrtime();
-  vc->last_activity_time = vc->start_time;
-  vc->ch                 = ch;
-  vc->token.alloc();
-  vc->token.ch_id = ch->id;
-  token           = vc->token;
-#ifdef CLUSTER_THREAD_STEALING
-  CLUSTER_INCREMENT_DYN_STAT(CLUSTER_CONNECTIONS_OPENNED_STAT);
-  CLUSTER_INCREMENT_DYN_STAT(CLUSTER_CONNECTIONS_OPEN_STAT);
-  MUTEX_TRY_LOCK(lock, ch->mutex, thread);
-  if (!lock.is_locked()) {
-#endif
-    if (immediate) {
-      clusterVCAllocator_free(vc);
-      return nullptr;
-    }
-    vc->action_ = cont;
-    ink_atomiclist_push(&ch->external_incoming_open_local, (void *)vc);
-    if (ch->thread && ch->thread->signal_hook)
-      ch->thread->signal_hook(ch->thread);
-    return CLUSTER_DELAYED_OPEN;
-
-#ifdef CLUSTER_THREAD_STEALING
-  } else {
-    if (!(immediate || allow_immediate))
-      vc->action_ = cont;
-    if (vc->start(thread) < 0) {
-      return nullptr;
-    }
-    if (immediate || allow_immediate) {
-      return vc;
-    } else {
-      return CLUSTER_DELAYED_OPEN;
-    }
-  }
-#endif
-}
-
-ClusterVConnection *
-ClusterProcessor::connect_local(Continuation *cont, ClusterVCToken *token, int channel, int options)
-{
-  //
-  // Establish VC connection initiated by remote node on the local node
-  // using the given token and channel id.
-  //
-  bool immediate       = ((options & CLUSTER_OPT_IMMEDIATE) ? true : false);
-  bool allow_immediate = ((options & CLUSTER_OPT_ALLOW_IMMEDIATE) ? true : false);
-
-#ifdef LOCAL_CLUSTER_TEST_MODE
-  int ip = inet_addr("127.0.0.1");
-  ClusterMachine *m;
-  m = this_cluster->current_configuration()->find(ip, token->ip_created);
-#else
-  ClusterMachine *m = this_cluster->current_configuration()->find(token->ip_created);
-#endif
-  if (!m)
-    return nullptr;
-  if (token->ch_id >= (uint32_t)m->num_connections)
-    return nullptr;
-  ClusterHandler *ch = m->clusterHandlers[token->ch_id];
-  if (!ch)
-    return nullptr;
-  EThread *t = ch->thread;
-  if (!t)
-    return nullptr;
-
-  EThread *thread        = this_ethread();
-  ProxyMutex *mutex      = thread->mutex.get();
-  ClusterVConnection *vc = clusterVCAllocator.alloc();
-  vc->new_connect_read   = (options & CLUSTER_OPT_CONN_READ ? 1 : 0);
-  vc->start_time         = Thread::get_hrtime();
-  vc->last_activity_time = vc->start_time;
-  vc->ch                 = ch;
-  vc->token              = *token;
-  vc->channel            = channel;
-#ifdef CLUSTER_THREAD_STEALING
-  CLUSTER_INCREMENT_DYN_STAT(CLUSTER_CONNECTIONS_OPENNED_STAT);
-  CLUSTER_INCREMENT_DYN_STAT(CLUSTER_CONNECTIONS_OPEN_STAT);
-  MUTEX_TRY_LOCK(lock, ch->mutex, thread);
-  if (!lock.is_locked()) {
-#endif
-    if (immediate) {
-      clusterVCAllocator_free(vc);
-      return nullptr;
-    }
-    vc->mutex   = ch->mutex;
-    vc->action_ = cont;
-    ch->thread->schedule_imm_signal(vc);
-    return CLUSTER_DELAYED_OPEN;
-#ifdef CLUSTER_THREAD_STEALING
-  } else {
-    if (!(immediate || allow_immediate))
-      vc->action_ = cont;
-    if (vc->start(thread) < 0) {
-      return nullptr;
-    }
-    if (immediate || allow_immediate) {
-      return vc;
-    } else {
-      return CLUSTER_DELAYED_OPEN;
-    }
-  }
-#endif
-}
-
-bool
-ClusterProcessor::disable_remote_cluster_ops(ClusterMachine *m)
-{
-  ClusterHandler *ch = m->pop_ClusterHandler(1);
-  if (ch) {
-    return ch->disable_remote_cluster_ops;
-  } else {
-    return true;
-  }
-}
-
-////////////////////////////////////////////////////////////////////////////
-// Simplify debug access to stats
-////////////////////////////////////////////////////////////////////////////
-////////////////////////////////////////////////////////////////////////////
-
-GlobalClusterPeriodicEvent *PeriodicClusterEvent;
-
-#ifdef CLUSTER_TOMCAT
-extern int cache_clustering_enabled;
-
-int CacheClusterMonitorEnabled      = 0;
-int CacheClusterMonitorIntervalSecs = 1;
-
-int cluster_send_buffer_size        = 0;
-int cluster_receive_buffer_size     = 0;
-unsigned long cluster_sockopt_flags = 0;
-unsigned long cluster_packet_mark   = 0;
-unsigned long cluster_packet_tos    = 0;
-
-int RPC_only_CacheCluster = 0;
-#endif
-
-int
-ClusterProcessor::init()
-{
-  cluster_rsb = RecAllocateRawStatBlock((int)cluster_stat_count);
-  //
-  // Statistics callbacks
-  //
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.connections_open", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_CONNECTIONS_OPEN_STAT, RecRawStatSyncSum);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_CONNECTIONS_OPEN_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.connections_opened", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_CONNECTIONS_OPENNED_STAT, RecRawStatSyncSum);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_CONNECTIONS_OPENNED_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.connections_closed", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_CON_TOTAL_TIME_STAT, RecRawStatSyncCount);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_CON_TOTAL_TIME_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.slow_ctrl_msgs_sent", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_SLOW_CTRL_MSGS_SENT_STAT, RecRawStatSyncCount);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_SLOW_CTRL_MSGS_SENT_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.connections_read_locked", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_CONNECTIONS_READ_LOCKED_STAT, RecRawStatSyncSum);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_CONNECTIONS_READ_LOCKED_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.connections_write_locked", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_CONNECTIONS_WRITE_LOCKED_STAT, RecRawStatSyncSum);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_CONNECTIONS_WRITE_LOCKED_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.reads", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_READ_BYTES_STAT, RecRawStatSyncCount);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_READ_BYTES_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.read_bytes", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_READ_BYTES_STAT, RecRawStatSyncSum);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_READ_BYTES_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.writes", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_WRITE_BYTES_STAT, RecRawStatSyncCount);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_WRITE_BYTES_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.write_bytes", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_WRITE_BYTES_STAT, RecRawStatSyncSum);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_WRITE_BYTES_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.control_messages_sent", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_CTRL_MSGS_SEND_TIME_STAT, RecRawStatSyncCount);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_CTRL_MSGS_SEND_TIME_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.control_messages_received", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_CTRL_MSGS_RECV_TIME_STAT, RecRawStatSyncCount);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_CTRL_MSGS_RECV_TIME_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.op_delayed_for_lock", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_OP_DELAYED_FOR_LOCK_STAT, RecRawStatSyncSum);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_OP_DELAYED_FOR_LOCK_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.connections_bumped", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_CONNECTIONS_BUMPED_STAT, RecRawStatSyncSum);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_CONNECTIONS_BUMPED_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.net_backup", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_NET_BACKUP_STAT, RecRawStatSyncCount);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_NET_BACKUP_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.nodes", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_NODES_STAT, RecRawStatSyncSum);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_NODES_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.machines_allocated", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_MACHINES_ALLOCATED_STAT, RecRawStatSyncSum);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_MACHINES_ALLOCATED_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.machines_freed", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_MACHINES_FREED_STAT, RecRawStatSyncSum);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_MACHINES_FREED_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.configuration_changes", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_CONFIGURATION_CHANGES_STAT, RecRawStatSyncCount);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_CONFIGURATION_CHANGES_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.delayed_reads", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_DELAYED_READS_STAT, RecRawStatSyncSum);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_DELAYED_READS_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.byte_bank_used", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_BYTE_BANK_USED_STAT, RecRawStatSyncSum);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_BYTE_BANK_USED_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.alloc_data_news", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_ALLOC_DATA_NEWS_STAT, RecRawStatSyncSum);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_ALLOC_DATA_NEWS_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.write_bb_mallocs", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_WRITE_BB_MALLOCS_STAT, RecRawStatSyncSum);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_WRITE_BB_MALLOCS_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.partial_reads", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_PARTIAL_READS_STAT, RecRawStatSyncSum);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_PARTIAL_READS_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.partial_writes", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_PARTIAL_WRITES_STAT, RecRawStatSyncSum);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_PARTIAL_WRITES_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.cache_outstanding", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_CACHE_OUTSTANDING_STAT, RecRawStatSyncSum);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_CACHE_OUTSTANDING_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.remote_op_timeouts", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_REMOTE_OP_TIMEOUTS_STAT, RecRawStatSyncSum);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_REMOTE_OP_TIMEOUTS_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.remote_op_reply_timeouts", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_REMOTE_OP_REPLY_TIMEOUTS_STAT, RecRawStatSyncSum);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_REMOTE_OP_REPLY_TIMEOUTS_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.chan_inuse", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_CHAN_INUSE_STAT, RecRawStatSyncSum);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_CHAN_INUSE_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.open_delays", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_OPEN_DELAY_TIME_STAT, RecRawStatSyncSum);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_OPEN_DELAY_TIME_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.connections_avg_time", RECD_FLOAT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_CON_TOTAL_TIME_STAT, RecRawStatSyncHrTimeAvg);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_CON_TOTAL_TIME_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.control_messages_avg_send_time", RECD_FLOAT,
-                     RECP_NON_PERSISTENT, (int)CLUSTER_CTRL_MSGS_SEND_TIME_STAT, RecRawStatSyncHrTimeAvg);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_CTRL_MSGS_SEND_TIME_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.control_messages_avg_receive_time", RECD_FLOAT,
-                     RECP_NON_PERSISTENT, (int)CLUSTER_CTRL_MSGS_RECV_TIME_STAT, RecRawStatSyncHrTimeAvg);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_CTRL_MSGS_RECV_TIME_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.open_delay_time", RECD_FLOAT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_OPEN_DELAY_TIME_STAT, RecRawStatSyncHrTimeAvg);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_OPEN_DELAY_TIME_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.cache_callback_time", RECD_FLOAT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_CACHE_CALLBACK_TIME_STAT, RecRawStatSyncHrTimeAvg);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_CACHE_CALLBACK_TIME_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.rmt_cache_callback_time", RECD_FLOAT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_CACHE_RMT_CALLBACK_TIME_STAT, RecRawStatSyncHrTimeAvg);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_CACHE_RMT_CALLBACK_TIME_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.lkrmt_cache_callback_time", RECD_FLOAT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_CACHE_LKRMT_CALLBACK_TIME_STAT, RecRawStatSyncHrTimeAvg);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_CACHE_LKRMT_CALLBACK_TIME_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.local_connection_time", RECD_FLOAT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_LOCAL_CONNECTION_TIME_STAT, RecRawStatSyncHrTimeAvg);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_LOCAL_CONNECTION_TIME_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.remote_connection_time", RECD_FLOAT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_REMOTE_CONNECTION_TIME_STAT, RecRawStatSyncHrTimeAvg);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_REMOTE_CONNECTION_TIME_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.rdmsg_assemble_time", RECD_FLOAT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_RDMSG_ASSEMBLE_TIME_STAT, RecRawStatSyncHrTimeAvg);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_RDMSG_ASSEMBLE_TIME_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.cluster_ping_time", RECD_FLOAT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_PING_TIME_STAT, RecRawStatSyncHrTimeAvg);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_PING_TIME_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.cache_callbacks", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_CACHE_CALLBACK_TIME_STAT, RecRawStatSyncCount);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_CACHE_CALLBACK_TIME_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.rmt_cache_callbacks", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_CACHE_RMT_CALLBACK_TIME_STAT, RecRawStatSyncCount);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_CACHE_RMT_CALLBACK_TIME_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.lkrmt_cache_callbacks", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_CACHE_LKRMT_CALLBACK_TIME_STAT, RecRawStatSyncCount);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_CACHE_LKRMT_CALLBACK_TIME_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.local_connections_closed", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_LOCAL_CONNECTION_TIME_STAT, RecRawStatSyncCount);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_LOCAL_CONNECTION_TIME_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.remote_connections_closed", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_REMOTE_CONNECTION_TIME_STAT, RecRawStatSyncCount);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_REMOTE_CONNECTION_TIME_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.setdata_no_clustervc", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)cluster_setdata_no_CLUSTERVC_STAT, RecRawStatSyncCount);
-  CLUSTER_CLEAR_DYN_STAT(cluster_setdata_no_CLUSTERVC_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.setdata_no_tunnel", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_SETDATA_NO_TUNNEL_STAT, RecRawStatSyncCount);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_SETDATA_NO_TUNNEL_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.setdata_no_cachevc", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_SETDATA_NO_CACHEVC_STAT, RecRawStatSyncCount);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_SETDATA_NO_CACHEVC_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.setdata_no_cluster", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)cluster_setdata_no_CLUSTER_STAT, RecRawStatSyncCount);
-  CLUSTER_CLEAR_DYN_STAT(cluster_setdata_no_CLUSTER_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.vc_write_stall", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_VC_WRITE_STALL_STAT, RecRawStatSyncCount);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_VC_WRITE_STALL_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.no_remote_space", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_NO_REMOTE_SPACE_STAT, RecRawStatSyncCount);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_NO_REMOTE_SPACE_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.level1_bank", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_LEVEL1_BANK_STAT, RecRawStatSyncCount);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_LEVEL1_BANK_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.multilevel_bank", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_MULTILEVEL_BANK_STAT, RecRawStatSyncCount);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_MULTILEVEL_BANK_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.vc_cache_insert_lock_misses", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_VC_CACHE_INSERT_LOCK_MISSES_STAT, RecRawStatSyncCount);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_VC_CACHE_INSERT_LOCK_MISSES_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.vc_cache_inserts", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_VC_CACHE_INSERTS_STAT, RecRawStatSyncCount);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_VC_CACHE_INSERTS_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.vc_cache_lookup_lock_misses", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_VC_CACHE_LOOKUP_LOCK_MISSES_STAT, RecRawStatSyncCount);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_VC_CACHE_LOOKUP_LOCK_MISSES_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.vc_cache_lookup_hits", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_VC_CACHE_LOOKUP_HITS_STAT, RecRawStatSyncCount);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_VC_CACHE_LOOKUP_HITS_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.vc_cache_lookup_misses", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_VC_CACHE_LOOKUP_MISSES_STAT, RecRawStatSyncCount);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_VC_CACHE_LOOKUP_MISSES_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.vc_cache_scans", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_VC_CACHE_SCANS_STAT, RecRawStatSyncCount);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_VC_CACHE_SCANS_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.vc_cache_scan_lock_misses", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_VC_CACHE_SCAN_LOCK_MISSES_STAT, RecRawStatSyncCount);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_VC_CACHE_SCAN_LOCK_MISSES_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.vc_cache_purges", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_VC_CACHE_PURGES_STAT, RecRawStatSyncCount);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_VC_CACHE_PURGES_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.write_lock_misses", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_WRITE_LOCK_MISSES_STAT, RecRawStatSyncCount);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_WRITE_LOCK_MISSES_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.vc_read_list_len", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_VC_READ_LIST_LEN_STAT, RecRawStatSyncAvg);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_VC_READ_LIST_LEN_STAT);
-  RecRegisterRawStat(cluster_rsb, RECT_PROCESS, "proxy.process.cluster.vc_write_list_len", RECD_INT, RECP_NON_PERSISTENT,
-                     (int)CLUSTER_VC_WRITE_LIST_LEN_STAT, RecRawStatSyncAvg);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_VC_WRITE_LIST_LEN_STAT);
-  CLUSTER_CLEAR_DYN_STAT(CLUSTER_NODES_STAT); // clear sum and count
-  // INKqa08033: win2k: ui: cluster warning light on
-  // Used to call CLUSTER_INCREMENT_DYN_STAT here; switch to SUM_GLOBAL_DYN_STAT
-  CLUSTER_SUM_GLOBAL_DYN_STAT(CLUSTER_NODES_STAT, 1); // one node in cluster, ME
-
-  REC_ReadConfigInteger(ClusterLoadMonitor::cf_monitor_enabled, "proxy.config.cluster.load_monitor_enabled");
-  REC_ReadConfigInteger(ClusterLoadMonitor::cf_ping_message_send_msec_interval, "proxy.config.cluster.ping_send_interval_msecs");
-  REC_ReadConfigInteger(ClusterLoadMonitor::cf_num_ping_response_buckets, "proxy.config.cluster.ping_response_buckets");
-  REC_ReadConfigInteger(ClusterLoadMonitor::cf_msecs_per_ping_response_bucket,
-                        "proxy.config.cluster.msecs_per_ping_response_bucket");
-  REC_ReadConfigInteger(ClusterLoadMonitor::cf_ping_latency_threshold_msecs, "proxy.config.cluster.ping_latency_threshold_msecs");
-  REC_ReadConfigInteger(ClusterLoadMonitor::cf_cluster_load_compute_msec_interval,
-                        "proxy.config.cluster.load_compute_interval_msecs");
-  REC_ReadConfigInteger(ClusterLoadMonitor::cf_cluster_periodic_msec_interval,
-                        "proxy.config.cluster.periodic_timer_interval_msecs");
-  REC_ReadConfigInteger(ClusterLoadMonitor::cf_ping_history_buf_length, "proxy.config.cluster.ping_history_buf_length");
-  REC_ReadConfigInteger(ClusterLoadMonitor::cf_cluster_load_clear_duration, "proxy.config.cluster.cluster_load_clear_duration");
-  REC_ReadConfigInteger(ClusterLoadMonitor::cf_cluster_load_exceed_duration, "proxy.config.cluster.cluster_load_exceed_duration");
-
-  //
-  // Configuration callbacks
-  //
-  if (cluster_port_number != DEFAULT_CLUSTER_PORT_NUMBER)
-    cluster_port = cluster_port_number;
-  else {
-    REC_ReadConfigInteger(cluster_port, "proxy.config.cluster.cluster_port");
-  }
-  if (num_of_cluster_threads == DEFAULT_NUMBER_OF_CLUSTER_THREADS)
-    REC_ReadConfigInteger(num_of_cluster_threads, "proxy.config.cluster.threads");
-
-  REC_EstablishStaticConfigInt32(CacheClusterMonitorEnabled, "proxy.config.cluster.enable_monitor");
-  REC_EstablishStaticConfigInt32(CacheClusterMonitorIntervalSecs, "proxy.config.cluster.monitor_interval_secs");
-  REC_ReadConfigInteger(cluster_receive_buffer_size, "proxy.config.cluster.receive_buffer_size");
-  REC_ReadConfigInteger(cluster_send_buffer_size, "proxy.config.cluster.send_buffer_size");
-  REC_ReadConfigInteger(cluster_sockopt_flags, "proxy.config.cluster.sock_option_flag");
-  REC_ReadConfigInteger(cluster_packet_mark, "proxy.config.cluster.sock_packet_mark");
-  REC_ReadConfigInteger(cluster_packet_tos, "proxy.config.cluster.sock_packet_tos");
-  REC_EstablishStaticConfigInt32(RPC_only_CacheCluster, "proxy.config.cluster.rpc_cache_cluster");
-
-  create_this_cluster_machine();
-  // Cluster API Initializations
-  clusterAPI_init();
-  // Start global Cluster periodic event
-  PeriodicClusterEvent = new GlobalClusterPeriodicEvent;
-  PeriodicClusterEvent->init();
-
-  this_cluster             = new Cluster;
-  ClusterConfiguration *cc = new ClusterConfiguration;
-  this_cluster->configurations.push(cc);
-  cc->n_machines  = 1;
-  cc->machines[0] = this_cluster_machine();
-  memset(cc->hash_table, 0, CLUSTER_HASH_TABLE_SIZE);
-  // 0 dummy output data
-
-  memset(channel_dummy_output, 0, sizeof(channel_dummy_output));
-
-  cache_clustering_enabled = 0;
-  Note("cache clustering disabled");
-
-  return 0;
-}
-
-// function added to adhere to the name calling convention of init functions
-int
-init_clusterprocessor()
-{
-  return clusterProcessor.init();
-}
-
-int
-ClusterProcessor::start()
-{
-#ifdef LOCAL_CLUSTER_TEST_MODE
-  this_cluster_machine()->cluster_port = cluster_port;
-#endif
-  if (cache_clustering_enabled && (cacheProcessor.IsCacheEnabled() == CACHE_INITIALIZED)) {
-    size_t stacksize;
-
-    REC_ReadConfigInteger(stacksize, "proxy.config.thread.default.stacksize");
-    ET_CLUSTER = eventProcessor.spawn_event_threads(num_of_cluster_threads, "ET_CLUSTER", stacksize);
-    for (int i = 0; i < eventProcessor.n_threads_for_type[ET_CLUSTER]; i++) {
-      initialize_thread_for_net(eventProcessor.eventthread[ET_CLUSTER][i]);
-    }
-    REC_RegisterConfigUpdateFunc("proxy.config.cluster.cluster_configuration", machine_config_change, (void *)CLUSTER_CONFIG);
-    do_machine_config_change((void *)CLUSTER_CONFIG, "proxy.config.cluster.cluster_configuration");
-// TODO: Remove this?
-#ifdef USE_SEPARATE_MACHINE_CONFIG
-    REC_RegisterConfigUpdateFunc("proxy.config.cluster.machine_configuration", machine_config_change, (void *)MACHINE_CONFIG);
-    do_machine_config_change((void *)MACHINE_CONFIG, "proxy.config.cluster.machine_configuration");
-#endif
-
-    accept_handler = new ClusterAccept(&cluster_port, cluster_receive_buffer_size, cluster_send_buffer_size);
-    accept_handler->Init();
-  }
-  return 0;
-}
-
-void
-ClusterProcessor::connect(char *hostname, int16_t id)
-{
-  //
-  // Construct a cluster link to the given machine
-  //
-  ClusterHandler *ch = new ClusterHandler;
-  SET_CONTINUATION_HANDLER(ch, (ClusterContHandler)&ClusterHandler::connectClusterEvent);
-  ch->hostname  = ats_strdup(hostname);
-  ch->connector = true;
-  ch->id        = id;
-  eventProcessor.schedule_imm(ch, ET_CLUSTER);
-}
-
-void
-ClusterProcessor::connect(unsigned int ip, int port, int16_t id, bool delay)
-{
-  //
-  // Construct a cluster link to the given machine
-  //
-  ClusterHandler *ch = new ClusterHandler;
-  SET_CONTINUATION_HANDLER(ch, (ClusterContHandler)&ClusterHandler::connectClusterEvent);
-  ch->ip        = ip;
-  ch->port      = port;
-  ch->connector = true;
-  ch->id        = id;
-  if (delay)
-    eventProcessor.schedule_in(ch, CLUSTER_MEMBER_DELAY, ET_CLUSTER);
-  else
-    eventProcessor.schedule_imm(ch, ET_CLUSTER);
-}
-
-void
-ClusterProcessor::send_machine_list(ClusterMachine *m)
-{
-  //
-  // In testing mode, cluster nodes automagically connect to all
-  // known hosts.  This function is called on connect to exchange those
-  // lists.
-  //
-  MachineListMessage mlistmsg;
-  int vers                 = MachineListMessage::protoToVersion(m->msg_proto_major);
-  ClusterConfiguration *cc = this_cluster->current_configuration();
-  void *data;
-  int len;
-
-  if (vers == MachineListMessage::MACHINE_LIST_MESSAGE_VERSION) {
-    int n                   = 0;
-    MachineListMessage *msg = &mlistmsg;
-
-    while (n < cc->n_machines) {
-      msg->ip[n] = cc->machines[n]->ip;
-      n++;
-    }
-    msg->n_ip = n;
-    data      = (void *)msg;
-    len       = msg->sizeof_fixedlen_msg() + (n * sizeof(uint32_t));
-
-  } else {
-    //////////////////////////////////////////////////////////////
-    // Create the specified down rev version of this message
-    //////////////////////////////////////////////////////////////
-    ink_release_assert(!"send_machine_list() bad msg version");
-  }
-  invoke_remote(m->pop_ClusterHandler(), MACHINE_LIST_CLUSTER_FUNCTION, data, len);
-}
-
-void
-ClusterProcessor::compute_cluster_mode()
-{
-  if (RPC_only_CacheCluster) {
-    if (cache_clustering_enabled > 0) {
-      cache_clustering_enabled = -1;
-      Note("RPC only cache clustering");
-    }
-  } else {
-    if (cache_clustering_enabled < 0) {
-      cache_clustering_enabled = 1;
-      Note("RPC only cache clustering disabled");
-    }
-  }
-}
-
-// End of ClusterProcessor.cc
diff --git a/iocore/cluster/ClusterRPC.cc b/iocore/cluster/ClusterRPC.cc
deleted file mode 100644
index 016c91c..0000000
--- a/iocore/cluster/ClusterRPC.cc
+++ /dev/null
@@ -1,375 +0,0 @@
-/** @file
-
-  A brief file description
-
-  @section license License
-
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
- */
-
-/****************************************************************************
-
-  ClusterRPC.cc
-****************************************************************************/
-
-#include "P_Cluster.h"
-/////////////////////////////////////////////////////////////////////////
-// All RPC function handlers (xxx_ClusterFunction() ) are invoked from
-// ClusterHandler::update_channels_read().
-/////////////////////////////////////////////////////////////////////////
-
-void
-ping_ClusterFunction(ClusterHandler *ch, void *data, int len)
-{
-  //
-  // Just return the data back
-  //
-  clusterProcessor.invoke_remote(ch, PING_REPLY_CLUSTER_FUNCTION, data, len);
-}
-
-void
-ping_reply_ClusterFunction(ClusterHandler *ch, void *data, int len)
-{
-  //
-  // Pass back the data.
-  //
-  PingMessage *msg = (PingMessage *)data;
-  msg->fn(ch, msg->data, (len - msg->sizeof_fixedlen_msg()));
-}
-
-void
-machine_list_ClusterFunction(ClusterHandler *from, void *data, int len)
-{
-  (void)from;
-  ClusterMessageHeader *mh = (ClusterMessageHeader *)data;
-  MachineListMessage *m    = (MachineListMessage *)data;
-
-  if (mh->GetMsgVersion() != MachineListMessage::MACHINE_LIST_MESSAGE_VERSION) { ////////////////////////////////////////////////
-    // Convert from old to current message format
-    ////////////////////////////////////////////////
-    ink_release_assert(!"machine_list_ClusterFunction() bad msg version");
-  }
-  if (m->NeedByteSwap())
-    m->SwapBytes();
-
-  ink_assert(m->n_ip == ((len - m->sizeof_fixedlen_msg()) / sizeof(uint32_t)));
-
-  //
-  // The machine list is a vector of ip's stored in network byte order.
-  // This list is exchanged whenever a new Cluster Connection is formed.
-  //
-  ClusterConfiguration *cc = this_cluster()->current_configuration();
-
-  for (unsigned int i = 0; i < m->n_ip; i++) {
-    for (int j = 0; j < cc->n_machines; j++) {
-      if (cc->machines[j]->ip == m->ip[i])
-        goto Lfound;
-    }
-    // not found, must be a new machine
-    {
-      int num_connections = this_cluster_machine()->num_connections;
-      for (int k = 0; k < num_connections; k++) {
-        clusterProcessor.connect(m->ip[i], k);
-      }
-    }
-  Lfound:;
-  }
-}
-
-void
-close_channel_ClusterFunction(ClusterHandler *ch, void *data, int len)
-{
-  ClusterMessageHeader *mh = (ClusterMessageHeader *)data;
-  CloseMessage *m          = (CloseMessage *)data;
-
-  if (mh->GetMsgVersion() != CloseMessage::CLOSE_CHAN_MESSAGE_VERSION) { ////////////////////////////////////////////////
-    // Convert from old to current message format
-    ////////////////////////////////////////////////
-    ink_release_assert(!"close_channel_ClusterFunction() bad msg version");
-  }
-  if (m->NeedByteSwap())
-    m->SwapBytes();
-
-  //
-  // Close the remote side of a VC connection (remote node is originator)
-  //
-  ink_assert(len >= (int)sizeof(CloseMessage));
-  if (!ch || !ch->channels)
-    return;
-  ClusterVConnection *vc = ch->channels[m->channel];
-  if (VALID_CHANNEL(vc) && vc->token.sequence_number == m->sequence_number) {
-    vc->remote_closed = m->status;
-    vc->remote_lerrno = m->lerrno;
-    ch->vcs_push(vc, vc->type);
-  }
-}
-
-void
-test_ClusterFunction(ClusterHandler *ch, void *data, int len)
-{
-  //
-  // Note: Only for testing.
-  //
-  if (ptest_ClusterFunction)
-    ptest_ClusterFunction(ch, data, len);
-}
-
-CacheVC *
-ChannelToCacheWriteVC(ClusterHandler *ch, int channel, uint32_t channel_seqno, ClusterVConnection **cluster_vc)
-{
-  EThread *thread   = this_ethread();
-  ProxyMutex *mutex = thread->mutex.get();
-
-  ClusterVConnection *cvc = ch->channels[channel];
-  if (!VALID_CHANNEL(cvc) || (channel_seqno != cvc->token.sequence_number) || (cvc->read.vio.op != VIO::READ)) {
-    CLUSTER_INCREMENT_DYN_STAT(cluster_setdata_no_CLUSTERVC_STAT);
-    return nullptr;
-  }
-  // Tunneling from cluster to cache (remote write).
-  // Get cache VC pointer.
-
-  OneWayTunnel *owt = (OneWayTunnel *)cvc->read.vio._cont;
-  if (!owt) {
-    CLUSTER_INCREMENT_DYN_STAT(CLUSTER_SETDATA_NO_TUNNEL_STAT);
-    return nullptr;
-  }
-  CacheVC *cache_vc = (CacheVC *)owt->vioTarget->vc_server;
-  if (!cache_vc) {
-    CLUSTER_INCREMENT_DYN_STAT(CLUSTER_SETDATA_NO_CACHEVC_STAT);
-    return nullptr;
-  }
-  *cluster_vc = cvc;
-  return cache_vc;
-}
-
-void
-set_channel_data_ClusterFunction(ClusterHandler *ch, void *tdata, int tlen)
-{
-  EThread *thread   = this_ethread();
-  ProxyMutex *mutex = thread->mutex.get();
-  // We are called on the ET_CLUSTER thread.
-
-  char *data;
-  int len;
-  int res;
-
-  // Allocate memory for set channel data and pass it to the cache
-  IncomingControl *ic = IncomingControl::alloc();
-  ic->len             = tlen;
-  ic->alloc_data();
-
-  data = ic->data + sizeof(int32_t); // free_remote_data expects d+sizeof(int32_t)
-  memcpy(data, tdata, tlen);
-  len = tlen;
-
-  ClusterMessageHeader *mh = (ClusterMessageHeader *)data;
-  SetChanDataMessage *m    = (SetChanDataMessage *)data;
-
-  if (mh->GetMsgVersion() !=
-      SetChanDataMessage::SET_CHANNEL_DATA_MESSAGE_VERSION) { ////////////////////////////////////////////////
-    // Convert from old to current message format
-    ////////////////////////////////////////////////
-    ink_release_assert(!"set_channel_data_ClusterFunction() bad msg version");
-  }
-  if (m->NeedByteSwap())
-    m->SwapBytes();
-
-  ClusterVConnection *cvc;
-  CacheVC *cache_vc;
-
-  if (ch) {
-    cache_vc = ChannelToCacheWriteVC(ch, m->channel, m->sequence_number, &cvc);
-    if (!cache_vc) {
-      ic->freeall();
-      return;
-    }
-    // Unmarshal data.
-    switch (m->data_type) {
-    case CACHE_DATA_HTTP_INFO: {
-      char *p = (char *)m + SetChanDataMessage::sizeof_fixedlen_msg();
-
-      IOBufferBlock *block_ref = ic->get_block();
-      res                      = HTTPInfo::unmarshal(p, len, block_ref);
-      ink_assert(res > 0);
-
-      CacheHTTPInfo h;
-      h.get_handle((char *)&m->data[0], len);
-      h.set_buffer_reference(block_ref);
-      cache_vc->set_http_info(&h);
-      ic->freeall();
-      break;
-    }
-    default: {
-      ink_release_assert(!"set_channel_data_ClusterFunction bad CacheDataType");
-    }
-    }                            // End of switch
-    ++cvc->n_recv_set_data_msgs; // note received messages
-
-  } else {
-    ic->freeall();
-    CLUSTER_INCREMENT_DYN_STAT(cluster_setdata_no_CLUSTER_STAT);
-  }
-}
-
-void
-post_setchan_send_ClusterFunction(ClusterHandler *ch, void *data, int /* len ATS_UNUSED */)
-{
-  EThread *thread   = this_ethread();
-  ProxyMutex *mutex = thread->mutex.get();
-  // We are called on the ET_CLUSTER thread.
-  // set_data() control message has been queued into cluster transfer message.
-  // This allows us to assume that it has been sent.
-  // Decrement Cluster VC n_set_data_msgs to allow transmission of
-  // initial open_write data after (n_set_data_msgs == 0).
-
-  SetChanDataMessage *m = (SetChanDataMessage *)data;
-  ClusterVConnection *cvc;
-
-  if (ch) {
-    cvc = ch->channels[m->channel];
-    if (VALID_CHANNEL(cvc)) {
-      ink_atomic_increment(&cvc->n_set_data_msgs, -1);
-    } else {
-      CLUSTER_INCREMENT_DYN_STAT(cluster_setdata_no_CLUSTERVC_STAT);
-    }
-  } else {
-    CLUSTER_INCREMENT_DYN_STAT(cluster_setdata_no_CLUSTER_STAT);
-  }
-}
-
-void
-set_channel_pin_ClusterFunction(ClusterHandler *ch, void *data, int /* len ATS_UNUSED */)
-{
-  // This isn't used. /leif
-  // EThread *thread = this_ethread();
-  // ProxyMutex *mutex = thread->mutex;
-
-  // We are called on the ET_CLUSTER thread.
-
-  ClusterMessageHeader *mh = (ClusterMessageHeader *)data;
-  SetChanPinMessage *m     = (SetChanPinMessage *)data;
-
-  if (mh->GetMsgVersion() != SetChanPinMessage::SET_CHANNEL_PIN_MESSAGE_VERSION) { ////////////////////////////////////////////////
-    // Convert from old to current message format
-    ////////////////////////////////////////////////
-    ink_release_assert(!"set_channel_pin_ClusterFunction() bad msg version");
-  }
-
-  if (m->NeedByteSwap())
-    m->SwapBytes();
-
-  ClusterVConnection *cvc = nullptr; // Just to make GCC happy
-  CacheVC *cache_vc;
-
-  if (ch != nullptr) {
-    cache_vc = ChannelToCacheWriteVC(ch, m->channel, m->sequence_number, &cvc);
-    if (cache_vc) {
-      cache_vc->set_pin_in_cache(m->pin_time);
-    }
-    // cvc is always set in ChannelToCacheWriteVC, so need to check it
-    ++cvc->n_recv_set_data_msgs; // note received messages
-  }
-}
-
-void
-post_setchan_pin_ClusterFunction(ClusterHandler *ch, void *data, int /* len ATS_UNUSED */)
-{
-  EThread *thread   = this_ethread();
-  ProxyMutex *mutex = thread->mutex.get();
-  // We are called on the ET_CLUSTER thread.
-  // Control message has been queued into cluster transfer message.
-  // This allows us to assume that it has been sent.
-  // Decrement Cluster VC n_set_data_msgs to allow transmission of
-  // initial open_write data after (n_set_data_msgs == 0).
-
-  SetChanPinMessage *m = (SetChanPinMessage *)data;
-  ClusterVConnection *cvc;
-
-  if (ch) {
-    cvc = ch->channels[m->channel];
-    if (VALID_CHANNEL(cvc)) {
-      ink_atomic_increment(&cvc->n_set_data_msgs, -1);
-    } else {
-      CLUSTER_INCREMENT_DYN_STAT(cluster_setdata_no_CLUSTERVC_STAT);
-    }
-  } else {
-    CLUSTER_INCREMENT_DYN_STAT(cluster_setdata_no_CLUSTER_STAT);
-  }
-}
-
-void
-set_channel_priority_ClusterFunction(ClusterHandler *ch, void *data, int /* len ATS_UNUSED */)
-{
-  // This isn't used.
-  // EThread *thread = this_ethread();
-  // ProxyMutex *mutex = thread->mutex;
-
-  // We are called on the ET_CLUSTER thread.
-
-  ClusterMessageHeader *mh  = (ClusterMessageHeader *)data;
-  SetChanPriorityMessage *m = (SetChanPriorityMessage *)data;
-
-  if (mh->GetMsgVersion() !=
-      SetChanPriorityMessage::SET_CHANNEL_PRIORITY_MESSAGE_VERSION) { ////////////////////////////////////////////////
-    // Convert from old to current message format
-    ////////////////////////////////////////////////
-    ink_release_assert(!"set_channel_priority_ClusterFunction() bad msg version");
-  }
-  if (m->NeedByteSwap())
-    m->SwapBytes();
-
-  ClusterVConnection *cvc = nullptr; // Just to make GCC happy
-  CacheVC *cache_vc;
-
-  if (ch != nullptr) {
-    cache_vc = ChannelToCacheWriteVC(ch, m->channel, m->sequence_number, &cvc);
-    if (cache_vc) {
-      cache_vc->set_disk_io_priority(m->disk_priority);
-    }
-    // cvc is always set in ChannelToCacheWriteVC, so need to check it
-    ++cvc->n_recv_set_data_msgs; // note received messages
-  }
-}
-
-void
-post_setchan_priority_ClusterFunction(ClusterHandler *ch, void *data, int /* len ATS_UNUSED */)
-{
-  EThread *thread   = this_ethread();
-  ProxyMutex *mutex = thread->mutex.get();
-
-  // We are called on the ET_CLUSTER thread.
-  // Control message has been queued into cluster transfer message.
-  // This allows us to assume that it has been sent.
-  // Decrement Cluster VC n_set_data_msgs to allow transmission of
-  // initial open_write data after (n_set_data_msgs == 0).
-
-  SetChanPriorityMessage *m = (SetChanPriorityMessage *)data;
-  ClusterVConnection *cvc;
-
-  if (ch) {
-    cvc = ch->channels[m->channel];
-    if (VALID_CHANNEL(cvc)) {
-      ink_atomic_increment(&cvc->n_set_data_msgs, -1);
-    } else {
-      CLUSTER_INCREMENT_DYN_STAT(cluster_setdata_no_CLUSTERVC_STAT);
-    }
-  } else {
-    CLUSTER_INCREMENT_DYN_STAT(cluster_setdata_no_CLUSTER_STAT);
-  }
-}
-
-// End of ClusterRPC.cc
diff --git a/iocore/cluster/ClusterVConnection.cc b/iocore/cluster/ClusterVConnection.cc
deleted file mode 100644
index f15a584..0000000
--- a/iocore/cluster/ClusterVConnection.cc
+++ /dev/null
@@ -1,655 +0,0 @@
-/** @file
-
-  A brief file description
-
-  @section license License
-
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
- */
-
-/****************************************************************************
-
-  ClusterVConnection.cc
-****************************************************************************/
-
-#include "P_Cluster.h"
-ClassAllocator<ClusterVConnection> clusterVCAllocator("clusterVCAllocator");
-ClassAllocator<ByteBankDescriptor> byteBankAllocator("byteBankAllocator");
-
-ByteBankDescriptor *
-ByteBankDescriptor::ByteBankDescriptor_alloc(Ptr<IOBufferBlock> &iob)
-{
-  ByteBankDescriptor *b = byteBankAllocator.alloc();
-  b->block              = iob;
-  return b;
-}
-
-void
-ByteBankDescriptor::ByteBankDescriptor_free(ByteBankDescriptor *b)
-{
-  b->block = nullptr;
-  byteBankAllocator.free(b);
-}
-
-void
-clusterVCAllocator_free(ClusterVConnection *vc)
-{
-  vc->mutex   = nullptr;
-  vc->action_ = nullptr;
-  vc->free();
-  if (vc->in_vcs) {
-    vc->type = VC_CLUSTER_CLOSED;
-    return;
-  }
-  clusterVCAllocator.free(vc);
-}
-
-ClusterVConnState::ClusterVConnState() : enabled(0), priority(1), vio(VIO::NONE), queue(nullptr), ifd(-1), delay_timeout(nullptr)
-{
-}
-
-ClusterVConnectionBase::ClusterVConnectionBase()
-  : thread(nullptr), closed(0), inactivity_timeout_in(0), active_timeout_in(0), inactivity_timeout(nullptr), active_timeout(nullptr)
-{
-}
-
-#ifdef DEBUG
-int ClusterVConnectionBase::enable_debug_trace = 0;
-#endif
-
-VIO *
-ClusterVConnectionBase::do_io_read(Continuation *acont, int64_t anbytes, MIOBuffer *abuffer)
-{
-  ink_assert(!closed);
-  read.vio.buffer.writer_for(abuffer);
-  read.vio.op = VIO::READ;
-  read.vio.set_continuation(acont);
-  read.vio.nbytes    = anbytes;
-  read.vio.ndone     = 0;
-  read.vio.vc_server = (VConnection *)this;
-  read.enabled       = 1;
-
-  ClusterVConnection *cvc = (ClusterVConnection *)this;
-  Debug("cluster_vc_xfer", "do_io_read [%s] chan %d", "", cvc->channel);
-  return &read.vio;
-}
-
-VIO *
-ClusterVConnectionBase::do_io_pread(Continuation * /* acont ATS_UNUSED */, int64_t /* anbytes ATS_UNUSED */,
-                                    MIOBuffer * /* abuffer ATS_UNUSED */, int64_t /* off ATS_UNUSED */)
-{
-  return nullptr;
-}
-
-int
-ClusterVConnection::get_header(void ** /* ptr ATS_UNUSED */, int * /*len ATS_UNUSED */)
-{
-  ink_assert(!"implemented");
-  return -1;
-}
-
-int
-ClusterVConnection::set_header(void * /* ptr ATS_UNUSED */, int /* len ATS_UNUSED */)
-{
-  ink_assert(!"implemented");
-  return -1;
-}
-
-int
-ClusterVConnection::get_single_data(void ** /* ptr ATS_UNUSED */, int * /* len ATS_UNUSED */)
-{
-  ink_assert(!"implemented");
-  return -1;
-}
-
-VIO *
-ClusterVConnectionBase::do_io_write(Continuation *acont, int64_t anbytes, IOBufferReader *abuffer, bool owner)
-{
-  ink_assert(!closed);
-  ink_assert(!owner);
-  write.vio.buffer.reader_for(abuffer);
-  write.vio.op = VIO::WRITE;
-  write.vio.set_continuation(acont);
-  write.vio.nbytes    = anbytes;
-  write.vio.ndone     = 0;
-  write.vio.vc_server = (VConnection *)this;
-  write.enabled       = 1;
-
-  return &write.vio;
-}
-
-void
-ClusterVConnectionBase::do_io_close(int alerrno)
-{
-  read.enabled  = 0;
-  write.enabled = 0;
-  read.vio.buffer.clear();
-  write.vio.buffer.clear();
-  INK_WRITE_MEMORY_BARRIER;
-  if (alerrno && alerrno != -1)
-    this->lerrno = alerrno;
-
-  if (alerrno == -1) {
-    closed = 1;
-  } else {
-    closed = -1;
-  }
-}
-
-void
-ClusterVConnection::reenable(VIO *vio)
-{
-  if (type == VC_CLUSTER_WRITE)
-    ch->vcs_push(this, VC_CLUSTER_WRITE);
-
-  ClusterVConnectionBase::reenable(vio);
-}
-
-void
-ClusterVConnectionBase::reenable(VIO *vio)
-{
-  ink_assert(!closed);
-  if (vio == &read.vio) {
-    read.enabled = 1;
-#ifdef DEBUG
-    if (enable_debug_trace && (vio->buffer.writer() && !vio->buffer.writer()->write_avail()))
-      printf("NetVConnection re-enabled for read when full\n");
-#endif
-  } else if (vio == &write.vio) {
-    write.enabled = 1;
-#ifdef DEBUG
-    if (enable_debug_trace && (vio->buffer.writer() && !vio->buffer.reader()->read_avail()))
-      printf("NetVConnection re-enabled for write when empty\n");
-#endif
-  } else {
-    ink_assert(!"bad vio");
-  }
-}
-
-void
-ClusterVConnectionBase::reenable_re(VIO *vio)
-{
-  reenable(vio);
-}
-
-ClusterVConnection::ClusterVConnection(int is_new_connect_read)
-  : ch(nullptr),
-    new_connect_read(is_new_connect_read),
-    remote_free(0),
-    last_local_free(0),
-    channel(0),
-    close_disabled(0),
-    remote_closed(0),
-    remote_close_disabled(1),
-    remote_lerrno(0),
-    in_vcs(0),
-    type(0),
-    start_time(0),
-    last_activity_time(0),
-    n_set_data_msgs(0),
-    n_recv_set_data_msgs(0),
-    pending_remote_fill(0),
-    remote_ram_cache_hit(false),
-    have_all_data(false),
-    initial_data_bytes(0),
-    current_cont(nullptr),
-    iov_map(CLUSTER_IOV_NOT_OPEN),
-    write_list_tail(nullptr),
-    write_list_bytes(0),
-    write_bytes_in_transit(0),
-    alternate(),
-    time_pin(0),
-    disk_io_priority(0)
-{
-#ifdef DEBUG
-  read.vio.buffer.name  = "ClusterVConnection.read";
-  write.vio.buffer.name = "ClusterVConnection.write";
-#endif
-  SET_HANDLER((ClusterVConnHandler)&ClusterVConnection::startEvent);
-}
-
-ClusterVConnection::~ClusterVConnection()
-{
-  free();
-}
-
-void
-ClusterVConnection::free()
-{
-  if (alternate.valid()) {
-    alternate.destroy();
-  }
-  ByteBankDescriptor *d;
-  while ((d = byte_bank_q.dequeue())) {
-    ByteBankDescriptor::ByteBankDescriptor_free(d);
-  }
-  read_block             = nullptr;
-  remote_write_block     = nullptr;
-  marshal_buf            = nullptr;
-  write_list             = nullptr;
-  write_list_tail        = nullptr;
-  write_list_bytes       = 0;
-  write_bytes_in_transit = 0;
-}
-
-VIO *
-ClusterVConnection::do_io_read(Continuation *c, int64_t nbytes, MIOBuffer *buf)
-{
-  if (type == VC_CLUSTER)
-    type = VC_CLUSTER_READ;
-  ch->vcs_push(this, VC_CLUSTER_READ);
-
-  return ClusterVConnectionBase::do_io_read(c, nbytes, buf);
-}
-
-VIO *
-ClusterVConnection::do_io_write(Continuation *c, int64_t nbytes, IOBufferReader *buf, bool owner)
-{
-  if (type == VC_CLUSTER)
-    type = VC_CLUSTER_WRITE;
-  ch->vcs_push(this, VC_CLUSTER_WRITE);
-
-  return ClusterVConnectionBase::do_io_write(c, nbytes, buf, owner);
-}
-
-void
-ClusterVConnection::do_io_close(int alerrno)
-{
-  if ((type == VC_CLUSTER) && current_cont) {
-    if (((CacheContinuation *)current_cont)->read_cluster_vc == this)
-      type = VC_CLUSTER_READ;
-    else if (((CacheContinuation *)current_cont)->write_cluster_vc == this)
-      type = VC_CLUSTER_WRITE;
-  }
-  ch->vcs_push(this, type);
-
-  ClusterVConnectionBase::do_io_close(alerrno);
-}
-
-int
-ClusterVConnection::startEvent(int event, Event *e)
-{
-  //
-  // Safe to call with e == nullptr from the same thread.
-  //
-  (void)event;
-  start(e ? e->ethread : (EThread *)nullptr);
-  return EVENT_DONE;
-}
-
-int
-ClusterVConnection::mainEvent(int event, Event *e)
-{
-  (void)event;
-  (void)e;
-  ink_assert(!"unexpected event");
-  return EVENT_DONE;
-}
-
-int
-ClusterVConnection::start(EThread *t)
-{
-  //
-  //  New channel connect protocol.  Establish VC locally and send the
-  //  channel id to the target.  Reverse of existing connect protocol
-  //
-  //////////////////////////////////////////////////////////////////////////
-  // In the new VC connect protocol, we always establish the local side
-  // of the connection followed by the remote side.
-  //
-  // Read connection notes:
-  // ----------------------
-  // The response message now consists of the standard reply message
-  // along with a portion of the  object data.  This data is always
-  // transferred in the same Cluster transfer message as channel data.
-  // In order to transfer data into a partially connected VC, we introduced
-  // a VC "pending_remote_fill" state allowing us to move the initial data
-  // using the existing user channel mechanism.
-  // Initially, both sides of the connection set "pending_remote_fill".
-  //
-  // "pending_remote_fill" allows us to make the following assumptions.
-  //   1) No free space messages are sent for VC(s) in this state.
-  //   2) Writer side, the initial write data is described by
-  //      vc->remote_write_block NOT by vc->write.vio.buffer, since
-  //      vc->write.vio is reserved for use in the OneWayTunnel.
-  //      OneWayTunnel is used when all the object data cannot be
-  //      contained in the initial send buffer.
-  //   3) Writer side, write vio mutex not acquired for initial data write.
-  ///////////////////////////////////////////////////////////////////////////
-
-  int status;
-  if (!channel) {
-#ifdef CLUSTER_TOMCAT
-    Ptr<ProxyMutex> m = action_.mutex;
-    if (!m) {
-      m = new_ProxyMutex();
-    }
-#else
-    Ptr<ProxyMutex> m = action_.mutex;
-#endif
-
-    // Establish the local side of the VC connection
-    MUTEX_TRY_LOCK(lock, m, t);
-    if (!lock.is_locked()) {
-      t->schedule_in(this, CLUSTER_CONNECT_RETRY);
-      return EVENT_DONE;
-    }
-    if (!ch) {
-      if (action_.continuation) {
-        action_.continuation->handleEvent(CLUSTER_EVENT_OPEN_FAILED, (void *)-ECLUSTER_NO_MACHINE);
-        clusterVCAllocator_free(this);
-        return EVENT_DONE;
-      } else {
-        // if we have been invoked immediately
-        clusterVCAllocator_free(this);
-        return -1;
-      }
-    }
-
-    channel = ch->alloc_channel(this);
-    if (channel < 0) {
-      if (action_.continuation) {
-        action_.continuation->handleEvent(CLUSTER_EVENT_OPEN_FAILED, (void *)-ECLUSTER_NOMORE_CHANNELS);
-        clusterVCAllocator_free(this);
-        return EVENT_DONE;
-      } else {
-        // if we have been invoked immediately
-        clusterVCAllocator_free(this);
-        return -1;
-      }
-
-    } else {
-      Debug(CL_TRACE, "VC start alloc local chan=%d VC=%p", channel, this);
-      if (new_connect_read)
-        this->pending_remote_fill = 1;
-    }
-
-  } else {
-    // Establish the remote side of the VC connection
-    if ((status = ch->alloc_channel(this, channel)) < 0) {
-      Debug(CL_TRACE, "VC start alloc remote failed chan=%d VC=%p", channel, this);
-      clusterVCAllocator_free(this);
-      return status; // Channel active or no more channels
-    } else {
-      Debug(CL_TRACE, "VC start alloc remote chan=%d VC=%p", channel, this);
-      if (new_connect_read)
-        this->pending_remote_fill = 1;
-      this->iov_map               = CLUSTER_IOV_NONE; // disable connect timeout
-    }
-  }
-  cluster_schedule(ch, this, &read);
-  cluster_schedule(ch, this, &write);
-  if (action_.continuation) {
-    action_.continuation->handleEvent(CLUSTER_EVENT_OPEN, this);
-  }
-  mutex = nullptr;
-  return EVENT_DONE;
-}
-
-int
-ClusterVConnection::was_closed()
-{
-  return (closed && !close_disabled);
-}
-
-void
-ClusterVConnection::allow_close()
-{
-  close_disabled = 0;
-}
-
-void
-ClusterVConnection::disable_close()
-{
-  close_disabled = 1;
-}
-
-int
-ClusterVConnection::was_remote_closed()
-{
-  if (!byte_bank_q.head && !remote_close_disabled)
-    return remote_closed;
-  else
-    return 0;
-}
-
-void
-ClusterVConnection::allow_remote_close()
-{
-  remote_close_disabled = 0;
-}
-
-bool
-ClusterVConnection::schedule_write()
-{
-  //
-  // Schedule write if we have all data or current write data is
-  // at least DEFAULT_MAX_BUFFER_SIZE.
-  //
-  if (write_list) {
-    if ((closed < 0) || remote_closed) {
-      // User aborted connection, dump data.
-
-      write_list       = nullptr;
-      write_list_tail  = nullptr;
-      write_list_bytes = 0;
-
-      return false;
-    }
-
-    if (closed || (write_list_bytes >= DEFAULT_MAX_BUFFER_SIZE)) {
-      // No more data to write or buffer list is full, start write
-      return true;
-    } else {
-      // Buffer list is not full, defer write
-      return false;
-    }
-  } else {
-    return false;
-  }
-}
-
-void
-ClusterVConnection::set_type(int options)
-{
-  new_connect_read = (options & CLUSTER_OPT_CONN_READ) ? 1 : 0;
-  if (new_connect_read) {
-    pending_remote_fill = 1;
-  } else {
-    pending_remote_fill = 0;
-  }
-}
-
-// Overide functions in base class VConnection.
-bool
-ClusterVConnection::get_data(int id, void * /* data ATS_UNUSED */)
-{
-  switch (id) {
-  case CACHE_DATA_HTTP_INFO: {
-    ink_release_assert(!"ClusterVConnection::get_data CACHE_DATA_HTTP_INFO not supported");
-  }
-  case CACHE_DATA_KEY: {
-    ink_release_assert(!"ClusterVConnection::get_data CACHE_DATA_KEY not supported");
-  }
-  default: {
-    ink_release_assert(!"ClusterVConnection::get_data invalid id");
-  }
-  }
-  return false;
-}
-
-void
-ClusterVConnection::get_http_info(CacheHTTPInfo **info)
-{
-  *info = &alternate;
-}
-
-int64_t
-ClusterVConnection::get_object_size()
-{
-  return alternate.object_size_get();
-}
-
-bool
-ClusterVConnection::is_pread_capable()
-{
-  return false;
-}
-
-void
-ClusterVConnection::set_http_info(CacheHTTPInfo *d)
-{
-  int flen, len;
-  void *data;
-  int res;
-  SetChanDataMessage *m;
-  SetChanDataMessage msg;
-
-  //
-  // set_http_info() is a mechanism to associate additional data with a
-  // open_write() ClusterVConnection.  It is only allowed after a
-  // successful open_write() and prior to issuing the do_io(VIO::WRITE).
-  // Cache semantics dictate that set_http_info() be established prior
-  // to transferring any data on the ClusterVConnection.
-  //
-  ink_release_assert(this->write.vio.op == VIO::NONE); // not true if do_io()
-  //   already done
-  ink_release_assert(this->read.vio.op == VIO::NONE); // should always be true
-
-  int vers = SetChanDataMessage::protoToVersion(ch->machine->msg_proto_major);
-  if (vers == SetChanDataMessage::SET_CHANNEL_DATA_MESSAGE_VERSION) {
-    flen = SetChanDataMessage::sizeof_fixedlen_msg();
-  } else {
-    //////////////////////////////////////////////////////////////
-    // Create the specified down rev version of this message
-    //////////////////////////////////////////////////////////////
-    ink_release_assert(!"ClusterVConnection::set_http_info() bad msg version");
-  }
-
-  // Create message and marshal data.
-
-  CacheHTTPInfo *r = d;
-  len              = r->marshal_length();
-  data             = (void *)ALLOCA_DOUBLE(flen + len);
-  memcpy((char *)data, (char *)&msg, sizeof(msg));
-  m            = (SetChanDataMessage *)data;
-  m->data_type = CACHE_DATA_HTTP_INFO;
-
-  char *p = (char *)m + flen;
-  res     = r->marshal(p, len);
-  if (res < 0) {
-    r->destroy();
-    return;
-  }
-  r->destroy();
-
-  m->channel         = channel;
-  m->sequence_number = token.sequence_number;
-
-  // note pending set_data() msgs on VC.
-  ink_atomic_increment(&n_set_data_msgs, 1);
-
-  clusterProcessor.invoke_remote(ch, SET_CHANNEL_DATA_CLUSTER_FUNCTION, data, flen + len);
-}
-
-bool
-ClusterVConnection::set_pin_in_cache(time_t t)
-{
-  SetChanPinMessage msg;
-
-  //
-  // set_pin_in_cache() is a mechanism to set an attribute on a
-  // open_write() ClusterVConnection.  It is only allowed after a
-  // successful open_write() and prior to issuing the do_io(VIO::WRITE).
-  //
-  ink_release_assert(this->write.vio.op == VIO::NONE); // not true if do_io()
-  //   already done
-  ink_release_assert(this->read.vio.op == VIO::NONE); // should always be true
-  time_pin = t;
-
-  int vers = SetChanPinMessage::protoToVersion(ch->machine->msg_proto_major);
-
-  if (vers == SetChanPinMessage::SET_CHANNEL_PIN_MESSAGE_VERSION) {
-    SetChanPinMessage::sizeof_fixedlen_msg();
-  } else {
-    //////////////////////////////////////////////////////////////
-    // Create the specified down rev version of this message
-    //////////////////////////////////////////////////////////////
-    ink_release_assert(!"ClusterVConnection::set_pin_in_cache() bad msg "
-                        "version");
-  }
-  msg.channel         = channel;
-  msg.sequence_number = token.sequence_number;
-  msg.pin_time        = time_pin;
-
-  // note pending set_data() msgs on VC.
-  ink_atomic_increment(&n_set_data_msgs, 1);
-
-  clusterProcessor.invoke_remote(ch, SET_CHANNEL_PIN_CLUSTER_FUNCTION, (char *)&msg, sizeof(msg));
-  return true;
-}
-
-time_t
-ClusterVConnection::get_pin_in_cache()
-{
-  return time_pin;
-}
-
-bool
-ClusterVConnection::set_disk_io_priority(int priority)
-{
-  SetChanPriorityMessage msg;
-
-  //
-  // set_disk_io_priority() is a mechanism to set an attribute on a
-  // open_write() ClusterVConnection.  It is only allowed after a
-  // successful open_write() and prior to issuing the do_io(VIO::WRITE).
-  //
-  ink_release_assert(this->write.vio.op == VIO::NONE); // not true if do_io()
-  //   already done
-  ink_release_assert(this->read.vio.op == VIO::NONE); // should always be true
-  disk_io_priority = priority;
-
-  int vers = SetChanPriorityMessage::protoToVersion(ch->machine->msg_proto_major);
-
-  if (vers == SetChanPriorityMessage::SET_CHANNEL_PRIORITY_MESSAGE_VERSION) {
-    SetChanPriorityMessage::sizeof_fixedlen_msg();
-  } else {
-    //////////////////////////////////////////////////////////////
-    // Create the specified down rev version of this message
-    //////////////////////////////////////////////////////////////
-    ink_release_assert(!"ClusterVConnection::set_disk_io_priority() bad msg "
-                        "version");
-  }
-  msg.channel         = channel;
-  msg.sequence_number = token.sequence_number;
-  msg.disk_priority   = priority;
-
-  // note pending set_data() msgs on VC.
-  ink_atomic_increment(&n_set_data_msgs, 1);
-
-  clusterProcessor.invoke_remote(ch, SET_CHANNEL_PRIORITY_CLUSTER_FUNCTION, (char *)&msg, sizeof(msg));
-  return true;
-}
-
-int
-ClusterVConnection::get_disk_io_priority()
-{
-  return disk_io_priority;
-}
-
-// End of ClusterVConnection.cc
diff --git a/iocore/cluster/Inline.cc b/iocore/cluster/Inline.cc
deleted file mode 100644
index 63ce713..0000000
--- a/iocore/cluster/Inline.cc
+++ /dev/null
@@ -1,30 +0,0 @@
-/** @file
-
-  A brief file description
-
-  @section license License
-
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
- */
-
-/*
- * Inline Functions as globals for users using the public interface
- *
- */
-
-#define TS_INLINE
-#include "P_Cluster.h"
diff --git a/iocore/cluster/Makefile.am b/iocore/cluster/Makefile.am
deleted file mode 100644
index 6ca245c..0000000
--- a/iocore/cluster/Makefile.am
+++ /dev/null
@@ -1,63 +0,0 @@
-# Makefile.am for the traffic/iocore/cluster hierarchy
-#
-#  Licensed to the Apache Software Foundation (ASF) under one
-#  or more contributor license agreements.  See the NOTICE file
-#  distributed with this work for additional information
-#  regarding copyright ownership.  The ASF licenses this file
-#  to you under the Apache License, Version 2.0 (the
-#  "License"); you may not use this file except in compliance
-#  with the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-AM_CPPFLAGS += \
-  $(iocore_include_dirs) \
-  -I$(abs_top_srcdir)/lib \
-  -I$(abs_top_srcdir)/lib/records \
-  -I$(abs_top_srcdir)/proxy/http \
-  -I$(abs_top_srcdir)/proxy/hdrs \
-  -I$(abs_top_srcdir)/proxy \
-  -I$(abs_top_srcdir)/mgmt \
-  -I$(abs_top_srcdir)/mgmt/utils
-
-noinst_LIBRARIES = libinkcluster.a
-
-libinkcluster_a_SOURCES = \
-  ClusterAPI.cc \
-  ClusterCache.cc \
-  ClusterConfig.cc \
-  ClusterHandler.cc \
-  ClusterHandlerBase.cc \
-  ClusterHash.cc \
-  ClusterLib.cc \
-  ClusterLoadMonitor.cc \
-  ClusterMachine.cc \
-  ClusterProcessor.cc \
-  ClusterRPC.cc \
-  ClusterVConnection.cc \
-  Inline.cc \
-  P_Cluster.h \
-  P_ClusterCache.h \
-  P_ClusterCacheInternal.h \
-  P_ClusterHandler.h \
-  P_ClusterInline.h \
-  P_ClusterInternal.h \
-  P_ClusterLib.h \
-  P_ClusterLoadMonitor.h \
-  P_ClusterMachine.h \
-  P_TimeTrace.h
-
-#test_Cluster_SOURCES = \
-#  test_I_Cluster.cc \
-#  test_P_Cluster.cc
-
-include $(top_srcdir)/build/tidy.mk
-
-tidy-local: $(DIST_SOURCES)
-	$(CXX_Clang_Tidy)
diff --git a/iocore/cluster/P_Cluster.h b/iocore/cluster/P_Cluster.h
deleted file mode 100644
index 4c64344..0000000
--- a/iocore/cluster/P_Cluster.h
+++ /dev/null
@@ -1,133 +0,0 @@
-/** @file
-
-  A brief file description
-
-  @section license License
-
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
- */
-
-#ifndef _P_CLUSTER_H__
-#define _P_CLUSTER_H__
-
-#include "ts/ink_platform.h"
-#include "P_EventSystem.h"
-#include "I_RecProcess.h"
-#include "P_Net.h"
-#include "P_Cache.h"
-
-#include "MIME.h"
-#include "HTTP.h"
-// #include "HttpTransactCache.h"
-
-#include "P_ClusterMachine.h"
-#include "P_ClusterCache.h"
-#include "P_ClusterCacheInternal.h"
-#include "P_ClusterInternal.h"
-#include "P_ClusterHandler.h"
-#include "P_ClusterInline.h"
-#include "P_ClusterLib.h"
-#include "P_ClusterLoadMonitor.h"
-#include "P_TimeTrace.h"
-
-#define ECLUSTER_NO_VC (CLUSTER_ERRNO + 0)
-#define ECLUSTER_NO_MACHINE (CLUSTER_ERRNO + 1)
-#define ECLUSTER_OP_TIMEOUT (CLUSTER_ERRNO + 2)
-#define ECLUSTER_ORB_DATA_READ (CLUSTER_ERRNO + 3)
-#define ECLUSTER_ORB_EIO (CLUSTER_ERRNO + 4)
-#define ECLUSTER_CHANNEL_INUSE (CLUSTER_ERRNO + 5)
-#define ECLUSTER_NOMORE_CHANNELS (CLUSTER_ERRNO + 6)
-
-int init_clusterprocessor(void);
-enum {
-  CLUSTER_CONNECTIONS_OPEN_STAT,
-  CLUSTER_CONNECTIONS_OPENNED_STAT,
-  CLUSTER_CON_TOTAL_TIME_STAT,
-  CLUSTER_CTRL_MSGS_SENT_STAT,
-  CLUSTER_SLOW_CTRL_MSGS_SENT_STAT,
-  CLUSTER_CTRL_MSGS_RECVD_STAT,
-  CLUSTER_SLOW_CTRL_MSGS_RECVD_STAT,
-  CLUSTER_CTRL_MSGS_SEND_TIME_STAT,
-  CLUSTER_CTRL_MSGS_RECV_TIME_STAT,
-  CLUSTER_READ_BYTES_STAT,
-  CLUSTER_WRITE_BYTES_STAT,
-  CLUSTER_OP_DELAYED_FOR_LOCK_STAT,
-  CLUSTER_CONNECTIONS_READ_LOCKED_STAT,
-  CLUSTER_CONNECTIONS_WRITE_LOCKED_STAT,
-  CLUSTER_CONNECTIONS_BUMPED_STAT,
-  CLUSTER_NODES_STAT,
-  CLUSTER_NET_BACKUP_STAT,
-  CLUSTER_MACHINES_ALLOCATED_STAT,
-  CLUSTER_MACHINES_FREED_STAT,
-  CLUSTER_CONFIGURATION_CHANGES_STAT,
-  CLUSTER_DELAYED_READS_STAT,
-  CLUSTER_BYTE_BANK_USED_STAT,
-  CLUSTER_ALLOC_DATA_NEWS_STAT,
-  CLUSTER_WRITE_BB_MALLOCS_STAT,
-  CLUSTER_PARTIAL_READS_STAT,
-  CLUSTER_PARTIAL_WRITES_STAT,
-  CLUSTER_CACHE_OUTSTANDING_STAT,
-  CLUSTER_REMOTE_OP_TIMEOUTS_STAT,
-  CLUSTER_REMOTE_OP_REPLY_TIMEOUTS_STAT,
-  CLUSTER_CHAN_INUSE_STAT,
-  CLUSTER_OPEN_DELAYS_STAT,
-  CLUSTER_OPEN_DELAY_TIME_STAT,
-  CLUSTER_CACHE_CALLBACKS_STAT,
-  CLUSTER_CACHE_CALLBACK_TIME_STAT,
-  CLUSTER_THREAD_STEAL_EXPIRES_STAT,
-  CLUSTER_RDMSG_ASSEMBLE_TIME_STAT,
-  CLUSTER_PING_TIME_STAT,
-  cluster_setdata_no_CLUSTERVC_STAT,
-  CLUSTER_SETDATA_NO_TUNNEL_STAT,
-  CLUSTER_SETDATA_NO_CACHEVC_STAT,
-  cluster_setdata_no_CLUSTER_STAT,
-  CLUSTER_VC_WRITE_STALL_STAT,
-  CLUSTER_NO_REMOTE_SPACE_STAT,
-  CLUSTER_LEVEL1_BANK_STAT,
-  CLUSTER_MULTILEVEL_BANK_STAT,
-  CLUSTER_VC_CACHE_INSERT_LOCK_MISSES_STAT,
-  CLUSTER_VC_CACHE_INSERTS_STAT,
-  CLUSTER_VC_CACHE_LOOKUP_LOCK_MISSES_STAT,
-  CLUSTER_VC_CACHE_LOOKUP_HITS_STAT,
-  CLUSTER_VC_CACHE_LOOKUP_MISSES_STAT,
-  CLUSTER_VC_CACHE_SCANS_STAT,
-  CLUSTER_VC_CACHE_SCAN_LOCK_MISSES_STAT,
-  CLUSTER_VC_CACHE_PURGES_STAT,
-  CLUSTER_WRITE_LOCK_MISSES_STAT,
-  CLUSTER_CACHE_RMT_CALLBACK_TIME_STAT,
-  CLUSTER_CACHE_LKRMT_CALLBACK_TIME_STAT,
-  CLUSTER_LOCAL_CONNECTION_TIME_STAT,
-  CLUSTER_REMOTE_CONNECTION_TIME_STAT,
-  CLUSTER_SETDATA_NO_CLUSTERVC_STAT,
-  CLUSTER_SETDATA_NO_CLUSTER_STAT,
-  CLUSTER_VC_READ_LIST_LEN_STAT,
-  CLUSTER_VC_WRITE_LIST_LEN_STAT,
-  cluster_stat_count
-};
-
-extern RecRawStatBlock *cluster_rsb;
-#define CLUSTER_INCREMENT_DYN_STAT(x) RecIncrRawStat(cluster_rsb, mutex->thread_holding, (int)x, 1);
-#define CLUSTER_DECREMENT_DYN_STAT(x) RecIncrRawStat(cluster_rsb, mutex->thread_holding, (int)x, -1);
-#define CLUSTER_SUM_DYN_STAT(x, y) RecIncrRawStat(cluster_rsb, mutex->thread_holding, (int)x, y);
-#define CLUSTER_SUM_GLOBAL_DYN_STAT(x, y) RecIncrGlobalRawStatSum(cluster_rsb, x, y)
-#define CLUSTER_CLEAR_DYN_STAT(x)          \
-  do {                                     \
-    RecSetRawStatSum(cluster_rsb, x, 0);   \
-    RecSetRawStatCount(cluster_rsb, x, 0); \
-  } while (0);
-
-#endif
diff --git a/iocore/cluster/P_ClusterCache.h b/iocore/cluster/P_ClusterCache.h
deleted file mode 100644
index c00790d..0000000
--- a/iocore/cluster/P_ClusterCache.h
+++ /dev/null
@@ -1,1175 +0,0 @@
-/** @file
-
-  A brief file description
-
-  @section license License
-
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
- */
-
-/****************************************************************************
-
-  Cluster.h
-
-
-****************************************************************************/
-
-#ifndef _P_Cluster_Cache_h
-#define _P_Cluster_Cache_h
-
-//*****************************************************************************
-// Initially derived from Cluster.h "1.77.2.11 1999/01/21 03:24:10"
-//*****************************************************************************
-
-/****************************************************************************/
-// #define LOCAL_CLUSTER_TEST_MODE 1
-// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-//  Set the above #define to enable local clustering.  "Local clustering"
-//  is a test only mode where all cluster nodes reside on the same host.
-//
-//  Configuration notes:
-//   - For "cluster.config" entries, always use "127.0.0.1" as the IP
-//     address and select a host unique cluster port.
-//
-//  Restrictions:
-//   1) Does not work with the manager.  You must only run with the server
-//      and hand configure "cluster.config".
-//   2) Currently, this has only been tested in a two node configuration.
-//
-/****************************************************************************/
-
-#include "P_ClusterMachine.h"
-
-//
-// Cluster Processor
-//
-// - monitors the status of the cluster
-// - provides communication between machines in the cluster
-// - provides callbacks to other processors when the cluster configuration
-//   changes
-//
-#define CLUSTER_MAJOR_VERSION 3
-#define CLUSTER_MINOR_VERSION 2
-
-// Lowest supported major/minor cluster version
-#define MIN_CLUSTER_MAJOR_VERSION CLUSTER_MAJOR_VERSION
-#define MIN_CLUSTER_MINOR_VERSION CLUSTER_MINOR_VERSION
-
-#define DEFAULT_CLUSTER_PORT_NUMBER 0
-#define DEFAULT_NUMBER_OF_CLUSTER_THREADS 1
-#define DEFAULT_CLUSTER_HOST ""
-
-#define MAX_CLUSTER_SEND_LENGTH INT_MAX
-
-#define CLUSTER_MAX_MACHINES 256
-// less than 1% disparity at 255 machines, 32707 is prime less than 2^15
-#define CLUSTER_HASH_TABLE_SIZE 32707
-
-// after timeout the configuration is "dead"
-#define CLUSTER_CONFIGURATION_TIMEOUT HRTIME_DAY
-// after zombie the configuration is deleted
-#define CLUSTER_CONFIGURATION_ZOMBIE (HRTIME_DAY * 2)
-
-// the number of configurations into the past we probe for data
-// one allows a new machine to come into or fall out of the
-// cluster without loss of data.  If the data is redistributed within
-// one day, no data will be lost.
-#define CONFIGURATION_HISTORY_PROBE_DEPTH 1
-
-// move these to a central event definition file (Event.h)
-#define CLUSTER_EVENT_CHANGE (CLUSTER_EVENT_EVENTS_START)
-#define CLUSTER_EVENT_CONFIGURATION (CLUSTER_EVENT_EVENTS_START + 1)
-#define CLUSTER_EVENT_OPEN (CLUSTER_EVENT_EVENTS_START + 2)
-#define CLUSTER_EVENT_OPEN_EXISTS (CLUSTER_EVENT_EVENTS_START + 3)
-#define CLUSTER_EVENT_OPEN_FAILED (CLUSTER_EVENT_EVENTS_START + 4)
-
-// internal event code
-#define CLUSTER_EVENT_STEAL_THREAD (CLUSTER_EVENT_EVENTS_START + 50)
-
-//////////////////////////////////////////////////////////////
-// Miscellaneous byte swap routines
-//////////////////////////////////////////////////////////////
-inline void
-ats_swap16(uint16_t *d)
-{
-  unsigned char *p = (unsigned char *)d;
-  *d               = ((p[1] << 8) | p[0]);
-}
-
-inline uint16_t
-ats_swap16(uint16_t d)
-{
-  ats_swap16(&d);
-  return d;
-}
-
-inline void
-ats_swap32(uint32_t *d)
-{
-  unsigned char *p = (unsigned char *)d;
-  *d               = ((p[3] << 24) | (p[2] << 16) | (p[1] << 8) | p[0]);
-}
-
-inline uint32_t
-ats_swap32(uint32_t d)
-{
-  ats_swap32(&d);
-  return d;
-}
-
-inline void
-ats_swap64(uint64_t *d)
-{
-  unsigned char *p = (unsigned char *)d;
-  *d = (((uint64_t)p[7] << 56) | ((uint64_t)p[6] << 48) | ((uint64_t)p[5] << 40) | ((uint64_t)p[4] << 32) | ((uint64_t)p[3] << 24) |
-        ((uint64_t)p[2] << 16) | ((uint64_t)p[1] << 8) | (uint64_t)p[0]);
-}
-
-inline uint64_t
-ats_swap64(uint64_t d)
-{
-  ats_swap64(&d);
-  return d;
-}
-
-//////////////////////////////////////////////////////////////
-
-struct ClusterConfiguration {
-  int n_machines;
-  ClusterMachine *machines[CLUSTER_MAX_MACHINES];
-
-  ClusterMachine *
-  machine_hash(unsigned int hash_value)
-  {
-    return machines[hash_table[hash_value % CLUSTER_HASH_TABLE_SIZE]];
-  }
-
-  ClusterMachine *
-  find(unsigned int ip, int port = 0)
-  {
-    for (int i = 0; i < n_machines; i++)
-      if (ip == machines[i]->ip && (!port || !machines[i]->cluster_port || machines[i]->cluster_port == port))
-        return machines[i];
-    return nullptr;
-  }
-
-  //
-  // Private
-  //
-  ClusterConfiguration();
-  unsigned char hash_table[CLUSTER_HASH_TABLE_SIZE];
-  ink_hrtime changed;
-  SLINK(ClusterConfiguration, link);
-};
-
-inline bool
-machine_in_vector(ClusterMachine *m, ClusterMachine **mm, int len)
-{
-  for (int i = 0; i < len; i++)
-    if (m == mm[i])
-      return true;
-  return false;
-}
-
-//
-// Returns either a machine or nullptr.
-// Finds a machine starting at probe_depth going up to
-//    CONFIGURATION_HISTORY_PROBE_DEPTH
-// which is up, not the current machine and has not yet been probed.
-// Updates: probe_depth and past_probes.
-//
-inkcoreapi ClusterMachine *cluster_machine_at_depth(unsigned int hash, int *probe_depth = nullptr,
-                                                    ClusterMachine **past_probes = nullptr);
-
-//
-// Cluster
-//   A cluster of machines which act as a single cache.
-//
-struct Cluster {
-  //
-  // Public Interface
-  //
-
-  //
-  // Cluster Hash Function
-  //
-
-  // Takes a hash value to a machine.  The hash function has the following
-  // properties:
-  //   1 - it divides input domain into the output range evenly (within 1%)
-  //   2 - it tends to produce the same Machine for the same hash_value's
-  //       for different configurations
-  //   3 - it produces the hash same function for a given configuration of
-  //       machines independent of the order they were added or removed
-  //       from the cluster.  (it is a pure function of the configuration)
-  //   Thread-safe
-  //
-  ClusterMachine *
-  machine_hash(unsigned int hash_value)
-  {
-    return current_configuration()->machine_hash(hash_value);
-  }
-
-  //
-  // Cluster Configuration
-  //
-
-  // Register callback for a cluster configuration change.
-  // calls cont->handleEvent(EVENT_CLUSTER_CHANGE);
-  //   Thread-safe
-  //
-  void cluster_change_callback(Continuation *cont);
-
-  // Return the current configuration
-  //   Thread-safe
-  //
-  ClusterConfiguration *
-  current_configuration()
-  {
-    return configurations.head;
-  }
-
-  // Return the previous configuration.
-  // Use from within the cluster_change_callback.
-  //   Thread-safe
-  //
-  ClusterConfiguration *
-  previous_configuration()
-  {
-    return configurations.head->link.next;
-  }
-
-  //
-  // Private
-  //
-  // The configurations are updated only in the thread which is
-  // accepting cluster connections.
-  //
-  SLL<ClusterConfiguration> configurations;
-
-  Cluster();
-};
-
-//
-// ClusterVCToken
-//   An token passed between nodes to represent a virtualized connection.
-//   (see ClusterProcessor::alloc_remote() and attach_remote() below)
-//
-struct ClusterVCToken {
-  //
-  // Marshal this data to send the token across the cluster
-  //
-  uint32_t ip_created;
-  uint32_t ch_id;
-  uint32_t sequence_number;
-
-  bool
-  is_clear()
-  {
-    return !ip_created;
-  }
-  void
-  clear()
-  {
-    ip_created      = 0;
-    sequence_number = 0;
-  }
-
-  ClusterVCToken(unsigned int aip = 0, unsigned int id = 0, unsigned int aseq = 0)
-    : ip_created(aip), ch_id(id), sequence_number(aseq)
-  {
-  }
-  //
-  // Private
-  //
-  void alloc();
-
-  inline void
-  SwapBytes()
-  {
-    ats_swap32(&ch_id);
-    ats_swap32(&sequence_number);
-  }
-};
-
-//
-// ClusterFunctionPtr
-//   A pointer to a procedure which can be invoked accross the cluster.
-//   This must be registered.
-//
-typedef void ClusterFunction(ClusterHandler *ch, void *data, int len);
-typedef ClusterFunction *ClusterFunctionPtr;
-
-struct ClusterVConnectionBase;
-
-struct ClusterVConnState {
-  //
-  // Private
-  //
-  volatile int enabled;
-  // multiples of XXX_PERIOD, high = less often
-  int priority;
-  VIO vio;
-  void *queue;
-  int ifd;
-  Event *delay_timeout;
-  Link<ClusterVConnectionBase> link;
-
-  // void enqueue(void * q, ClusterVConnection * vc);
-  ClusterVConnState();
-};
-
-struct ClusterVConnectionBase : public CacheVConnection {
-  //
-  // Initiate an IO operation.
-  // "data" is unused.
-  // Only one READ and one WRITE may be active at one time.
-  //   THREAD-SAFE, may be called when not handling an event from
-  //                the ClusterVConnectionBase, or the ClusterVConnectionBase
-  //                creation callback.
-  //
-
-  virtual VIO *do_io_read(Continuation *c, int64_t nbytes, MIOBuffer *buf);
-  virtual VIO *do_io_write(Continuation *c, int64_t nbytes, IOBufferReader *buf, bool owner = false);
-  virtual void
-  do_io_shutdown(ShutdownHowTo_t howto)
-  {
-    (void)howto;
-    ink_assert(!"shutdown of cluster connection");
-  }
-  virtual void do_io_close(int lerrno = -1);
-  virtual VIO *do_io_pread(Continuation *, int64_t, MIOBuffer *, int64_t);
-
-  // Set the timeouts associated with this connection.
-  // active_timeout is for the total elasped time of the connection.
-  // inactivity_timeout is the elapsed time *while an operation was
-  //   enabled* during which the connection was unable to sink/provide data.
-  // calling these functions repeatedly resets the timeout.
-  //   NOT THREAD-SAFE, may only be called when handing an event from this
-  //                    ClusterVConnectionBase, or the ClusterVConnectionBase
-  //                    creation callback.
-  //
-  void set_active_timeout(ink_hrtime timeout_in);
-  void set_inactivity_timeout(ink_hrtime timeout_in);
-  void cancel_active_timeout();
-  void cancel_inactivity_timeout();
-
-  ClusterVConnectionBase();
-
-#ifdef DEBUG
-  // Class static data
-  static int enable_debug_trace;
-#endif
-  Action action_;
-  EThread *thread;
-  volatile int closed;
-  ClusterVConnState read;
-  ClusterVConnState write;
-  LINKM(ClusterVConnectionBase, read, link)
-  LINKM(ClusterVConnectionBase, write, link)
-  ink_hrtime inactivity_timeout_in;
-  ink_hrtime active_timeout_in;
-  Event *inactivity_timeout;
-  Event *active_timeout;
-
-  virtual void reenable(VIO *);
-  virtual void reenable_re(VIO *);
-};
-
-inline void
-ClusterVConnectionBase::set_active_timeout(ink_hrtime timeout_in)
-{
-  active_timeout_in = timeout_in;
-  if (active_timeout) {
-    ink_assert(!active_timeout->cancelled);
-    if (active_timeout->ethread == this_ethread())
-      active_timeout->schedule_in(timeout_in);
-    else {
-      active_timeout->cancel(this);
-      active_timeout = thread->schedule_in(this, timeout_in);
-    }
-  } else {
-    if (thread) {
-      active_timeout = thread->schedule_in(this, timeout_in);
-    }
-  }
-}
-
-inline void
-ClusterVConnectionBase::set_inactivity_timeout(ink_hrtime timeout_in)
-{
-  inactivity_timeout_in = timeout_in;
-  if (inactivity_timeout) {
-    ink_assert(!inactivity_timeout->cancelled);
-    if (inactivity_timeout->ethread == this_ethread())
-      inactivity_timeout->schedule_in(timeout_in);
-    else {
-      inactivity_timeout->cancel(this);
-      inactivity_timeout = thread->schedule_in(this, timeout_in);
-    }
-  } else {
-    if (thread) {
-      inactivity_timeout = thread->schedule_in(this, timeout_in);
-    }
-  }
-}
-
-inline void
-ClusterVConnectionBase::cancel_active_timeout()
-{
-  if (active_timeout) {
-    active_timeout->cancel(this);
-    active_timeout    = nullptr;
-    active_timeout_in = 0;
-  }
-}
-
-inline void
-ClusterVConnectionBase::cancel_inactivity_timeout()
-{
-  if (inactivity_timeout) {
-    inactivity_timeout->cancel(this);
-    inactivity_timeout    = nullptr;
-    inactivity_timeout_in = 0;
-  }
-}
-
-// Data debt owed to VC which is deferred due to lock miss
-class ByteBankDescriptor
-{
-public:
-  ByteBankDescriptor() {}
-  IOBufferBlock *
-  get_block() const
-  {
-    return block.get();
-  }
-
-  static ByteBankDescriptor *ByteBankDescriptor_alloc(Ptr<IOBufferBlock> &);
-  static void ByteBankDescriptor_free(ByteBankDescriptor *);
-
-public:
-  LINK(ByteBankDescriptor, link);
-
-private:
-  Ptr<IOBufferBlock> block; // holder of bank bytes
-};
-
-enum TypeVConnection {
-  VC_NULL,
-  VC_CLUSTER,
-  VC_CLUSTER_READ,
-  VC_CLUSTER_WRITE,
-  VC_CLUSTER_CLOSED,
-};
-
-//
-// ClusterVConnection
-//
-struct ClusterVConnection : public ClusterVConnectionBase {
-  //
-  // Public Interface (included from ClusterVConnectionBase)
-  //
-  // Thread-safe  (see Net.h for details)
-  //
-  // virtual VIO * do_io(
-  //   int                   op,
-  //   Continuation        * c = nullptr,
-  //   int                   nbytes = INT64_MAX,
-  //   MIOBuffer           * buf = 0,
-  //   int                   whence = SEEK_CUR);
-  //
-  // NOT Thread-safe (see Net.h for details)
-  //
-  // void set_active_timeout(ink_hrtime timeout_in);
-  // void set_inactivity_timeout(ink_hrtime timeout_in);
-
-  //
-  // Private
-  //
-
-  int startEvent(int event, Event *e);
-  int mainEvent(int event, Event *e);
-
-  // 0 on success -1 on failure
-  int start(EThread *t); // New connect protocol
-
-  ClusterVConnection(int is_new_connect_read = 0);
-  ~ClusterVConnection();
-  void free(); // Destructor actions (we are using ClassAllocator)
-
-  virtual void do_io_close(int lerrno = -1);
-  virtual VIO *do_io_read(Continuation *c, int64_t nbytes, MIOBuffer *buf);
-  virtual VIO *do_io_write(Continuation *c, int64_t nbytes, IOBufferReader *buf, bool owner = false);
-  virtual void reenable(VIO *vio);
-
-  ClusterHandler *ch;
-  //
-  //  Read Channel: (new_connect_read == true)
-  //     - open_local()    caller is reader
-  //     - connect_local() caller is writer
-  //
-  //  Write Channel: (new_connect_read == false)
-  //     - open_local()    caller is writer
-  //     - connect_local() caller is reader
-  //
-  int new_connect_read; // Data flow direction wrt origin node
-  int remote_free;
-  int last_local_free;
-  int channel;
-  ClusterVCToken token;
-  volatile int close_disabled;
-  volatile int remote_closed;
-  volatile int remote_close_disabled;
-  volatile int remote_lerrno;
-  volatile uint32_t in_vcs;
-  volatile uint32_t type;
-  SLINK(ClusterVConnection, ready_alink);
-  int was_closed();
-  void allow_close();
-  void disable_close();
-  int was_remote_closed();
-  void allow_remote_close();
-  bool schedule_write();
-  void set_type(int);
-  ink_hrtime start_time;
-  ink_hrtime last_activity_time;
-  Queue<ByteBankDescriptor> byte_bank_q; // done awaiting completion
-  int n_set_data_msgs;                   // # pending set_data() msgs on VC
-  int n_recv_set_data_msgs;              // # set_data() msgs received on VC
-  volatile int pending_remote_fill;      // Remote fill pending on connection
-  Ptr<IOBufferBlock> read_block;         // Hold current data for open read
-  bool remote_ram_cache_hit;             // Entire object was from remote ram cache
-  bool have_all_data;                    // All data in read_block
-  int initial_data_bytes;                // bytes in open_read buffer
-  Ptr<IOBufferBlock> remote_write_block; // Write side data for remote fill
-  void *current_cont;                    // Track current continuation (debug)
-
-#define CLUSTER_IOV_NOT_OPEN -2
-#define CLUSTER_IOV_NONE -1
-  int iov_map; // which iov?
-
-  Ptr<ProxyMutex> read_locked;
-  Ptr<ProxyMutex> write_locked;
-
-  // Data buffer for unmarshaled objects from remote node.
-  Ptr<IOBufferData> marshal_buf;
-
-  // Pending write data
-  Ptr<IOBufferBlock> write_list;
-  IOBufferBlock *write_list_tail;
-  int write_list_bytes;
-  int write_bytes_in_transit;
-
-  CacheHTTPInfo alternate;
-  time_t time_pin;
-  int disk_io_priority;
-  void set_remote_fill_action(Action *);
-
-  // Indicates whether a cache hit was from an peering cluster cache
-  bool
-  is_ram_cache_hit() const
-  {
-    return remote_ram_cache_hit;
-  };
-  void
-  set_ram_cache_hit(bool remote_hit)
-  {
-    remote_ram_cache_hit = remote_hit;
-  }
-
-  // For VC(s) established via OPEN_READ, we are passed a CacheHTTPInfo
-  //  in the reply.
-  virtual bool get_data(int id, void *data); // backward compatibility
-  virtual void get_http_info(CacheHTTPInfo **);
-  virtual int64_t get_object_size();
-  virtual bool is_pread_capable();
-
-  // For VC(s) established via the HTTP version of OPEN_WRITE, additional
-  //  data for the VC is passed in a second message.  This additional
-  //  data has a lifetime equal to the cache VC
-  virtual void set_http_info(CacheHTTPInfo *);
-
-  virtual bool set_pin_in_cache(time_t time_pin);
-  virtual time_t get_pin_in_cache();
-  virtual bool set_disk_io_priority(int priority);
-  virtual int get_disk_io_priority();
-  virtual int get_header(void **ptr, int *len);
-  virtual int set_header(void *ptr, int len);
-  virtual int get_single_data(void **ptr, int *len);
-};
-
-//
-// Cluster operation options
-//
-#define CLUSTER_OPT_STEAL 0x0001            // allow thread stealing
-#define CLUSTER_OPT_IMMEDIATE 0x0002        // require immediate response
-#define CLUSTER_OPT_ALLOW_IMMEDIATE 0x0004  // allow immediate response
-#define CLUSTER_OPT_DELAY 0x0008            // require delayed response
-#define CLUSTER_OPT_CONN_READ 0x0010        // new conn read
-#define CLUSTER_OPT_CONN_WRITE 0x0020       // new conn write
-#define CLUSTER_OPT_DATA_IS_OCONTROL 0x0040 // data in OutgoingControl
-#define CLUSTER_FUNCTION_MALLOCED -1
-
-struct ClusterRemoteDataHeader {
-  int32_t cluster_function;
-};
-//
-// ClusterProcessor
-//
-class ClusterAccept;
-
-struct ClusterProcessor {
-  //
-  // Public Interface
-  //
-
-  // Invoke a function on a remote node
-  //   marshal your own data, provide a continuation for timeouts and errors
-  //
-  // Options: CLUSTER_OPT_DELAY, CLUSTER_OPT_STEAL, CLUSTER_OPT_DATA_IS_OCONTROL
-  // Returns: 1 for immediate send, 0 for delayed, -1 for error
-
-  int invoke_remote(ClusterHandler *ch, int cluster_fn_index, void *data, int len, int options = CLUSTER_OPT_STEAL);
-
-  int invoke_remote_data(ClusterHandler *ch, int cluster_fn_index, void *data, int data_len, IOBufferBlock *buf,
-                         int logical_channel, ClusterVCToken *token, void (*bufdata_free)(void *), void *bufdata_free_arg,
-                         int options = CLUSTER_OPT_STEAL);
-
-  // Pass the data in as a malloc'ed block to be freed by callee
-  int
-  invoke_remote_malloced(ClusterHandler *ch, ClusterRemoteDataHeader *data, int len /* including header */)
-  {
-    return invoke_remote(ch, CLUSTER_FUNCTION_MALLOCED, data, len);
-  }
-  void free_remote_data(char *data, int len);
-
-// Allocate the local side of a remote VConnection.
-// returns a token which can be passed to the remote side
-// through an existing link and passed to attach_remoteVC()
-// if CLUSTER_OPT_IMMEDIATE is set, CLUSTER_DELAYED_OPEN will not be returned
-//
-// Options: CLUSTER_OPT_IMMEDIATE, CLUSTER_OPT_ALLOW_IMMEDIATE
-// Returns: pointer for CLUSTER_OPT_IMMEDIATE
-//            or CLUSTER_DELAYED_OPEN on success,
-//          nullptr on failure
-// calls:  cont->handleEvent( CLUSTER_EVENT_OPEN, ClusterVConnection *)
-//         on delayed success.
-//
-// NOTE: the CLUSTER_EVENT_OPEN may be called before "open/connect" returns
-
-#define CLUSTER_DELAYED_OPEN ((ClusterVConnection *)-1)
-#define CLUSTER_NODE_DOWN ((ClusterVConnection *)-2)
-  ClusterVConnection *open_local(Continuation *cont, ClusterMachine *mp, ClusterVCToken &token, int options = 0);
-
-  // Get the other side of a remote VConnection which was previously
-  // allocated with open.
-  //
-  // Options: CLUSTER_OPT_IMMEDIATE, CLUSTER_OPT_ALLOW_IMMEDIATE
-  // return a pointer or CLUSTER_DELAYED_OPEN success, nullptr on failure
-  //
-  ClusterVConnection *connect_local(Continuation *cont, ClusterVCToken *token, int channel, int options = 0);
-  inkcoreapi bool disable_remote_cluster_ops(ClusterMachine *);
-
-  //
-  // Processor interface
-  //
-  virtual int init();
-  virtual int start();
-
-  ClusterProcessor();
-  virtual ~ClusterProcessor();
-
-  //
-  // Private
-  //
-  ClusterAccept *accept_handler;
-  Cluster *this_cluster;
-  // Connect to a new cluster machine
-  void connect(char *hostname, int16_t id = -1);
-  void connect(unsigned int ip, int port = 0, int16_t id = -1, bool delay = false);
-  // send the list of known machines to new machine
-  void send_machine_list(ClusterMachine *m);
-  void compute_cluster_mode();
-  // Internal invoke_remote interface
-  int internal_invoke_remote(ClusterHandler *m, int cluster_fn, void *data, int len, int options, void *cmsg);
-};
-
-inkcoreapi extern ClusterProcessor clusterProcessor;
-
-inline Cluster *
-this_cluster()
-{
-  return clusterProcessor.this_cluster;
-}
-
-//
-// Set up a thread to receive events from the ClusterProcessor
-// This function should be called for all threads created to
-// accept such events by the EventProcesor.
-//
-void initialize_thread_for_cluster(EThread *thread);
-
-//
-// ClusterFunction Registry
-//
-//   Declare an instance of this class here to register
-//   a function.   In order to allow older versions of software
-//   to co-exist with newer versions, always add to the bottom
-//   of the list.
-//
-
-extern ClusterFunction *ptest_ClusterFunction;
-
-extern ClusterFunction test_ClusterFunction;
-extern ClusterFunction ping_ClusterFunction;
-extern ClusterFunction ping_reply_ClusterFunction;
-extern ClusterFunction machine_list_ClusterFunction;
-extern ClusterFunction close_channel_ClusterFunction;
-extern ClusterFunction get_hostinfo_ClusterFunction;
-extern ClusterFunction put_hostinfo_ClusterFunction;
-extern ClusterFunction cache_lookup_ClusterFunction;
-extern ClusterFunction cache_op_ClusterFunction;
-extern ClusterFunction cache_op_malloc_ClusterFunction;
-extern ClusterFunction cache_op_result_ClusterFunction;
-extern ClusterFunction set_channel_data_ClusterFunction;
-extern ClusterFunction post_setchan_send_ClusterFunction;
-extern ClusterFunction set_channel_pin_ClusterFunction;
-extern ClusterFunction post_setchan_pin_ClusterFunction;
-extern ClusterFunction set_channel_priority_ClusterFunction;
-extern ClusterFunction post_setchan_priority_ClusterFunction;
-extern ClusterFunction default_api_ClusterFunction;
-
-struct ClusterFunctionDescriptor {
-  bool fMalloced;   // the function will free the data
-  bool ClusterFunc; // Process incoming message only
-  //   in ET_CLUSTER thread.
-  int q_priority; // lower is higher priority
-  ClusterFunctionPtr pfn;
-  ClusterFunctionPtr post_pfn; // msg queue/send callout
-};
-
-#define CLUSTER_CMSG_QUEUES 2
-#define CMSG_MAX_PRI 0
-#define CMSG_LOW_PRI (CLUSTER_CMSG_QUEUES - 1)
-
-#ifndef DEFINE_CLUSTER_FUNCTIONS
-extern
-#endif
-  ClusterFunctionDescriptor clusterFunction[]
-#ifdef DEFINE_CLUSTER_FUNCTIONS
-  =
-    {
-      {false, true, CMSG_LOW_PRI, test_ClusterFunction, 0},
-      {false, true, CMSG_LOW_PRI, ping_ClusterFunction, 0},
-      {false, true, CMSG_LOW_PRI, ping_reply_ClusterFunction, 0},
-      {false, true, CMSG_LOW_PRI, machine_list_ClusterFunction, 0},
-      {false, true, CMSG_LOW_PRI, close_channel_ClusterFunction, 0},
-      {false, false, CMSG_LOW_PRI, get_hostinfo_ClusterFunction, 0}, // in HostDB.cc
-      {false, false, CMSG_LOW_PRI, put_hostinfo_ClusterFunction, 0}, // in HostDB.cc
-      {false, true, CMSG_LOW_PRI, cache_lookup_ClusterFunction, 0},  // in CacheCont.cc
-      {true, true, CMSG_LOW_PRI, cache_op_malloc_ClusterFunction, 0},
-      {false, true, CMSG_LOW_PRI, cache_op_ClusterFunction, 0},
-      {false, false, CMSG_LOW_PRI, cache_op_result_ClusterFunction, 0},
-      {false, false, CMSG_LOW_PRI, 0, 0}, // OBSOLETE
-      {false, false, CMSG_LOW_PRI, 0, 0}, // OBSOLETE
-      {false, false, CMSG_LOW_PRI, 0, 0}, // OBSOLETE
-      {false, true, CMSG_MAX_PRI, set_channel_data_ClusterFunction, post_setchan_send_ClusterFunction},
-      {false, true, CMSG_MAX_PRI, set_channel_pin_ClusterFunction, post_setchan_pin_ClusterFunction},
-      {false, true, CMSG_MAX_PRI, set_channel_priority_ClusterFunction, post_setchan_priority_ClusterFunction},
-      /********************************************
-       * RESERVED for future cluster internal use *
-       ********************************************/
-      {false, false, CMSG_LOW_PRI, 0, 0},
-      {false, false, CMSG_LOW_PRI, 0, 0},
-      {false, false, CMSG_LOW_PRI, 0, 0},
-      {false, false, CMSG_LOW_PRI, 0, 0},
-      {false, false, CMSG_LOW_PRI, 0, 0},
-      {false, false, CMSG_LOW_PRI, 0, 0},
-      {false, false, CMSG_LOW_PRI, 0, 0},
-      {false, false, CMSG_LOW_PRI, 0, 0},
-      {false, false, CMSG_LOW_PRI, 0, 0},
-      {false, false, CMSG_LOW_PRI, 0, 0},
-      {false, false, CMSG_LOW_PRI, 0, 0},
-      {false, false, CMSG_LOW_PRI, 0, 0},
-      {false, false, CMSG_LOW_PRI, 0, 0},
-      {false, false, CMSG_LOW_PRI, 0, 0},
-      {false, false, CMSG_LOW_PRI, 0, 0},
-      {false, false, CMSG_LOW_PRI, 0, 0},
-      {false, false, CMSG_LOW_PRI, 0, 0},
-      {false, false, CMSG_LOW_PRI, 0, 0},
-      {false, false, CMSG_LOW_PRI, 0, 0},
-      {false, false, CMSG_LOW_PRI, 0, 0},
-      {false, false, CMSG_LOW_PRI, 0, 0},
-      {false, false, CMSG_LOW_PRI, 0, 0},
-      {false, false, CMSG_LOW_PRI, 0, 0},
-      {false, false, CMSG_LOW_PRI, 0, 0},
-      {false, false, CMSG_LOW_PRI, 0, 0},
-      {false, false, CMSG_LOW_PRI, 0, 0},
-      {false, false, CMSG_LOW_PRI, 0, 0},
-      {false, false, CMSG_LOW_PRI, 0, 0},
-      {false, false, CMSG_LOW_PRI, 0, 0},
-      {false, false, CMSG_LOW_PRI, 0, 0},
-      {false, false, CMSG_LOW_PRI, 0, 0},
-      {false, false, CMSG_LOW_PRI, 0, 0},
-      {false, false, CMSG_LOW_PRI, 0, 0},
-      {false, false, CMSG_LOW_PRI, 0, 0},
-
-      /*********************************************
-       * RESERVED for Cluster RPC API use		*
-       *********************************************/
-      {true, false, CMSG_LOW_PRI, default_api_ClusterFunction, 0},
-      {true, false, CMSG_LOW_PRI, default_api_ClusterFunction, 0},
-      {true, false, CMSG_LOW_PRI, default_api_ClusterFunction, 0},
-      {true, false, CMSG_LOW_PRI, default_api_ClusterFunction, 0},
-      {true, false, CMSG_LOW_PRI, default_api_ClusterFunction, 0},
-      {true, false, CMSG_LOW_PRI, default_api_ClusterFunction, 0},
-      {true, false, CMSG_LOW_PRI, default_api_ClusterFunction, 0},
-      {true, false, CMSG_LOW_PRI, default_api_ClusterFunction, 0},
-      {true, false, CMSG_LOW_PRI, default_api_ClusterFunction, 0},
-      {true, false, CMSG_LOW_PRI, default_api_ClusterFunction, 0},
-      {true, false, CMSG_LOW_PRI, default_api_ClusterFunction, 0},
-      {true, false, CMSG_LOW_PRI, default_api_ClusterFunction, 0},
-      {true, false, CMSG_LOW_PRI, default_api_ClusterFunction, 0},
-      {true, false, CMSG_LOW_PRI, default_api_ClusterFunction, 0},
-      {true, false, CMSG_LOW_PRI, default_api_ClusterFunction, 0},
-      {true, false, CMSG_LOW_PRI, default_api_ClusterFunction, 0},
-      {true, false, CMSG_LOW_PRI, default_api_ClusterFunction, 0},
-      {true, false, CMSG_LOW_PRI, default_api_ClusterFunction, 0},
-      {true, false, CMSG_LOW_PRI, default_api_ClusterFunction, 0},
-      {true, false, CMSG_LOW_PRI, default_api_ClusterFunction, 0},
-      {true, false, CMSG_LOW_PRI, default_api_ClusterFunction, 0},
-      {true, false, CMSG_LOW_PRI, default_api_ClusterFunction, 0},
-      {true, false, CMSG_LOW_PRI, default_api_ClusterFunction, 0},
-      {true, false, CMSG_LOW_PRI, default_api_ClusterFunction, 0},
-      {true, false, CMSG_LOW_PRI, default_api_ClusterFunction, 0},
-      {true, false, CMSG_LOW_PRI, default_api_ClusterFunction, 0},
-      {true, false, CMSG_LOW_PRI, default_api_ClusterFunction, 0},
-      {true, false, CMSG_LOW_PRI, default_api_ClusterFunction, 0},
-      {true, false, CMSG_LOW_PRI, default_api_ClusterFunction, 0},
-      {true, false, CMSG_LOW_PRI, default_api_ClusterFunction, 0}
-      // ********** ADD NEW ENTRIES ABOVE THIS LINE ************
-}
-#endif
-
-;
-extern unsigned SIZE_clusterFunction; // clusterFunction[] entries
-
-//////////////////////////////////////////////////////////////
-// Map from Cluster Function code to send queue priority
-//////////////////////////////////////////////////////////////
-inline int
-ClusterFuncToQpri(int cluster_func)
-{
-  if (cluster_func < 0) {
-    return CMSG_LOW_PRI;
-  } else {
-    return clusterFunction[cluster_func].q_priority;
-  }
-}
-
-//
-// This table had better match the above list
-//
-#define TEST_CLUSTER_FUNCTION 0
-#define PING_CLUSTER_FUNCTION 1
-#define PING_REPLY_CLUSTER_FUNCTION 2
-#define MACHINE_LIST_CLUSTER_FUNCTION 3
-#define CLOSE_CHANNEL_CLUSTER_FUNCTION 4
-#define GET_HOSTINFO_CLUSTER_FUNCTION 5
-#define PUT_HOSTINFO_CLUSTER_FUNCTION 6
-#define CACHE_LOOKUP_CLUSTER_FUNCTION 7
-#define CACHE_OP_MALLOCED_CLUSTER_FUNCTION 8
-#define CACHE_OP_CLUSTER_FUNCTION 9
-#define CACHE_OP_RESULT_CLUSTER_FUNCTION 10
-#define SET_CHANNEL_DATA_CLUSTER_FUNCTION 14
-#define SET_CHANNEL_PIN_CLUSTER_FUNCTION 15
-#define SET_CHANNEL_PRIORITY_CLUSTER_FUNCTION 16
-
-/********************************************
- * RESERVED for future cluster internal use *
- ********************************************/
-#define INTERNAL_RESERVED1_CLUSTER_FUNCTION 17
-#define INTERNAL_RESERVED2_CLUSTER_FUNCTION 18
-#define INTERNAL_RESERVED3_CLUSTER_FUNCTION 19
-#define INTERNAL_RESERVED4_CLUSTER_FUNCTION 20
-#define INTERNAL_RESERVED5_CLUSTER_FUNCTION 21
-#define INTERNAL_RESERVED6_CLUSTER_FUNCTION 22
-#define INTERNAL_RESERVED7_CLUSTER_FUNCTION 23
-#define INTERNAL_RESERVED8_CLUSTER_FUNCTION 24
-#define INTERNAL_RESERVED9_CLUSTER_FUNCTION 25
-#define INTERNAL_RESERVED10_CLUSTER_FUNCTION 26
-#define INTERNAL_RESERVED11_CLUSTER_FUNCTION 27
-#define INTERNAL_RESERVED12_CLUSTER_FUNCTION 28
-#define INTERNAL_RESERVED13_CLUSTER_FUNCTION 29
-#define INTERNAL_RESERVED14_CLUSTER_FUNCTION 30
-#define INTERNAL_RESERVED15_CLUSTER_FUNCTION 31
-#define INTERNAL_RESERVED16_CLUSTER_FUNCTION 32
-#define INTERNAL_RESERVED17_CLUSTER_FUNCTION 33
-#define INTERNAL_RESERVED18_CLUSTER_FUNCTION 34
-#define INTERNAL_RESERVED19_CLUSTER_FUNCTION 35
-#define INTERNAL_RESERVED20_CLUSTER_FUNCTION 36
-#define INTERNAL_RESERVED21_CLUSTER_FUNCTION 37
-#define INTERNAL_RESERVED22_CLUSTER_FUNCTION 38
-#define INTERNAL_RESERVED23_CLUSTER_FUNCTION 39
-#define INTERNAL_RESERVED24_CLUSTER_FUNCTION 40
-#define INTERNAL_RESERVED25_CLUSTER_FUNCTION 41
-#define INTERNAL_RESERVED26_CLUSTER_FUNCTION 42
-#define INTERNAL_RESERVED27_CLUSTER_FUNCTION 43
-#define INTERNAL_RESERVED28_CLUSTER_FUNCTION 44
-#define INTERNAL_RESERVED29_CLUSTER_FUNCTION 45
-#define INTERNAL_RESERVED30_CLUSTER_FUNCTION 46
-#define INTERNAL_RESERVED31_CLUSTER_FUNCTION 47
-#define INTERNAL_RESERVED32_CLUSTER_FUNCTION 48
-#define INTERNAL_RESERVED33_CLUSTER_FUNCTION 49
-#define INTERNAL_RESERVED34_CLUSTER_FUNCTION 50
-
-/****************************************************************************
- * Cluster RPC API definitions.						    *
- *									    *
- ****************************************************************************
- * Note: All of the following must be kept in sync with INKClusterRPCKey_t  *
- * 	 definition in ts/ts.h and ts/experimental.h			    *
- ****************************************************************************/
-
-/************************************************
- * RESERVED for Wireless Group			*
- ************************************************/
-#define API_F01_CLUSTER_FUNCTION 51
-#define API_F02_CLUSTER_FUNCTION 52
-#define API_F03_CLUSTER_FUNCTION 53
-#define API_F04_CLUSTER_FUNCTION 54
-#define API_F05_CLUSTER_FUNCTION 55
-#define API_F06_CLUSTER_FUNCTION 56
-#define API_F07_CLUSTER_FUNCTION 57
-#define API_F08_CLUSTER_FUNCTION 58
-#define API_F09_CLUSTER_FUNCTION 59
-#define API_F10_CLUSTER_FUNCTION 60
-
-/************************************************
- * RESERVED for future use			*
- ************************************************/
-#define API_F11_CLUSTER_FUNCTION 61
-#define API_F12_CLUSTER_FUNCTION 62
-#define API_F13_CLUSTER_FUNCTION 63
-#define API_F14_CLUSTER_FUNCTION 64
-#define API_F15_CLUSTER_FUNCTION 65
-#define API_F16_CLUSTER_FUNCTION 66
-#define API_F17_CLUSTER_FUNCTION 67
-#define API_F18_CLUSTER_FUNCTION 68
-#define API_F19_CLUSTER_FUNCTION 69
-#define API_F20_CLUSTER_FUNCTION 70
-
-#define API_F21_CLUSTER_FUNCTION 71
-#define API_F22_CLUSTER_FUNCTION 72
-#define API_F23_CLUSTER_FUNCTION 73
-#define API_F24_CLUSTER_FUNCTION 74
-#define API_F25_CLUSTER_FUNCTION 75
-#define API_F26_CLUSTER_FUNCTION 76
-#define API_F27_CLUSTER_FUNCTION 77
-#define API_F28_CLUSTER_FUNCTION 78
-#define API_F29_CLUSTER_FUNCTION 79
-#define API_F30_CLUSTER_FUNCTION 80
-
-#define API_STARECT_CLUSTER_FUNCTION API_F01_CLUSTER_FUNCTION
-#define API_END_CLUSTER_FUNCTION API_F30_CLUSTER_FUNCTION
-
-#define UNDEFINED_CLUSTER_FUNCTION 0xFDEFFDEF
-
-//////////////////////////////////////////////
-// Initial cluster connect exchange message
-//////////////////////////////////////////////
-struct ClusterHelloMessage {
-  uint16_t _NativeByteOrder;
-  uint16_t _major;
-  uint16_t _minor;
-  uint16_t _min_major;
-  uint16_t _min_minor;
-  int16_t _id;
-#ifdef LOCAL_CLUSTER_TEST_MODE
-  int16_t _port;
-  char _pad[114]; // pad out to 128 bytes
-#else
-  char _pad[116]; // pad out to 128 bytes
-#endif
-
-  ClusterHelloMessage() : _NativeByteOrder(1)
-  {
-    _major     = CLUSTER_MAJOR_VERSION;
-    _minor     = CLUSTER_MINOR_VERSION;
-    _min_major = MIN_CLUSTER_MAJOR_VERSION;
-    _min_minor = MIN_CLUSTER_MINOR_VERSION;
-    memset(_pad, '\0', sizeof(_pad));
-  }
-  int
-  NativeByteOrder()
-  {
-    return (_NativeByteOrder == 1);
-  }
-  void
-  AdjustByteOrder()
-  {
-    if (!NativeByteOrder()) {
-      ats_swap16(&_major);
-      ats_swap16(&_minor);
-      ats_swap16(&_min_major);
-      ats_swap16(&_min_minor);
-    }
-  }
-};
-
-///////////////////////////////////////////////////////////////////
-// Cluster message header definition.
-///////////////////////////////////////////////////////////////////
-struct ClusterMessageHeader {
-  uint16_t _InNativeByteOrder; // always non-zero
-  uint16_t _MsgVersion;        // always non-zero
-
-  void
-  _init(uint16_t msg_version)
-  {
-    _InNativeByteOrder = 1;
-    _MsgVersion        = msg_version;
-  }
-  ClusterMessageHeader() : _InNativeByteOrder(0), _MsgVersion(0) {}
-  ClusterMessageHeader(uint16_t msg_version) { _init(msg_version); }
-  int
-  MsgInNativeByteOrder()
-  {
-    return (_InNativeByteOrder == 1);
-  }
-  int
-  NeedByteSwap()
-  {
-    return (_InNativeByteOrder != 1);
-  }
-  int
-  GetMsgVersion()
-  {
-    if (NeedByteSwap()) {
-      return ats_swap16(_MsgVersion);
-    } else {
-      return _MsgVersion;
-    }
-  }
-};
-
-///////////////////////////////////////////////////////////////////
-
-//
-// cluster_ping
-//
-typedef void (*PingReturnFunction)(ClusterHandler *, void *data, int len);
-
-struct PingMessage : public ClusterMessageHeader {
-  PingReturnFunction fn; // Note: Pointer to a func
-  char data[1];          // start of data
-
-  enum {
-    MIN_VERSION          = 1,
-    MAX_VERSION          = 1,
-    PING_MESSAGE_VERSION = MAX_VERSION,
-  };
-
-  PingMessage(uint16_t vers = PING_MESSAGE_VERSION) : ClusterMessageHeader(vers), fn(nullptr) { data[0] = '\0'; }
-  /////////////////////////////////////////////////////////////////////////////
-  static int
-  protoToVersion(int protoMajor)
-  {
-    (void)protoMajor;
-    return PING_MESSAGE_VERSION;
-  }
-  static int
-  sizeof_fixedlen_msg()
-  {
-    PingMessage *p = 0;
-    // Maybe use offsetof here instead. /leif
-    return (uintptr_t)(&p->data[0]);
-  }
-  void
-  init(uint16_t vers = PING_MESSAGE_VERSION)
-  {
-    _init(vers);
-  }
-  inline void
-  SwapBytes()
-  {
-  } // No action, message is always reflected back
-  /////////////////////////////////////////////////////////////////////////////
-};
-
-inline void
-cluster_ping(ClusterHandler *ch, PingReturnFunction fn, void *data, int len)
-{
-  PingMessage *msg = (PingMessage *)alloca(PingMessage::sizeof_fixedlen_msg() + len);
-  msg->init();
-  msg->fn = fn;
-  memcpy(msg->data, data, len);
-  clusterProcessor.invoke_remote(ch, PING_CLUSTER_FUNCTION, (void *)msg, (msg->sizeof_fixedlen_msg() + len));
-}
-
-// filled with 0's
-extern char channel_dummy_output[DEFAULT_MAX_BUFFER_SIZE];
-
-//
-// Private (for testing)
-//
-ClusterConfiguration *configuration_add_machine(ClusterConfiguration *c, ClusterMachine *m);
-ClusterConfiguration *configuration_remove_machine(ClusterConfiguration *c, ClusterMachine *m);
-extern bool machineClusterHash;
-extern bool boundClusterHash;
-extern bool randClusterHash;
-
-void build_cluster_hash_table(ClusterConfiguration *);
-
-inline void
-ClusterVC_enqueue_read(Queue<ClusterVConnectionBase, ClusterVConnectionBase::Link_read_link> &q, ClusterVConnectionBase *vc)
-{
-  ClusterVConnState *cs = &vc->read;
-  ink_assert(!cs->queue);
-  cs->queue = &q;
-  q.enqueue(vc);
-}
-
-inline void
-ClusterVC_enqueue_write(Queue<ClusterVConnectionBase, ClusterVConnectionBase::Link_write_link> &q, ClusterVConnectionBase *vc)
-{
-  ClusterVConnState *cs = &vc->write;
-  ink_assert(!cs->queue);
-  cs->queue = &q;
-  q.enqueue(vc);
-}
-
-inline void
-ClusterVC_remove_read(ClusterVConnectionBase *vc)
-{
-  ClusterVConnState *cs = &vc->read;
-  ink_assert(cs->queue);
-  ((Queue<ClusterVConnectionBase, ClusterVConnectionBase::Link_read_link> *)cs->queue)->remove(vc);
-  cs->queue = nullptr;
-}
-
-inline void
-ClusterVC_remove_write(ClusterVConnectionBase *vc)
-{
-  ClusterVConnState *cs = &vc->write;
-  ink_assert(cs->queue);
-  ((Queue<ClusterVConnectionBase, ClusterVConnectionBase::Link_write_link> *)cs->queue)->remove(vc);
-  cs->queue = nullptr;
-}
-
-#endif /* _Cluster_h */
diff --git a/iocore/cluster/P_ClusterCacheInternal.h b/iocore/cluster/P_ClusterCacheInternal.h
deleted file mode 100644
index 9b6cce9..0000000
--- a/iocore/cluster/P_ClusterCacheInternal.h
+++ /dev/null
@@ -1,857 +0,0 @@
-/** @file
-
-  A brief file description
-
-  @section license License
-
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
- */
-
-/****************************************************************************
-
-  ClusterCacheInternal.h
-****************************************************************************/
-
-#ifndef __P_CLUSTERCACHEINTERNAL_H__
-#define __P_CLUSTERCACHEINTERNAL_H__
-#include "P_ClusterCache.h"
-#include "I_OneWayTunnel.h"
-
-//
-// Compilation Options
-//
-#define CACHE_USE_OPEN_VIO 0 // EXPERIMENTAL: not fully tested
-#define DO_REPLICATION 0     // EXPERIMENTAL: not fully tested
-
-//
-// Constants
-//
-#define META_DATA_FAST_ALLOC_LIMIT 1
-#define CACHE_CLUSTER_TIMEOUT HRTIME_MSECONDS(5000)
-#define CACHE_RETRY_PERIOD HRTIME_MSECONDS(10)
-#define REMOTE_CONNECT_HASH (16 * 1024)
-
-//
-// Macros
-//
-#define FOLDHASH(_ip, _seq) (_seq % REMOTE_CONNECT_HASH)
-#define ALIGN_DOUBLE(_p) ((((uintptr_t)(_p)) + 7) & ~7)
-#define ALLOCA_DOUBLE(_sz) ALIGN_DOUBLE(alloca((_sz) + 8))
-
-//
-// Testing
-//
-#define TEST(_x)
-//#define TEST(_x) _x
-
-//#define TTEST(_x)
-// fprintf(stderr, _x " at: %d\n",
-//      ((unsigned int)(ink_get_hrtime()/HRTIME_MSECOND)) % 1000)
-#define TTEST(_x)
-
-//#define TIMEOUT_TEST(_x) _x
-#define TIMEOUT_TEST(_x)
-
-extern int cache_migrate_on_demand;
-extern int ET_CLUSTER;
-//
-// Compile time options.
-//
-// Only one of PROBE_LOCAL_CACHE_FIRST or PROBE_LOCAL_CACHE_LAST
-// should be set.  These indicate that the local cache should be
-// probed at this point regardless of the dedicated location of the
-// object.  Note, if the owning machine goes down the local machine
-// will be probed anyway.
-//
-#define PROBE_LOCAL_CACHE_FIRST DO_REPLICATION
-#define PROBE_LOCAL_CACHE_LAST false
-
-//
-// This continuation handles all cache cluster traffic, on both
-// sides (state machine client and cache server)
-//
-struct CacheContinuation;
-typedef int (CacheContinuation::*CacheContHandler)(int, void *);
-struct CacheContinuation : public Continuation {
-  enum {
-    MagicNo = 0x92183123,
-  };
-  int magicno;
-  void *callback_data;
-  void *callback_data_2;
-  INK_MD5 url_md5;
-  Event *timeout;
-  Action action;
-  ClusterMachine *target_machine;
-  int probe_depth;
-  ClusterMachine *past_probes[CONFIGURATION_HISTORY_PROBE_DEPTH];
-  ink_hrtime start_time;
-  ClusterMachine *from;
-  ClusterHandler *ch;
-  VConnection *cache_vc;
-  bool cache_read;
-  int result;       // return event code
-  int result_error; // error code associated with event
-  ClusterVCToken token;
-  unsigned int seq_number;
-  uint16_t cfl_flags; // Request flags; see CFL_XXX defines
-  CacheFragType frag_type;
-  int nbytes;
-  unsigned int target_ip;
-  int request_opcode;
-  bool request_purge;
-  bool local_lookup_only;
-  bool no_reply_message;
-  bool request_timeout; // timeout occurred before
-  //   op complete
-  bool expect_cache_callback;
-
-  // remove_and_delete() specific data
-  bool use_deferred_callback;
-
-  // open_read/write data
-
-  time_t pin_in_cache;
-
-  // setMsgBufferLen(), allocMsgBuffer() and freeMsgBuffer() data
-
-  Ptr<IOBufferData> rw_buf_msg;
-  int rw_buf_msg_len;
-
-  // open data
-
-  ClusterVConnection *read_cluster_vc;
-  ClusterVConnection *write_cluster_vc;
-  int cluster_vc_channel;
-  ClusterVCToken open_local_token;
-
-  // Readahead on open read specific data
-
-  int caller_buf_freebytes; // remote bufsize for
-  //  initial data
-  VIO *readahead_vio;
-  IOBufferReader *readahead_reader;
-  Ptr<IOBufferBlock> readahead_data;
-  bool have_all_data; // all object data in response
-
-  CacheHTTPInfo cache_vc_info;
-  OneWayTunnel *tunnel;
-  Ptr<ProxyMutex> tunnel_mutex;
-  CacheContinuation *tunnel_cont;
-  bool tunnel_closed;
-  Action *cache_action;
-  Event *lookup_open_write_vc_event;
-
-  // Incoming data generated from unmarshaling request/response ops
-
-  Arena ic_arena;
-  CacheHTTPHdr ic_request;
-  CacheHTTPHdr ic_response;
-  HttpConfigParams *http_config_params;
-  CacheHTTPInfo ic_old_info;
-  CacheHTTPInfo ic_new_info;
-  Ptr<IOBufferData> ic_hostname;
-  int ic_hostname_len;
-
-  // debugging
-  int cache_op_ClusterFunction;
-
-  int lookupEvent(int event, void *d);
-  int probeLookupEvent(int event, void *d);
-  int remoteOpEvent(int event, Event *e);
-  int replyLookupEvent(int event, void *d);
-  int replyOpEvent(int event, VConnection *vc);
-  int handleReplyEvent(int event, Event *e);
-  int callbackEvent(int event, Event *e);
-  int setupVCdataRead(int event, VConnection *vc);
-  int VCdataRead(int event, VIO *target_vio);
-  int setupReadWriteVC(int, VConnection *);
-  ClusterVConnection *lookupOpenWriteVC();
-  int lookupOpenWriteVCEvent(int, Event *);
-  int localVCsetupEvent(int event, ClusterVConnection *vc);
-  void insert_cache_callback_user(ClusterVConnection *, int, void *);
-  int insertCallbackEvent(int, Event *);
-  void callback_user(int result, void *d);
-  void defer_callback_result(int result, void *d);
-  int callbackResultEvent(int event, Event *e);
-  void setupReadBufTunnel(VConnection *, VConnection *);
-  int tunnelClosedEvent(int event, void *);
-  int remove_and_delete(int, Event *);
-
-  inline void
-  setMsgBufferLen(int l, IOBufferData *b = 0)
-  {
-    ink_assert(!rw_buf_msg);
-    ink_assert(rw_buf_msg_len == 0);
-
-    rw_buf_msg     = b;
-    rw_buf_msg_len = l;
-  }
-
-  inline int
-  getMsgBufferLen()
-  {
-    return rw_buf_msg_len;
-  }
-
-  inline void
-  allocMsgBuffer()
-  {
-    ink_assert(!rw_buf_msg);
-    ink_assert(rw_buf_msg_len);
-    if (rw_buf_msg_len <= DEFAULT_MAX_BUFFER_SIZE) {
-      rw_buf_msg = new_IOBufferData(buffer_size_to_index(rw_buf_msg_len, MAX_BUFFER_SIZE_INDEX));
-    } else {
-      rw_buf_msg = new_xmalloc_IOBufferData(ats_malloc(rw_buf_msg_len), rw_buf_msg_len);
-    }
-  }
-
-  inline char *
-  getMsgBuffer()
-  {
-    ink_assert(rw_buf_msg);
-    return rw_buf_msg->data();
-  }
-
-  inline IOBufferData *
-  getMsgBufferIOBData() const
-  {
-    return rw_buf_msg.get();
-  }
-
-  inline void
-  freeMsgBuffer()
-  {
-    if (rw_buf_msg) {
-      rw_buf_msg     = 0;
-      rw_buf_msg_len = 0;
-    }
-  }
-
-  inline void
-  free()
-  {
-    token.clear();
-
-    if (cache_vc_info.valid()) {
-      cache_vc_info.destroy();
-    }
-
-    if (ic_request.valid()) {
-      ic_request.clear();
-    }
-    if (ic_response.valid()) {
-      ic_response.clear();
-    }
-    if (ic_old_info.valid()) {
-      ic_old_info.destroy();
-    }
-    if (ic_new_info.valid()) {
-      ic_new_info.destroy();
-    }
-
-    if (http_config_params) {
-      HttpConfig::release(http_config_params);
-      http_config_params = 0;
-    }
-
-    ic_arena.reset();
-    freeMsgBuffer();
-
-    tunnel_mutex   = 0;
-    readahead_data = 0;
-    ic_hostname    = 0;
-  }
-
-  CacheContinuation()
-    : Continuation(nullptr),
-      magicno(MagicNo),
-      callback_data(0),
-      callback_data_2(0),
-      timeout(0),
-      target_machine(0),
-      probe_depth(0),
-      start_time(0),
-      cache_read(false),
-      result(0),
-      result_error(0),
-      seq_number(0),
-      cfl_flags(0),
-      frag_type(CACHE_FRAG_TYPE_NONE),
-      nbytes(0),
-      target_ip(0),
-      request_opcode(0),
-      request_purge(false),
-      local_lookup_only(0),
-      no_reply_message(0),
-      request_timeout(0),
-      expect_cache_callback(true),
-      use_deferred_callback(0),
-      pin_in_cache(0),
-      rw_buf_msg_len(0),
-      read_cluster_vc(0),
-      write_cluster_vc(0),
-      cluster_vc_channel(0),
-      caller_buf_freebytes(0),
-      readahead_vio(0),
-      readahead_reader(0),
-      have_all_data(false),
-      cache_vc_info(),
-      tunnel(0),
-      tunnel_cont(0),
-      tunnel_closed(0),
-      lookup_open_write_vc_event(0),
-      ic_arena(),
-      ic_request(),
-      ic_response(),
-      http_config_params(nullptr),
-      ic_old_info(),
-      ic_new_info(),
-      ic_hostname_len(0),
-      cache_op_ClusterFunction(0)
-  {
-    token.clear();
-    SET_HANDLER((CacheContHandler)&CacheContinuation::remoteOpEvent);
-  }
-
-  inline static bool
-  is_ClusterThread(EThread *et)
-  {
-    int etype = ET_CLUSTER;
-    int i;
-    for (i = 0; i < eventProcessor.n_threads_for_type[etype]; ++i) {
-      if (et == eventProcessor.eventthread[etype][i]) {
-        return true;
-      }
-    }
-    return false;
-  }
-
-  // Static class member functions
-  static int init();
-  static CacheContinuation *cacheContAllocator_alloc();
-  static void cacheContAllocator_free(CacheContinuation *);
-  inkcoreapi static Action *callback_failure(Action *, int, int, CacheContinuation *this_cc = 0);
-  static Action *do_remote_lookup(Continuation *, const CacheKey *, CacheContinuation *, CacheFragType, const char *, int);
-  inkcoreapi static Action *do_op(Continuation *, ClusterMachine *, void *, int, char *, int, int nbytes = -1, MIOBuffer *b = 0);
-  static int setup_local_vc(char *data, int data_len, CacheContinuation *cc, ClusterMachine *mp, Action **);
-  static void disposeOfDataBuffer(void *buf);
-  static int handleDisposeEvent(int event, CacheContinuation *cc);
-  static int32_t getObjectSize(VConnection *, int, CacheHTTPInfo *);
-};
-
-/////////////////////////////////////////
-// Cache OP specific args for do_op()  //
-/////////////////////////////////////////
-
-// Bit definitions for cfl_flags.
-// Note: Limited to 16 bits
-#define CFL_OVERWRITE_ON_WRITE (1 << 1)
-#define CFL_REMOVE_USER_AGENTS (1 << 2) // Historical, now unused
-#define CFL_REMOVE_LINK (1 << 3)        // Historical, now unused
-#define CFL_LOPENWRITE_HAVE_OLDINFO (1 << 4)
-#define CFL_ALLOW_MULTIPLE_WRITES (1 << 5)
-#define CFL_MAX (1 << 15)
-
-struct CacheOpArgs_General {
-  const INK_MD5 *url_md5;
-  time_t pin_in_cache; // open_write() specific arg
-  CacheFragType frag_type;
-  uint16_t cfl_flags;
-
-  CacheOpArgs_General() : url_md5(nullptr), pin_in_cache(0), frag_type(CACHE_FRAG_TYPE_NONE), cfl_flags(0) {}
-};
-
-struct CacheOpArgs_Link {
-  INK_MD5 *from;
-  INK_MD5 *to;
-  uint16_t cfl_flags; // see CFL_XXX defines
-  CacheFragType frag_type;
-
-  CacheOpArgs_Link() : from(nullptr), to(nullptr), cfl_flags(0), frag_type(CACHE_FRAG_TYPE_NONE) {}
-};
-
-struct CacheOpArgs_Deref {
-  INK_MD5 *md5;
-  uint16_t cfl_flags; // see CFL_XXX defines
-  CacheFragType frag_type;
-
-  CacheOpArgs_Deref() : md5(nullptr), cfl_flags(0), frag_type(CACHE_FRAG_TYPE_NONE) {}
-};
-
-///////////////////////////////////
-// Over the wire message formats //
-///////////////////////////////////
-struct CacheLookupMsg : public ClusterMessageHeader {
-  INK_MD5 url_md5;
-  uint32_t seq_number;
-  uint32_t frag_type;
-  Alias32 moi;
-  enum {
-    MIN_VERSION                  = 1,
-    MAX_VERSION                  = 1,
-    CACHE_LOOKUP_MESSAGE_VERSION = MAX_VERSION,
-  };
-  CacheLookupMsg(uint16_t vers = CACHE_LOOKUP_MESSAGE_VERSION) : ClusterMessageHeader(vers), seq_number(0), frag_type(0)
-  {
-    moi.u32 = 0;
-  }
-
-  //////////////////////////////////////////////////////////////////////////
-  static int
-  protoToVersion(int protoMajor)
-  {
-    (void)protoMajor;
-    return CACHE_LOOKUP_MESSAGE_VERSION;
-  }
-  static int
-  sizeof_fixedlen_msg()
-  {
-    return (int)ALIGN_DOUBLE(offsetof(CacheLookupMsg, moi));
-  }
-  void
-  init(uint16_t vers = CACHE_LOOKUP_MESSAGE_VERSION)
-  {
-    _init(vers);
-  }
-  inline void
-  SwapBytes()
-  {
-    if (NeedByteSwap()) {
-      ink_release_assert(!"No byte swap for INK_MD5");
-      ats_swap32(&seq_number);
-      ats_swap32(&frag_type);
-    }
-  }
-  //////////////////////////////////////////////////////////////////////////
-};
-
-struct CacheOpMsg_long : public ClusterMessageHeader {
-  uint8_t opcode;
-  uint8_t frag_type;
-  uint16_t cfl_flags; // see CFL_XXX defines
-  INK_MD5 url_md5;
-  uint32_t seq_number;
-  uint32_t nbytes;
-  uint32_t data;   // used by open_write()
-  int32_t channel; // used by open interfaces
-  ClusterVCToken token;
-  int32_t buffer_size; // used by open read interface
-  Alias32 moi;
-  enum {
-    MIN_VERSION                   = 1,
-    MAX_VERSION                   = 1,
-    CACHE_OP_LONG_MESSAGE_VERSION = MAX_VERSION,
-  };
-  CacheOpMsg_long(uint16_t vers = CACHE_OP_LONG_MESSAGE_VERSION)
-    : ClusterMessageHeader(vers),
-      opcode(0),
-      frag_type(0),
-      cfl_flags(0),
-      seq_number(0),
-      nbytes(0),
-      data(0),
-      channel(0),
-      buffer_size(0)
-  {
-    moi.u32 = 0;
-  }
-
-  //////////////////////////////////////////////////////////////////////////
-  static int
-  protoToVersion(int protoMajor)
-  {
-    (void)protoMajor;
-    return CACHE_OP_LONG_MESSAGE_VERSION;
-  }
-  static int
-  sizeof_fixedlen_msg()
-  {
-    return (int)ALIGN_DOUBLE(offsetof(CacheOpMsg_long, moi));
-  }
-  void
-  init(uint16_t vers = CACHE_OP_LONG_MESSAGE_VERSION)
-  {
-    _init(vers);
-  }
-  inline void
-  SwapBytes()
-  {
-    if (NeedByteSwap()) {
-      ink_release_assert(!"No byte swap for INK_MD5");
-      ats_swap16(&cfl_flags);
-      ats_swap32(&seq_number);
-      ats_swap32(&nbytes);
-      ats_swap32(&data);
-      ats_swap32((uint32_t *)&channel);
-      token.SwapBytes();
-      ats_swap32((uint32_t *)&buffer_size);
-      ats_swap32((uint32_t *)&frag_type);
-    }
-  }
-  //////////////////////////////////////////////////////////////////////////
-};
-
-struct CacheOpMsg_short : public ClusterMessageHeader {
-  uint8_t opcode;
-  uint8_t frag_type;  // currently used by open_write() (low level)
-  uint16_t cfl_flags; // see CFL_XXX defines
-  INK_MD5 md5;
-  uint32_t seq_number;
-  uint32_t nbytes;
-  uint32_t data;        // currently used by open_write() (low level)
-  int32_t channel;      // used by open interfaces
-  ClusterVCToken token; // used by open interfaces
-  int32_t buffer_size;  // used by open read interface
-
-  // Variable portion of message
-  Alias32 moi;
-  enum {
-    MIN_VERSION                    = 1,
-    MAX_VERSION                    = 1,
-    CACHE_OP_SHORT_MESSAGE_VERSION = MAX_VERSION,
-  };
-  CacheOpMsg_short(uint16_t vers = CACHE_OP_SHORT_MESSAGE_VERSION)
-    : ClusterMessageHeader(vers),
-      opcode(0),
-      frag_type(0),
-      cfl_flags(0),
-      seq_number(0),
-      nbytes(0),
-      data(0),
-      channel(0),
-      buffer_size(0)
-  {
-    moi.u32 = 0;
-  }
-
-  //////////////////////////////////////////////////////////////////////////
-  static int
-  protoToVersion(int protoMajor)
-  {
-    (void)protoMajor;
-    return CACHE_OP_SHORT_MESSAGE_VERSION;
-  }
-  static int
-  sizeof_fixedlen_msg()
-  {
-    return (int)ALIGN_DOUBLE(offsetof(CacheOpMsg_short, moi));
-  }
-  void
-  init(uint16_t vers = CACHE_OP_SHORT_MESSAGE_VERSION)
-  {
-    _init(vers);
-  }
-  inline void
-  SwapBytes()
-  {
-    if (NeedByteSwap()) {
-      ink_release_assert(!"No byte swap for INK_MD5");
-      ats_swap16(&cfl_flags);
-      ats_swap32(&seq_number);
-      ats_swap32(&nbytes);
-      ats_swap32(&data);
-      if (opcode == CACHE_OPEN_READ) {
-        ats_swap32((uint32_t *)&buffer_size);
-        ats_swap32((uint32_t *)&channel);
-        token.SwapBytes();
-      }
-    }
-  }
-  //////////////////////////////////////////////////////////////////////////
-};
-
-struct CacheOpMsg_short_2 : public ClusterMessageHeader {
-  uint8_t opcode;
-  uint8_t frag_type;
-  uint16_t cfl_flags; // see CFL_XXX defines
-  INK_MD5 md5_1;
-  INK_MD5 md5_2;
-  uint32_t seq_number;
-  Alias32 moi;
-  enum {
-    MIN_VERSION                      = 1,
-    MAX_VERSION                      = 1,
-    CACHE_OP_SHORT_2_MESSAGE_VERSION = MAX_VERSION,
-  };
-  CacheOpMsg_short_2(uint16_t vers = CACHE_OP_SHORT_2_MESSAGE_VERSION)
-    : ClusterMessageHeader(vers), opcode(0), frag_type(0), cfl_flags(0), seq_number(0)
-  {
-    moi.u32 = 0;
-  }
-  //////////////////////////////////////////////////////////////////////////
-  static int
-  protoToVersion(int protoMajor)
-  {
-    (void)protoMajor;
-    return CACHE_OP_SHORT_2_MESSAGE_VERSION;
-  }
-  static int
-  sizeof_fixedlen_msg()
-  {
-    return (int)ALIGN_DOUBLE(offsetof(CacheOpMsg_short_2, moi));
-  }
-  void
-  init(uint16_t vers = CACHE_OP_SHORT_2_MESSAGE_VERSION)
-  {
-    _init(vers);
-  }
-  inline void
-  SwapBytes()
-  {
-    if (NeedByteSwap()) {
-      ink_release_assert(!"No byte swap for MD5_1");
-      ink_release_assert(!"No byte swap for MD5_2");
-      ats_swap16(&cfl_flags);
-      ats_swap32(&seq_number);
-    }
-  }
-  //////////////////////////////////////////////////////////////////////////
-};
-
-struct CacheOpReplyMsg : public ClusterMessageHeader {
-  uint32_t seq_number;
-  int32_t result;
-  ClusterVCToken token;
-  bool is_ram_cache_hit; // Entire object was from ram cache
-  Alias32 moi;           // Used by CACHE_OPEN_READ & CACHE_LINK reply
-  enum {
-    MIN_VERSION                    = 1,
-    MAX_VERSION                    = 1,
-    CACHE_OP_REPLY_MESSAGE_VERSION = MAX_VERSION,
-  };
-  CacheOpReplyMsg(uint16_t vers = CACHE_OP_REPLY_MESSAGE_VERSION)
-    : ClusterMessageHeader(vers), seq_number(0), result(0), is_ram_cache_hit(false)
-  {
-    moi.u32 = 0;
-  }
-
-  //////////////////////////////////////////////////////////////////////////
-  static int
-  protoToVersion(int protoMajor)
-  {
-    (void)protoMajor;
-    return CACHE_OP_REPLY_MESSAGE_VERSION;
-  }
-  static int
-  sizeof_fixedlen_msg()
-  {
-    return (int)ALIGN_DOUBLE(offsetof(CacheOpReplyMsg, moi));
-  }
-  void
-  init(uint16_t vers = CACHE_OP_REPLY_MESSAGE_VERSION)
-  {
-    _init(vers);
-  }
-  inline void
-  SwapBytes()
-  {
-    if (NeedByteSwap()) {
-      ats_swap32(&seq_number);
-      ats_swap32((uint32_t *)&result);
-      token.SwapBytes();
-    }
-  }
-  //////////////////////////////////////////////////////////////////////////
-};
-
-inline int
-maxval(int a, int b)
-{
-  return ((a > b) ? a : b);
-}
-
-inline int
-op_to_sizeof_fixedlen_msg(int op)
-{
-  switch (op) {
-  case CACHE_LOOKUP_OP: {
-    return CacheLookupMsg::sizeof_fixedlen_msg();
-  }
-  case CACHE_OPEN_WRITE_BUFFER:
-  case CACHE_OPEN_WRITE_BUFFER_LONG: {
-    ink_release_assert(!"op_to_sizeof_fixedlen_msg() op not supported");
-    return 0;
-  }
-  case CACHE_OPEN_WRITE:
-  case CACHE_OPEN_READ:
-  case CACHE_OPEN_READ_BUFFER: {
-    return CacheOpMsg_short::sizeof_fixedlen_msg();
-  }
-  case CACHE_OPEN_READ_LONG:
-  case CACHE_OPEN_READ_BUFFER_LONG:
-  case CACHE_OPEN_WRITE_LONG: {
-    return CacheOpMsg_long::sizeof_fixedlen_msg();
-  }
-  case CACHE_UPDATE:
-  case CACHE_REMOVE:
-  case CACHE_DEREF: {
-    return CacheOpMsg_short::sizeof_fixedlen_msg();
-  }
-  case CACHE_LINK: {
-    return CacheOpMsg_short_2::sizeof_fixedlen_msg();
-  }
-  default: {
-    ink_release_assert(!"op_to_sizeof_fixedlen_msg() unknown op");
-    return 0;
-  }
-  } // End of switch
-}
-
-//////////////////////////////////////////////////////////////////////////////
-
-static inline bool
-event_is_lookup(int event)
-{
-  switch (event) {
-  default:
-    return false;
-  case CACHE_EVENT_LOOKUP:
-  case CACHE_EVENT_LOOKUP_FAILED:
-    return true;
-  }
-}
-
-static inline bool
-event_is_open(int event)
-{
-  switch (event) {
-  default:
-    return false;
-  case CACHE_EVENT_OPEN_READ:
-  case CACHE_EVENT_OPEN_WRITE:
-    return true;
-  }
-}
-
-static inline bool
-op_is_read(int opcode)
-{
-  switch (opcode) {
-  case CACHE_OPEN_READ:
-  case CACHE_OPEN_READ_LONG:
-  case CACHE_OPEN_READ_BUFFER:
-  case CACHE_OPEN_READ_BUFFER_LONG:
-    return true;
-  default:
-    return false;
-  }
-}
-
-static inline bool
-op_is_shortform(int opcode)
-{
-  switch (opcode) {
-  case CACHE_OPEN_READ:
-  case CACHE_OPEN_READ_BUFFER:
-  case CACHE_OPEN_WRITE:
-  case CACHE_OPEN_WRITE_BUFFER:
-    return true;
-  default:
-    return false;
-  }
-}
-
-static inline int
-op_failure(int opcode)
-{
-  switch (opcode) {
-  case CACHE_OPEN_WRITE:
-  case CACHE_OPEN_WRITE_LONG:
-  case CACHE_OPEN_WRITE_BUFFER:
-  case CACHE_OPEN_WRITE_BUFFER_LONG:
-    return CACHE_EVENT_OPEN_WRITE_FAILED;
-
-  case CACHE_OPEN_READ:
-  case CACHE_OPEN_READ_LONG:
-  case CACHE_OPEN_READ_BUFFER:
-  case CACHE_OPEN_READ_BUFFER_LONG:
-    return CACHE_EVENT_OPEN_READ_FAILED;
-
-  case CACHE_UPDATE:
-    return CACHE_EVENT_UPDATE_FAILED;
-  case CACHE_REMOVE:
-    return CACHE_EVENT_REMOVE_FAILED;
-  case CACHE_LINK:
-    return CACHE_EVENT_LINK_FAILED;
-  case CACHE_DEREF:
-    return CACHE_EVENT_DEREF_FAILED;
-  }
-  return -1;
-}
-
-static inline int
-op_needs_marshalled_coi(int opcode)
-{
-  switch (opcode) {
-  case CACHE_OPEN_WRITE:
-  case CACHE_OPEN_WRITE_BUFFER:
-  case CACHE_OPEN_READ:
-  case CACHE_OPEN_READ_BUFFER:
-  case CACHE_REMOVE:
-  case CACHE_LINK:
-  case CACHE_DEREF:
-    return 0;
-
-  case CACHE_OPEN_WRITE_LONG:
-  case CACHE_OPEN_WRITE_BUFFER_LONG:
-  case CACHE_OPEN_READ_LONG:
-  case CACHE_OPEN_READ_BUFFER_LONG:
-  case CACHE_UPDATE:
-    return 0;
-
-  default:
-    return 0;
-  }
-}
-
-static inline int
-event_reply_may_have_moi(int event)
-{
-  switch (event) {
-  case CACHE_EVENT_OPEN_READ:
-  case CACHE_EVENT_LINK:
-  case CACHE_EVENT_LINK_FAILED:
-  case CACHE_EVENT_OPEN_READ_FAILED:
-  case CACHE_EVENT_OPEN_WRITE_FAILED:
-  case CACHE_EVENT_REMOVE_FAILED:
-  case CACHE_EVENT_UPDATE_FAILED:
-  case CACHE_EVENT_DEREF_FAILED:
-    return true;
-  default:
-    return false;
-  }
-}
-
-static inline int
-event_is_failure(int event)
-{
-  switch (event) {
-  case CACHE_EVENT_LOOKUP_FAILED:
-  case CACHE_EVENT_OPEN_READ_FAILED:
-  case CACHE_EVENT_OPEN_WRITE_FAILED:
-  case CACHE_EVENT_UPDATE_FAILED:
-  case CACHE_EVENT_REMOVE_FAILED:
-  case CACHE_EVENT_LINK_FAILED:
-  case CACHE_EVENT_DEREF_FAILED:
-    return true;
-  default:
-    return false;
-  }
-}
-
-#endif // __CLUSTERCACHEINTERNAL_H__
diff --git a/iocore/cluster/P_ClusterHandler.h b/iocore/cluster/P_ClusterHandler.h
deleted file mode 100644
index 7157ed1..0000000
--- a/iocore/cluster/P_ClusterHandler.h
+++ /dev/null
@@ -1,678 +0,0 @@
-/** @file
-
-  A brief file description
-
-  @section license License
-
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
- */
-
-/****************************************************************************
-
-  ClusterHandler.h
-****************************************************************************/
-
-#ifndef _P_ClusterHandler_h
-#define _P_ClusterHandler_h
-
-class ClusterLoadMonitor;
-
-struct ClusterCalloutContinuation;
-typedef int (ClusterCalloutContinuation::*ClstCoutContHandler)(int, void *);
-
-struct ClusterCalloutContinuation : public Continuation {
-  struct ClusterHandler *_ch;
-
-  int CalloutHandler(int event, Event *e);
-  ClusterCalloutContinuation(struct ClusterHandler *ch);
-  ~ClusterCalloutContinuation();
-};
-
-struct ClusterControl : public Continuation {
-  int len; // TODO: Should this be 64-bit ?
-  char size_index;
-  int64_t *real_data;
-  char *data;
-  void (*free_proc)(void *);
-  void *free_proc_arg;
-  Ptr<IOBufferBlock> iob_block;
-
-  IOBufferBlock *
-  get_block() const
-  {
-    return iob_block.get();
-  }
-
-  bool
-  fast_data() const
-  {
-    return (len <= MAX_FAST_CONTROL_MESSAGE);
-  }
-
-  bool
-  valid_alloc_data() const
-  {
-    return iob_block && real_data && data;
-  }
-
-  enum {
-    // DATA_HDR = size_index (1 byte) + magicno (1 byte) + sizeof(this)
-
-    DATA_HDR = (sizeof(int64_t) * 2) // must be multiple of sizeof(int64_t)
-  };
-
-  ClusterControl();
-  void real_alloc_data(int, bool);
-  void free_data();
-  virtual void freeall() = 0;
-};
-
-struct OutgoingControl : public ClusterControl {
-  ClusterHandler *ch;
-  ink_hrtime submit_time;
-
-  static OutgoingControl *alloc();
-
-  OutgoingControl();
-  void
-  alloc_data(bool align_int32_on_non_int64_boundary = true)
-  {
-    real_alloc_data(1, align_int32_on_non_int64_boundary); /* read access */
-  }
-
-  void
-  set_data(char *adata, int alen)
-  {
-    data          = adata;
-    len           = alen;
-    free_proc     = 0;
-    free_proc_arg = 0;
-    real_data     = 0;
-
-    // Create IOBufferBlock wrapper around passed data.
-
-    iob_block = new_IOBufferBlock();
-    iob_block->set_internal(adata, alen, BUFFER_SIZE_FOR_XMALLOC(alen));
-    iob_block->_buf_end = iob_block->end();
-  }
-
-  void
-  set_data(IOBufferBlock *buf, void (*free_data_proc)(void *), void *free_data_arg)
-  {
-    data          = buf->data->data();
-    len           = bytes_IOBufferBlockList(buf, 1); // read avail bytes
-    free_proc     = free_data_proc;
-    free_proc_arg = free_data_arg;
-    real_data     = 0;
-    iob_block     = buf;
-  }
-  int startEvent(int event, Event *e);
-  virtual void freeall();
-};
-
-//
-// incoming control messsage are received by this machine
-//
-struct IncomingControl : public ClusterControl {
-  ink_hrtime recognized_time;
-
-  static IncomingControl *alloc();
-
-  IncomingControl();
-  void
-  alloc_data(bool align_int32_on_non_int64_boundary = true)
-  {
-    real_alloc_data(0, align_int32_on_non_int64_boundary); /* write access */
-  }
-  virtual void freeall();
-};
-
-//
-// Interface structure for internal_invoke_remote()
-//
-struct invoke_remote_data_args {
-  int32_t magicno;
-  OutgoingControl *msg_oc;
-  OutgoingControl *data_oc;
-  int dest_channel;
-  ClusterVCToken token;
-
-  enum {
-    MagicNo = 0x04141998,
-  };
-  invoke_remote_data_args() : magicno(MagicNo), msg_oc(nullptr), data_oc(nullptr), dest_channel(0) {}
-};
-
-//
-// Descriptor of a chunk of a message (see Memo.ClusterIODesign)
-// This has been tested for aligment on the Sparc using TestDescriptor.cc
-//
-
-// type
-#define CLUSTER_SEND_FREE 0
-#define CLUSTER_SEND_DATA 1
-#define CLUSTER_SEQUENCE_NUMBER(_x) (((unsigned int)_x) & 0xFFFF)
-
-struct Descriptor { // Note: Over the Wire structure
-  uint32_t type : 1;
-  uint32_t channel : 15;
-  uint16_t sequence_number; // lower 16 bits of the ClusterVCToken.seq
-  uint32_t length;
-
-  inline void
-  SwapBytes()
-  {
-    ats_swap16((uint16_t *)this); // Hack
-    ats_swap16((uint16_t *)&sequence_number);
-    ats_swap32((uint32_t *)&length);
-  }
-};
-
-struct ClusterMsgHeader { // Note: Over the Wire structure
-  uint16_t count;
-  uint16_t descriptor_cksum;
-  uint16_t control_bytes_cksum;
-  uint16_t unused;
-  uint32_t control_bytes;
-  uint32_t count_check;
-
-  void
-  clear()
-  {
-    count               = 0;
-    descriptor_cksum    = 0;
-    control_bytes_cksum = 0;
-    unused              = 0;
-    control_bytes       = 0;
-    count_check         = 0;
-  }
-  ClusterMsgHeader() : count(0), descriptor_cksum(0), control_bytes_cksum(0), unused(0), control_bytes(0), count_check(0) {}
-  inline void
-  SwapBytes()
-  {
-    ats_swap16((uint16_t *)&count);
-    ats_swap16((uint16_t *)&descriptor_cksum);
-    ats_swap16((uint16_t *)&control_bytes_cksum);
-    ats_swap16((uint16_t *)&unused);
-    ats_swap32((uint32_t *)&control_bytes);
-    ats_swap32((uint32_t *)&count_check);
-  }
-};
-
-struct ClusterMsg {
-  Descriptor *descriptor;
-  Ptr<IOBufferBlock> iob_descriptor_block;
-  int count;
-  int control_bytes;
-  int descriptor_cksum;
-  int control_bytes_cksum;
-  int unused;
-  int state; // Only used by read to denote
-             //   read phase (count, descriptor, data)
-  Queue<OutgoingControl> outgoing_control;
-  Queue<OutgoingControl> outgoing_small_control;
-  Queue<OutgoingControl> outgoing_callout; // compound msg callbacks
-
-  // read processing usage.
-  int control_data_offset;
-  int did_small_control_set_data;
-  int did_large_control_set_data;
-  int did_small_control_msgs;
-  int did_large_control_msgs;
-  int did_freespace_msgs;
-
-  ClusterMsgHeader *
-  hdr()
-  {
-    return (ClusterMsgHeader *)(((char *)descriptor) - sizeof(ClusterMsgHeader));
-  }
-
-  IOBufferBlock *
-  get_block() const
-  {
-    return iob_descriptor_block.get();
-  }
-
-  IOBufferBlock *
-  get_block_header()
-  {
-    int start_offset;
-
-    start_offset = (char *)hdr() - iob_descriptor_block->buf();
-    iob_descriptor_block->reset();
-    iob_descriptor_block->next = 0;
-    iob_descriptor_block->fill(start_offset);
-    iob_descriptor_block->consume(start_offset);
-    return iob_descriptor_block.get();
-  }
-
-  IOBufferBlock *
-  get_block_descriptor()
-  {
-    int start_offset;
-
-    start_offset = ((char *)hdr() + sizeof(ClusterMsgHeader)) - iob_descriptor_block->buf();
-    iob_descriptor_block->reset();
-    iob_descriptor_block->next = 0;
-    iob_descriptor_block->fill(start_offset);
-    iob_descriptor_block->consume(start_offset);
-    return iob_descriptor_block.get();
-  }
-
-  void
-  clear()
-  {
-    hdr()->clear();
-    count               = 0;
-    control_bytes       = 0;
-    descriptor_cksum    = 0;
-    control_bytes_cksum = 0;
-    unused              = 0;
-    state               = 0;
-    outgoing_control.clear();
-    outgoing_small_control.clear();
-    control_data_offset        = 0;
-    did_small_control_set_data = 0;
-    did_large_control_set_data = 0;
-    did_small_control_msgs     = 0;
-    did_large_control_msgs     = 0;
-    did_freespace_msgs         = 0;
-  }
-  uint16_t
-  calc_control_bytes_cksum()
-  {
-    uint16_t cksum = 0;
-    char *p        = (char *)&descriptor[count];
-    char *endp     = p + control_bytes;
-    while (p < endp) {
-      cksum += *p;
-      ++p;
-    }
-    return cksum;
-  }
-  uint16_t
-  calc_descriptor_cksum()
-  {
-    uint16_t cksum = 0;
-    char *p        = (char *)&descriptor[0];
-    char *endp     = (char *)&descriptor[count];
-    while (p < endp) {
-      cksum += *p;
-      ++p;
-    }
-    return cksum;
-  }
-  ClusterMsg()
-    : descriptor(nullptr),
-      iob_descriptor_block(nullptr),
-      count(0),
-      control_bytes(0),
-      descriptor_cksum(0),
-      control_bytes_cksum(0),
-      unused(0),
-      state(0),
-      control_data_offset(0),
-      did_small_control_set_data(0),
-      did_large_control_set_data(0),
-      did_small_control_msgs(0),
-      did_large_control_msgs(0),
-      did_freespace_msgs(0)
-  {
-  }
-};
-
-//
-// State for a particular (read/write) direction of a cluster link
-//
-struct ClusterHandler;
-struct ClusterState : public Continuation {
-  ClusterHandler *ch;
-  bool read_channel;
-  bool do_iodone_event; // schedule_imm() on i/o complete
-  int n_descriptors;
-  ClusterMsg msg;
-  unsigned int sequence_number;
-  int to_do;             // # of bytes to transact
-  int did;               // # of bytes transacted
-  int n_iov;             // defined iov(s) in this operation
-  int io_complete;       // current i/o complete
-  int io_complete_event; // current i/o complete event
-  VIO *v;                // VIO associated with current op
-  int bytes_xfered;      // bytes xfered at last callback
-  int last_ndone;        // last do_io ndone
-  int total_bytes_xfered;
-  IOVec *iov; // io vector for readv, writev
-  Ptr<IOBufferData> iob_iov;
-
-  // Write byte bank structures
-  char *byte_bank;    // bytes buffered for transit
-  int n_byte_bank;    // number of bytes buffered for transit
-  int byte_bank_size; // allocated size of byte bank
-
-  int missed;
-  bool missed_msg;
-  ink_hrtime last_time;
-  ink_hrtime start_time;
-
-  Ptr<IOBufferBlock> block[MAX_TCOUNT];
-  class MIOBuffer *mbuf;
-  int state; // See enum defs below
-
-  enum {
-    READ_START = 1,
-    READ_HEADER,
-    READ_AWAIT_HEADER,
-    READ_SETUP_DESCRIPTOR,
-    READ_DESCRIPTOR,
-    READ_AWAIT_DESCRIPTOR,
-    READ_SETUP_DATA,
-    READ_DATA,
-    READ_AWAIT_DATA,
-    READ_POST_COMPLETE,
-    READ_COMPLETE
-  } read_state_t;
-
-  enum {
-    WRITE_START = 1,
-    WRITE_SETUP,
-    WRITE_INITIATE,
-    WRITE_AWAIT_COMPLETION,
-    WRITE_POST_COMPLETE,
-    WRITE_COMPLETE,
-  } write_state_t;
-
-  ClusterState(ClusterHandler *, bool);
-  ~ClusterState();
-  IOBufferData *get_data();
-  void build_do_io_vector();
-  int doIO();
-  int doIO_read_event(int, void *);
-  int doIO_write_event(int, void *);
-  void IOComplete();
-};
-
-//
-// ClusterHandlerBase superclass for processors with
-// bi-directional VConnections.
-//
-struct ClusterHandlerBase : public Continuation {
-  //
-  // Private
-  //
-  Queue<ClusterVConnectionBase, ClusterVConnection::Link_read_link> *read_vcs;
-  Queue<ClusterVConnectionBase, ClusterVConnection::Link_write_link> *write_vcs;
-  int cur_vcs;
-  int min_priority;
-  Event *trigger_event;
-
-  ClusterHandlerBase() : Continuation(nullptr), read_vcs(nullptr), write_vcs(NULL), cur_vcs(0), min_priority(1) {}
-};
-
-struct ClusterHandler : public ClusterHandlerBase {
-#ifdef MSG_TRACE
-  FILE *t_fd;
-#endif
-  NetVConnection *net_vc;
-  EThread *thread;
-  unsigned int ip;
-  int port;
-  char *hostname;
-  ClusterMachine *machine;
-  int ifd;
-  int id;
-  bool dead;
-  bool downing;
-
-  int32_t active; // handler currently running
-  bool on_stolen_thread;
-
-  struct ChannelData {
-    int channel_number;
-    LINK(ChannelData, link);
-  };
-
-  int n_channels;
-  ClusterVConnection **channels;
-  struct ChannelData **channel_data;
-  Queue<ChannelData> free_local_channels;
-
-  bool connector;
-  int cluster_connect_state; // see clcon_state_t enum
-  ClusterHelloMessage clusteringVersion;
-  ClusterHelloMessage nodeClusteringVersion;
-  bool needByteSwap;
-  int configLookupFails;
-
-#define CONFIG_LOOKUP_RETRIES 10
-
-  enum {
-    CLCON_INITIAL = 1,
-    CLCON_SEND_MSG,
-    CLCON_SEND_MSG_COMPLETE,
-    CLCON_READ_MSG,
-    CLCON_READ_MSG_COMPLETE,
-    CLCON_VALIDATE_MSG,
-    CLCON_CONN_BIND_CLEAR,
-    CLCON_CONN_BIND,
-    CLCON_CONN_BIND_OK,
-    CLCON_ABORT_CONNECT,
-    CLCON_DELETE_CONNECT
-  } clcon_state_t;
-
-  InkAtomicList outgoing_control_al[CLUSTER_CMSG_QUEUES];
-  InkAtomicList external_incoming_control;
-  InkAtomicList external_incoming_open_local;
-  ClusterCalloutContinuation *callout_cont[MAX_COMPLETION_CALLBACK_EVENTS];
-  Event *callout_events[MAX_COMPLETION_CALLBACK_EVENTS];
-  Event *cluster_periodic_event;
-  Queue<OutgoingControl> outgoing_control[CLUSTER_CMSG_QUEUES];
-  Queue<IncomingControl> incoming_control;
-  InkAtomicList read_vcs_ready;
-  InkAtomicList write_vcs_ready;
-  ClusterState read;
-  ClusterState write;
-
-  ink_hrtime current_time;
-  ink_hrtime last;
-  ink_hrtime last_report;
-  int n_since_last_report;
-  ink_hrtime last_cluster_op_enable;
-  ink_hrtime last_trace_dump;
-
-  DLL<ClusterVConnectionBase> delayed_reads;
-  ClusterLoadMonitor *clm;
-  bool disable_remote_cluster_ops;
-
-  // process_write() state data
-  int pw_write_descriptors_built;
-  int pw_freespace_descriptors_built;
-  int pw_controldata_descriptors_built;
-  int pw_time_expired;
-  bool started_on_stolen_thread;
-  bool control_message_write;
-
-#ifdef CLUSTER_STATS
-  Ptr<IOBufferBlock> message_blk;
-
-  int64_t _vc_writes;
-  int64_t _vc_write_bytes;
-  int64_t _control_write_bytes;
-  int _dw_missed_lock;
-  int _dw_not_enabled;
-  int _dw_wait_remote_fill;
-  int _dw_no_active_vio;
-  int _dw_not_enabled_or_no_write;
-  int _dw_set_data_pending;
-  int _dw_no_free_space;
-  int _fw_missed_lock;
-  int _fw_not_enabled;
-  int _fw_wait_remote_fill;
-  int _fw_no_active_vio;
-  int _fw_not_enabled_or_no_read;
-  int _process_read_calls;
-  int _n_read_start;
-  int _n_read_header;
-  int _n_read_await_header;
-  int _n_read_setup_descriptor;
-  int _n_read_descriptor;
-  int _n_read_await_descriptor;
-  int _n_read_setup_data;
-  int _n_read_data;
-  int _n_read_await_data;
-  int _n_read_post_complete;
-  int _n_read_complete;
-  int _process_write_calls;
-  int _n_write_start;
-  int _n_write_setup;
-  int _n_write_initiate;
-  int _n_write_await_completion;
-  int _n_write_post_complete;
-  int _n_write_complete;
-
-  void
-  clear_cluster_stats()
-  {
-    _vc_writes                  = 0;
-    _vc_write_bytes             = 0;
-    _control_write_bytes        = 0;
-    _dw_missed_lock             = 0;
-    _dw_not_enabled             = 0;
-    _dw_wait_remote_fill        = 0;
-    _dw_no_active_vio           = 0;
-    _dw_not_enabled_or_no_write = 0;
-    _dw_set_data_pending        = 0;
-    _dw_no_free_space           = 0;
-    _fw_missed_lock             = 0;
-    _fw_not_enabled             = 0;
-    _fw_wait_remote_fill        = 0;
-    _fw_no_active_vio           = 0;
-    _fw_not_enabled_or_no_read  = 0;
-    _process_read_calls         = 0;
-    _n_read_start               = 0;
-    _n_read_header              = 0;
-    _n_read_await_header        = 0;
-    _n_read_setup_descriptor    = 0;
-    _n_read_descriptor          = 0;
-    _n_read_await_descriptor    = 0;
-    _n_read_setup_data          = 0;
-    _n_read_data                = 0;
-    _n_read_await_data          = 0;
-    _n_read_post_complete       = 0;
-    _n_read_complete            = 0;
-    _process_write_calls        = 0;
-    _n_write_start              = 0;
-    _n_write_setup              = 0;
-    _n_write_initiate           = 0;
-    _n_write_await_completion   = 0;
-    _n_write_post_complete      = 0;
-    _n_write_complete           = 0;
-  }
-#endif // CLUSTER_STATS
-
-  ClusterHandler();
-  ~ClusterHandler();
-  bool check_channel(int c);
-  int alloc_channel(ClusterVConnection *vc, int requested_channel = 0);
-  void free_channel(ClusterVConnection *vc);
-  //
-  //  local_channel()
-  //  - Initiator node-node TCP socket  &&  Odd channel  => Local Channel
-  //  - !Initiator node-node TCP socket &&  Even channel => Local Channel
-  inline bool
-  local_channel(int i)
-  {
-    return !connector == !(i & 1);
-  }
-
-  void close_ClusterVConnection(ClusterVConnection *);
-  int cluster_signal_and_update(int event, ClusterVConnection *vc, ClusterVConnState *s);
-  int cluster_signal_and_update_locked(int event, ClusterVConnection *vc, ClusterVConnState *s);
-  int cluster_signal_error_and_update(ClusterVConnection *vc, ClusterVConnState *s, int lerrno);
-  void close_free_lock(ClusterVConnection *, ClusterVConnState *);
-
-#define CLUSTER_READ true
-#define CLUSTER_WRITE false
-
-  bool build_data_vector(char *, int, bool);
-  bool build_initial_vector(bool);
-
-  void add_to_byte_bank(ClusterVConnection *);
-  void update_channels_read();
-  int process_incoming_callouts(ProxyMutex *);
-  void update_channels_partial_read();
-  void process_set_data_msgs();
-  void process_small_control_msgs();
-  void process_large_control_msgs();
-  void process_freespace_msgs();
-  bool complete_channel_read(int, ClusterVConnection *vc);
-  void finish_delayed_reads();
-  // returns: false if the channel was closed
-
-  void update_channels_written();
-
-  int build_write_descriptors();
-  int build_freespace_descriptors();
-  int build_controlmsg_descriptors();
-  int add_small_controlmsg_descriptors();
-  int valid_for_data_write(ClusterVConnection *vc);
-  int valid_for_freespace_write(ClusterVConnection *vc);
-
-  int machine_down();
-  int remote_close(ClusterVConnection *vc, ClusterVConnState *ns);
-  void steal_thread(EThread *t);
-
-#define CLUSTER_FREE_ALL_LOCKS -1
-  void free_locks(bool read_flag, int i = CLUSTER_FREE_ALL_LOCKS);
-  bool get_read_locks();
-  bool get_write_locks();
-  int zombify(Event *e = nullptr); // optional event to use
-
-  int connectClusterEvent(int event, Event *e);
-  int startClusterEvent(int event, Event *e);
-  int mainClusterEvent(int event, Event *e);
-  int beginClusterEvent(int event, Event *e);
-  int zombieClusterEvent(int event, Event *e);
-  int protoZombieEvent(int event, Event *e);
-
-  void vcs_push(ClusterVConnection *vc, int type);
-  bool vc_ok_read(ClusterVConnection *);
-  bool vc_ok_write(ClusterVConnection *);
-  int do_open_local_requests();
-  void swap_descriptor_bytes();
-  int process_read(ink_hrtime);
-  int process_write(ink_hrtime, bool);
-
-  void dump_write_msg(int);
-  void dump_read_msg();
-  int compute_active_channels();
-  void dump_internal_data();
-
-#ifdef CLUSTER_IMMEDIATE_NETIO
-  void build_poll(bool);
-#endif
-};
-
-// Valid (ClusterVConnection *) in ClusterHandler.channels[]
-#define VALID_CHANNEL(vc) (vc && !(((uintptr_t)vc) & 1))
-
-// outgoing control continuations
-extern ClassAllocator<OutgoingControl> outControlAllocator;
-
-// incoming control descriptors
-extern ClassAllocator<IncomingControl> inControlAllocator;
-
-#endif /* _ClusterHandler_h */
diff --git a/iocore/cluster/P_ClusterInline.h b/iocore/cluster/P_ClusterInline.h
deleted file mode 100644
index fc7e54e..0000000
--- a/iocore/cluster/P_ClusterInline.h
+++ /dev/null
@@ -1,375 +0,0 @@
-/** @file
-
-  A brief file description
-
-  @section license License
-
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
- */
-
-/****************************************************************************
-
-  ClusterInline.h
-****************************************************************************/
-
-#ifndef __P_CLUSTERINLINE_H__
-#define __P_CLUSTERINLINE_H__
-#include "P_ClusterCacheInternal.h"
-#include "P_CacheInternal.h"
-#include "P_ClusterHandler.h"
-
-inline Action *
-Cluster_lookup(Continuation *cont, const CacheKey *key, CacheFragType frag_type, const char *hostname, int host_len)
-{
-  // Try to send remote, if not possible, handle locally
-  Action *retAct;
-  ClusterMachine *m = cluster_machine_at_depth(cache_hash(*key));
-  if (m && !clusterProcessor.disable_remote_cluster_ops(m)) {
-    CacheContinuation *cc = CacheContinuation::cacheContAllocator_alloc();
-    cc->action            = cont;
-    cc->mutex             = cont->mutex;
-    retAct                = CacheContinuation::do_remote_lookup(cont, key, cc, frag_type, hostname, host_len);
-    if (retAct) {
-      return retAct;
-    } else {
-      // not remote, do local lookup
-      CacheContinuation::cacheContAllocator_free(cc);
-      return (Action *)nullptr;
-    }
-  } else {
-    Action a;
-    a = cont;
-    return CacheContinuation::callback_failure(&a, CACHE_EVENT_LOOKUP_FAILED, 0);
-  }
-  return (Action *)nullptr;
-}
-
-inline Action *
-Cluster_read(ClusterMachine *owner_machine, int opcode, Continuation *cont, MIOBuffer *buf, CacheHTTPHdr *request,
-             const CacheKey *key, time_t pin_in_cache, CacheFragType frag_type, const char *hostname, int hostlen)
-{
-  if (clusterProcessor.disable_remote_cluster_ops(owner_machine)) {
-    Action a;
-    a = cont;
-    return CacheContinuation::callback_failure(&a, CACHE_EVENT_OPEN_READ_FAILED, 0);
-  }
-  int vers = CacheOpMsg_long::protoToVersion(owner_machine->msg_proto_major);
-  int flen;
-  int len = 0;
-  int res = 0;
-  char *msg;
-  char *data;
-
-  if (vers == CacheOpMsg_long::CACHE_OP_LONG_MESSAGE_VERSION) {
-    if ((opcode == CACHE_OPEN_READ_LONG) || (opcode == CACHE_OPEN_READ_BUFFER_LONG)) {
-      ink_assert(hostname);
-      ink_assert(hostlen);
-
-      // Determine length of data to Marshal
-      flen = op_to_sizeof_fixedlen_msg(opcode);
-
-      len += request->m_heap->marshal_length();
-      len += hostlen;
-
-      if ((flen + len) > DEFAULT_MAX_BUFFER_SIZE) // Bound marshalled data
-        goto err_exit;
-
-      // Perform data Marshal operation
-      msg  = (char *)ALLOCA_DOUBLE(flen + len);
-      data = msg + flen;
-
-      res = request->m_heap->marshal(data, len);
-      if (res < 0) {
-        goto err_exit;
-      }
-      data += res;
-      memcpy(data, hostname, hostlen);
-
-      CacheOpArgs_General readArgs;
-      readArgs.url_md5      = key;
-      readArgs.pin_in_cache = pin_in_cache;
-      readArgs.frag_type    = frag_type;
-      return CacheContinuation::do_op(cont, owner_machine, (void *)&readArgs, opcode, (char *)msg, (flen + len), -1, buf);
-    } else {
-      // Build message if we have host data.
-
-      if (hostlen) {
-        // Determine length of data to Marshal
-        flen = op_to_sizeof_fixedlen_msg(opcode);
-        len  = hostlen;
-
-        if ((flen + len) > DEFAULT_MAX_BUFFER_SIZE) // Bound marshalled data
-          goto err_exit;
-
-        msg  = (char *)ALLOCA_DOUBLE(flen + len);
-        data = msg + flen;
-        memcpy(data, hostname, hostlen);
-
-      } else {
-        msg  = 0;
-        flen = 0;
-        len  = 0;
-      }
-      CacheOpArgs_General readArgs;
-      readArgs.url_md5   = key;
-      readArgs.frag_type = frag_type;
-      return CacheContinuation::do_op(cont, owner_machine, (void *)&readArgs, opcode, (char *)msg, (flen + len), -1, buf);
-    }
-
-  } else {
-    //////////////////////////////////////////////////////////////
-    // Create the specified down rev version of this message
-    //////////////////////////////////////////////////////////////
-    ink_release_assert(!"CacheOpMsg_long [read] bad msg version");
-  }
-err_exit:
-  Action a;
-  a = cont;
-  return CacheContinuation::callback_failure(&a, CACHE_EVENT_OPEN_READ_FAILED, 0);
-}
-
-inline Action *
-Cluster_write(Continuation *cont, int expected_size, MIOBuffer *buf, ClusterMachine *m, const CacheKey *url_md5, CacheFragType ft,
-              int options, time_t pin_in_cache, int opcode, CacheHTTPHdr *request, CacheHTTPInfo *old_info, const char *hostname,
-              int hostlen)
-{
-  (void)request;
-  if (clusterProcessor.disable_remote_cluster_ops(m)) {
-    Action a;
-    a = cont;
-    return CacheContinuation::callback_failure(&a, CACHE_EVENT_OPEN_WRITE_FAILED, 0);
-  }
-  char *msg                 = 0;
-  char *data                = 0;
-  int allow_multiple_writes = 0;
-  int len                   = 0;
-  int flen                  = 0;
-  int vers                  = CacheOpMsg_long::protoToVersion(m->msg_proto_major);
-
-  switch (opcode) {
-  case CACHE_OPEN_WRITE: {
-    // Build message if we have host data
-    if (hostlen) {
-      // Determine length of data to Marshal
-      flen = op_to_sizeof_fixedlen_msg(CACHE_OPEN_WRITE);
-      len  = hostlen;
-
-      if ((flen + len) > DEFAULT_MAX_BUFFER_SIZE) // Bound marshalled data
-        goto err_exit;
-
-      msg  = (char *)ALLOCA_DOUBLE(flen + len);
-      data = msg + flen;
-
-      memcpy(data, hostname, hostlen);
-    }
-    break;
-  }
-  case CACHE_OPEN_WRITE_LONG: {
-    ink_assert(hostname);
-    ink_assert(hostlen);
-
-    // Determine length of data to Marshal
-    flen = op_to_sizeof_fixedlen_msg(CACHE_OPEN_WRITE_LONG);
-    len  = 0;
-
-    if (old_info == (CacheHTTPInfo *)CACHE_ALLOW_MULTIPLE_WRITES) {
-      old_info              = 0;
-      allow_multiple_writes = 1;
-    }
-    if (old_info) {
-      len += old_info->marshal_length();
-    }
-    len += hostlen;
-
-    if ((flen + len) > DEFAULT_MAX_BUFFER_SIZE) // Bound marshalled data
-      goto err_exit;
-
-    // Perform data Marshal operation
-    msg  = (char *)ALLOCA_DOUBLE(flen + len);
-    data = msg + flen;
-
-    if (old_info) {
-      int res = old_info->marshal(data, len);
-
-      if (res < 0) {
-        goto err_exit;
-      }
-      data += res;
-    }
-    memcpy(data, hostname, hostlen);
-    break;
-  }
-  default: {
-    ink_release_assert(!"open_write_internal invalid opcode.");
-  } // End of case
-  } // End of switch
-
-  if (vers == CacheOpMsg_long::CACHE_OP_LONG_MESSAGE_VERSION) {
-    // Do remote open_write()
-    CacheOpArgs_General writeArgs;
-    writeArgs.url_md5      = url_md5;
-    writeArgs.pin_in_cache = pin_in_cache;
-    writeArgs.frag_type    = ft;
-    writeArgs.cfl_flags |= (options & CACHE_WRITE_OPT_OVERWRITE ? CFL_OVERWRITE_ON_WRITE : 0);
-    writeArgs.cfl_flags |= (old_info ? CFL_LOPENWRITE_HAVE_OLDINFO : 0);
-    writeArgs.cfl_flags |= (allow_multiple_writes ? CFL_ALLOW_MULTIPLE_WRITES : 0);
-
-    return CacheContinuation::do_op(cont, m, (void *)&writeArgs, opcode, msg, flen + len, expected_size, buf);
-  } else {
-    //////////////////////////////////////////////////////////////
-    // Create the specified down rev version of this message
-    //////////////////////////////////////////////////////////////
-    ink_release_assert(!"CacheOpMsg_long [write] bad msg version");
-    return (Action *)0;
-  }
-
-err_exit:
-  Action a;
-  a = cont;
-  return CacheContinuation::callback_failure(&a, CACHE_EVENT_OPEN_WRITE_FAILED, 0);
-}
-
-inline Action *
-Cluster_link(ClusterMachine *m, Continuation *cont, CacheKey *from, CacheKey *to, CacheFragType type, char *hostname, int host_len)
-{
-  if (clusterProcessor.disable_remote_cluster_ops(m)) {
-    Action a;
-    a = cont;
-    return CacheContinuation::callback_failure(&a, CACHE_EVENT_LINK_FAILED, 0);
-  }
-
-  int vers = CacheOpMsg_short_2::protoToVersion(m->msg_proto_major);
-  if (vers == CacheOpMsg_short_2::CACHE_OP_SHORT_2_MESSAGE_VERSION) {
-    // Do remote link
-
-    // Allocate memory for message header
-    int flen = op_to_sizeof_fixedlen_msg(CACHE_LINK);
-    int len  = host_len;
-
-    if ((flen + len) > DEFAULT_MAX_BUFFER_SIZE) // Bound marshalled data
-      goto err_exit;
-
-    char *msg = (char *)ALLOCA_DOUBLE(flen + len);
-    memcpy((msg + flen), hostname, host_len);
-
-    // Setup args for remote link
-    CacheOpArgs_Link linkArgs;
-    linkArgs.from      = from;
-    linkArgs.to        = to;
-    linkArgs.frag_type = type;
-    return CacheContinuation::do_op(cont, m, (void *)&linkArgs, CACHE_LINK, msg, (flen + len));
-  } else {
-    //////////////////////////////////////////////////////////////
-    // Create the specified down rev version of this message
-    //////////////////////////////////////////////////////////////
-    ink_release_assert(!"CacheOpMsg_short_2 [CACHE_LINK] bad msg version");
-    return 0;
-  }
-
-err_exit:
-  Action a;
-  a = cont;
-  return CacheContinuation::callback_failure(&a, CACHE_EVENT_LINK_FAILED, 0);
-}
-
-inline Action *
-Cluster_deref(ClusterMachine *m, Continuation *cont, CacheKey *key, CacheFragType type, char *hostname, int host_len)
-{
-  if (clusterProcessor.disable_remote_cluster_ops(m)) {
-    Action a;
-    a = cont;
-    return CacheContinuation::callback_failure(&a, CACHE_EVENT_DEREF_FAILED, 0);
-  }
-
-  int vers = CacheOpMsg_short::protoToVersion(m->msg_proto_major);
-  if (vers == CacheOpMsg_short::CACHE_OP_SHORT_MESSAGE_VERSION) {
-    // Do remote deref
-
-    // Allocate memory for message header
-    int flen = op_to_sizeof_fixedlen_msg(CACHE_DEREF);
-    int len  = host_len;
-
-    if ((flen + len) > DEFAULT_MAX_BUFFER_SIZE) // Bound marshalled data
-      goto err_exit;
-
-    char *msg = (char *)ALLOCA_DOUBLE(flen + len);
-    memcpy((msg + flen), hostname, host_len);
-
-    // Setup args for remote deref
-    CacheOpArgs_Deref drefArgs;
-    drefArgs.md5       = key;
-    drefArgs.frag_type = type;
-    return CacheContinuation::do_op(cont, m, (void *)&drefArgs, CACHE_DEREF, msg, (flen + len));
-  } else {
-    //////////////////////////////////////////////////////////////
-    // Create the specified down rev version of this message
-    //////////////////////////////////////////////////////////////
-    ink_release_assert(!"CacheOpMsg_short [CACHE_DEREF] bad msg version");
-    return 0;
-  }
-
-err_exit:
-  Action a;
-  a = cont;
-  return CacheContinuation::callback_failure(&a, CACHE_EVENT_DEREF_FAILED, 0);
-}
-
-inline Action *
-Cluster_remove(ClusterMachine *m, Continuation *cont, const CacheKey *key, CacheFragType frag_type, const char *hostname,
-               int host_len)
-{
-  if (clusterProcessor.disable_remote_cluster_ops(m)) {
-    Action a;
-    a = cont;
-    return CacheContinuation::callback_failure(&a, CACHE_EVENT_REMOVE_FAILED, 0);
-  }
-
-  int vers = CacheOpMsg_short::protoToVersion(m->msg_proto_major);
-  if (vers == CacheOpMsg_short::CACHE_OP_SHORT_MESSAGE_VERSION) {
-    // Do remote update
-
-    // Allocate memory for message header
-    int flen = op_to_sizeof_fixedlen_msg(CACHE_REMOVE);
-    int len  = host_len;
-
-    if ((flen + len) > DEFAULT_MAX_BUFFER_SIZE) // Bound marshalled data
-      goto err_exit;
-
-    char *msg = (char *)ALLOCA_DOUBLE(flen + len);
-    memcpy((msg + flen), hostname, host_len);
-
-    // Setup args for remote update
-    CacheOpArgs_General updateArgs;
-    ink_zero(updateArgs);
-    updateArgs.url_md5   = key;
-    updateArgs.frag_type = frag_type;
-    return CacheContinuation::do_op(cont, m, (void *)&updateArgs, CACHE_REMOVE, msg, (flen + len));
-  } else {
-    //////////////////////////////////////////////////////////////
-    // Create the specified down rev version of this message
-    //////////////////////////////////////////////////////////////
-    ink_release_assert(!"CacheOpMsg_short [CACHE_REMOVE] bad msg version");
-    return (Action *)0;
-  }
-
-err_exit:
-  Action a;
-  a = cont;
-  return CacheContinuation::callback_failure(&a, CACHE_EVENT_REMOVE_FAILED, 0);
-}
-
-#endif /* __CLUSTERINLINE_H__ */
diff --git a/iocore/cluster/P_ClusterInternal.h b/iocore/cluster/P_ClusterInternal.h
deleted file mode 100644
index 28cfe55..0000000
--- a/iocore/cluster/P_ClusterInternal.h
+++ /dev/null
@@ -1,534 +0,0 @@
-/** @file
-
-  A brief file description
-
-  @section license License
-
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
- */
-
-/****************************************************************************
-
-  ClusterInternal.h
-****************************************************************************/
-#include "P_ClusterCache.h"
-
-#ifndef _P_ClusterInternal_h
-#define _P_ClusterInternal_h
-
-/*************************************************************************/
-// Compilation Options
-/*************************************************************************/
-#define CLUSTER_THREAD_STEALING 1
-#define CLUSTER_TOMCAT 1
-#define CLUSTER_STATS 1
-
-#define ALIGN_DOUBLE(_p) ((((uintptr_t)(_p)) + 7) & ~7)
-#define ALLOCA_DOUBLE(_sz) ALIGN_DOUBLE(alloca((_sz) + 8))
-
-/*************************************************************************/
-// Configuration Parameters
-/*************************************************************************/
-// Note: MAX_TCOUNT must be power of 2
-#define MAX_TCOUNT 128
-#define CONTROL_DATA (128 * 1024)
-#define READ_BANK_BUF_SIZE DEFAULT_MAX_BUFFER_SIZE
-#define READ_BANK_BUF_INDEX (DEFAULT_BUFFER_SIZES - 1)
-#define ALLOC_DATA_MAGIC 0xA5 // 8 bits in size
-#define READ_LOCK_SPIN_COUNT 1
-#define WRITE_LOCK_SPIN_COUNT 1
-
-// Unix specific optimizations
-// #define CLUSTER_IMMEDIATE_NETIO       1
-
-// (see ClusterHandler::mainClusterEvent)
-// this is equivalent to a max of 0.7 seconds
-#define CLUSTER_BUCKETS 64
-#define CLUSTER_PERIOD HRTIME_MSECONDS(10)
-
-// Per instance maximum time allotted to cluster thread
-#define CLUSTER_MAX_RUN_TIME HRTIME_MSECONDS(100)
-// Per instance maximum time allotted to thread stealing
-#define CLUSTER_MAX_THREAD_STEAL_TIME HRTIME_MSECONDS(10)
-
-// minimum number of channels to allocate
-#define MIN_CHANNELS 4096
-#define MAX_CHANNELS ((32 * 1024) - 1) // 15 bits in Descriptor
-
-#define CLUSTER_CONTROL_CHANNEL 0
-#define LAST_DEDICATED_CHANNEL 0
-
-#define CLUSTER_PHASES 1
-
-#define CLUSTER_INITIAL_PRIORITY CLUSTER_PHASES
-// how often to retry connect to machines which are supposed to be in the
-// cluster
-#define CLUSTER_BUMP_LENGTH 1
-#define CLUSTER_MEMBER_DELAY HRTIME_SECONDS(1)
-// How long to leave an unconnected ClusterVConnection waiting
-// Note: assumes (CLUSTER_CONNECT_TIMEOUT == 2 * CACHE_CLUSTER_TIMEOUT)
-#ifdef CLUSTER_TEST_DEBUG
-#define CLUSTER_CONNECT_TIMEOUT HRTIME_SECONDS(65536)
-#else
-#define CLUSTER_CONNECT_TIMEOUT HRTIME_SECONDS(10)
-#endif
-#define CLUSTER_CONNECT_RETRY HRTIME_MSECONDS(20)
-#define CLUSTER_RETRY HRTIME_MSECONDS(10)
-#define CLUSTER_DELAY_BETWEEN_WRITES HRTIME_MSECONDS(10)
-
-// Force close on cluster channel if no activity detected in this interval
-#ifdef CLUSTER_TEST_DEBUG
-#define CLUSTER_CHANNEL_INACTIVITY_TIMEOUT (65536 * HRTIME_SECONDS(60))
-#else
-#define CLUSTER_CHANNEL_INACTIVITY_TIMEOUT (10 * HRTIME_SECONDS(60))
-#endif
-
-// Defines for work deferred to ET_NET threads
-#define COMPLETION_CALLBACK_PERIOD HRTIME_MSECONDS(10)
-#define MAX_COMPLETION_CALLBACK_EVENTS 16
-
-// ClusterHandler::mainClusterEvent() thread active state
-#define CLUSTER_ACTIVE 1
-#define CLUSTER_NOT_ACTIVE 0
-
-// defines for ClusterHandler::remote_closed
-#define FORCE_CLOSE_ON_OPEN_CHANNEL -2
-
-// defines for machine_config_change()
-#define MACHINE_CONFIG 0
-#define CLUSTER_CONFIG 1
-
-// Debug interface category definitions
-#define CL_NOTE "cluster_note"
-#define CL_WARN "cluster_warn"
-#define CL_PROTO "cluster_proto"
-#define CL_TRACE "cluster_trace"
-
-/*************************************************************************/
-// Constants
-/*************************************************************************/
-#define MAX_FAST_CONTROL_MESSAGE 504                   // 512 - 4 (cluster func #) - 4 align
-#define SMALL_CONTROL_MESSAGE MAX_FAST_CONTROL_MESSAGE // copied instead
-                                                       //  of vectored
-#define WRITE_MESSAGE_ALREADY_BUILT -1
-
-#define MAGIC_COUNT(_x)                                                                                              \
-  (0xBADBAD ^ ~(uint32_t)_x.msg.count ^ ~(uint32_t)_x.msg.descriptor_cksum ^ ~(uint32_t)_x.msg.control_bytes_cksum ^ \
-   ~(uint32_t)_x.msg.unused ^ ~((uint32_t)_x.msg.control_bytes << 16) ^ _x.sequence_number)
-
-#define DOUBLE_ALIGN(_x) ((((uintptr_t)_x) + 7) & ~7)
-
-/*************************************************************************/
-// Testing Defines
-/*************************************************************************/
-#define MISS_TEST 0
-#define TEST_PARTIAL_WRITES 0
-#define TEST_PARTIAL_READS 0
-#define TEST_TIMING 0
-#define TEST_READ_LOCKS_MISSED 0
-#define TEST_WRITE_LOCKS_MISSED 0
-#define TEST_ENTER_EXIT 0
-#define TEST_ENTER_EXIT 0
-
-//
-// Timing testing
-//
-#if TEST_TIMING
-#define TTTEST(_x) fprintf(stderr, _x " at: %u\n", ((unsigned int)(ink_get_hrtime() / HRTIME_MSECOND)) % 1000)
-#define TTEST(_x) fprintf(stderr, _x " for: %d at: %u\n", vc->channel, ((unsigned int)(ink_get_hrtime() / HRTIME_MSECOND)) % 1000)
-#define TIMEOUT_TESTS(_s, _d)                                                \
-  if (*(int *)_d == 8)                                                       \
-    fprintf(stderr, _s " lookup  %d\n", *(int *)(_d + 20));                  \
-  else if (*(int *)_d == 10)                                                 \
-    fprintf(stderr, _s " op %d %d\n", *(int *)(_d + 36), *(int *)(_d + 40)); \
-  else if (*(int *)_d == 11)                                                 \
-  fprintf(stderr, _s " rop %d %d\n", *(int *)(_d + 4), *(int *)(_d + 8))
-#else
-#define TTTEST(_x)
-#define TTEST(_x)
-#define TIMEOUT_TESTS(_x, _y)
-#endif
-
-#if (TEST_READ_LOCKS_MISSED || TEST_WRITE_LOCKS_MISSED)
-static unsigned int test_cluster_locks_missed = 0;
-static test_cluster_lock_might_fail()
-{
-  return (!(rand_r(&test_cluster_locks_missed) % 13));
-}
-#endif
-#if TEST_READ_LOCKS_MISSED
-#define TEST_READ_LOCK_MIGHT_FAIL test_cluster_lock_might_fail()
-#else
-#define TEST_READ_LOCK_MIGHT_FAIL false
-#endif
-#if TEST_WRITE_LOCKS_MISSED
-#define TEST_WRITE_LOCK_MIGHT_FAIL test_cluster_lock_might_fail()
-#else
-#define TEST_WRITE_LOCK_MIGHT_FAIL false
-#endif
-
-#if TEST_ENTER_EXIT
-struct enter_exit_class {
-  int *outv;
-  enter_exit_class(int *in, int *out) : outv(out) { (*in)++; }
-  ~enter_exit_class() { (*outv)++; }
-};
-
-#define enter_exit(_x, _y) enter_exit_class a(_x, _y)
-#else
-#define enter_exit(_x, _y)
-#endif
-
-#define DOT_SEPARATED(_x) \
-  ((unsigned char *)&(_x))[0], ((unsigned char *)&(_x))[1], ((unsigned char *)&(_x))[2], ((unsigned char *)&(_x))[3]
-
-//
-// RPC message for CLOSE_CHANNEL_CLUSTER_FUNCTION
-//
-struct CloseMessage : public ClusterMessageHeader {
-  uint32_t channel;
-  int32_t status;
-  int32_t lerrno;
-  uint32_t sequence_number;
-
-  enum {
-    MIN_VERSION                = 1,
-    MAX_VERSION                = 1,
-    CLOSE_CHAN_MESSAGE_VERSION = MAX_VERSION,
-  };
-
-  CloseMessage(uint16_t vers = CLOSE_CHAN_MESSAGE_VERSION)
-    : ClusterMessageHeader(vers), channel(0), status(0), lerrno(0), sequence_number(0)
-  {
-  }
-  ////////////////////////////////////////////////////////////////////////////
-  static int
-  protoToVersion(int protoMajor)
-  {
-    (void)protoMajor;
-    return CLOSE_CHAN_MESSAGE_VERSION;
-  }
-  static int
-  sizeof_fixedlen_msg()
-  {
-    return sizeof(CloseMessage);
-  }
-  void
-  init(uint16_t vers = CLOSE_CHAN_MESSAGE_VERSION)
-  {
-    _init(vers);
-  }
-  inline void
-  SwapBytes()
-  {
-    if (NeedByteSwap()) {
-      ats_swap32(&channel);
-      ats_swap32((uint32_t *)&status);
-      ats_swap32((uint32_t *)&lerrno);
-      ats_swap32(&sequence_number);
-    }
-  }
-  ////////////////////////////////////////////////////////////////////////////
-};
-
-//
-// RPC message for MACHINE_LIST_CLUSTER_FUNCTION
-//
-struct MachineListMessage : public ClusterMessageHeader {
-  uint32_t n_ip;                     // Valid entries in ip[]
-  uint32_t ip[CLUSTER_MAX_MACHINES]; // variable length data
-
-  enum {
-    MIN_VERSION                  = 1,
-    MAX_VERSION                  = 1,
-    MACHINE_LIST_MESSAGE_VERSION = MAX_VERSION,
-  };
-
-  MachineListMessage() : ClusterMessageHeader(MACHINE_LIST_MESSAGE_VERSION), n_ip(0) { memset(ip, 0, sizeof(ip)); }
-  ////////////////////////////////////////////////////////////////////////////
-  static int
-  protoToVersion(int protoMajor)
-  {
-    (void)protoMajor;
-    return MACHINE_LIST_MESSAGE_VERSION;
-  }
-  static int
-  sizeof_fixedlen_msg()
-  {
-    return sizeof(ClusterMessageHeader);
-  }
-  void
-  init(uint16_t vers = MACHINE_LIST_MESSAGE_VERSION)
-  {
-    _init(vers);
-  }
-  inline void
-  SwapBytes()
-  {
-    ats_swap32(&n_ip);
-  }
-  ////////////////////////////////////////////////////////////////////////////
-};
-
-//
-// RPC message for SET_CHANNEL_DATA_CLUSTER_FUNCTION
-//
-struct SetChanDataMessage : public ClusterMessageHeader {
-  uint32_t channel;
-  uint32_t sequence_number;
-  uint32_t data_type; // enum CacheDataType
-  char data[4];
-
-  enum {
-    MIN_VERSION                      = 1,
-    MAX_VERSION                      = 1,
-    SET_CHANNEL_DATA_MESSAGE_VERSION = MAX_VERSION,
-  };
-
-  SetChanDataMessage(uint16_t vers = SET_CHANNEL_DATA_MESSAGE_VERSION)
-    : ClusterMessageHeader(vers), channel(0), sequence_number(0), data_type(0)
-  {
-    memset(data, 0, sizeof(data));
-  }
-  ////////////////////////////////////////////////////////////////////////////
-  static int
-  protoToVersion(int protoMajor)
-  {
-    (void)protoMajor;
-    return SET_CHANNEL_DATA_MESSAGE_VERSION;
-  }
-  static int
-  sizeof_fixedlen_msg()
-  {
-    SetChanDataMessage *p = 0;
-    return (int)DOUBLE_ALIGN((int64_t)((char *)&p->data[0] - (char *)p));
-  }
-  void
-  init(uint16_t vers = SET_CHANNEL_DATA_MESSAGE_VERSION)
-  {
-    _init(vers);
-  }
-  inline void
-  SwapBytes()
-  {
-    if (NeedByteSwap()) {
-      ats_swap32(&channel);
-      ats_swap32(&sequence_number);
-      ats_swap32(&data_type);
-    }
-  }
-  ////////////////////////////////////////////////////////////////////////////
-};
-
-//
-// RPC message for SET_CHANNEL_PIN_CLUSTER_FUNCTION
-//
-struct SetChanPinMessage : public ClusterMessageHeader {
-  uint32_t channel;
-  uint32_t sequence_number;
-  uint32_t pin_time;
-
-  enum {
-    MIN_VERSION                     = 1,
-    MAX_VERSION                     = 1,
-    SET_CHANNEL_PIN_MESSAGE_VERSION = MAX_VERSION,
-  };
-
-  SetChanPinMessage(uint16_t vers = SET_CHANNEL_PIN_MESSAGE_VERSION)
-    : ClusterMessageHeader(vers), channel(0), sequence_number(0), pin_time(0)
-  {
-  }
-  ////////////////////////////////////////////////////////////////////////////
-  static int
-  protoToVersion(int protoMajor)
-  {
-    (void)protoMajor;
-    return SET_CHANNEL_PIN_MESSAGE_VERSION;
-  }
-  static int
-  sizeof_fixedlen_msg()
-  {
-    return (int)sizeof(SetChanPinMessage);
-  }
-  void
-  init(uint16_t vers = SET_CHANNEL_PIN_MESSAGE_VERSION)
-  {
-    _init(vers);
-  }
-  inline void
-  SwapBytes()
-  {
-    if (NeedByteSwap()) {
-      ats_swap32(&channel);
-      ats_swap32(&sequence_number);
-      ats_swap32(&pin_time);
-    }
-  }
-  ////////////////////////////////////////////////////////////////////////////
-};
-
-//
-// RPC message for SET_CHANNEL_PRIORITY_CLUSTER_FUNCTION
-//
-struct SetChanPriorityMessage : public ClusterMessageHeader {
-  uint32_t channel;
-  uint32_t sequence_number;
-  uint32_t disk_priority;
-
-  enum {
-    MIN_VERSION                          = 1,
-    MAX_VERSION                          = 1,
-    SET_CHANNEL_PRIORITY_MESSAGE_VERSION = MAX_VERSION,
-  };
-
-  SetChanPriorityMessage(uint16_t vers = SET_CHANNEL_PRIORITY_MESSAGE_VERSION)
-    : ClusterMessageHeader(vers), channel(0), sequence_number(0), disk_priority(0)
-  {
-  }
-  ////////////////////////////////////////////////////////////////////////////
-  static int
-  protoToVersion(int protoMajor)
-  {
-    (void)protoMajor;
-    return SET_CHANNEL_PRIORITY_MESSAGE_VERSION;
-  }
-  static int
-  sizeof_fixedlen_msg()
-  {
-    return (int)sizeof(SetChanPriorityMessage);
-  }
-  void
-  init(uint16_t vers = SET_CHANNEL_PRIORITY_MESSAGE_VERSION)
-  {
-    _init(vers);
-  }
-  inline void
-  SwapBytes()
-  {
-    if (NeedByteSwap()) {
-      ats_swap32(&channel);
-      ats_swap32(&sequence_number);
-      ats_swap32(&disk_priority);
-    }
-  }
-  ////////////////////////////////////////////////////////////////////////////
-};
-
-inline void
-SetHighBit(int *val)
-{
-  *val |= (1 << ((sizeof(int) * 8) - 1));
-}
-
-inline void
-ClearHighBit(int *val)
-{
-  *val &= ~(1 << ((sizeof(int) * 8) - 1));
-}
-
-inline int
-IsHighBitSet(int *val)
-{
-  return (*val & (1 << ((sizeof(int) * 8) - 1)));
-}
-
-/////////////////////////////////////////////////////////////////
-// ClusterAccept -- Handle cluster connect events from peer
-//                  cluster nodes.
-/////////////////////////////////////////////////////////////////
-class ClusterAccept : public Continuation
-{
-public:
-  ClusterAccept(int *, int, int);
-  void Init();
-  void ShutdownDelete();
-  int ClusterAcceptEvent(int, void *);
-  int ClusterAcceptMachine(NetVConnection *);
-
-  ~ClusterAccept();
-
-private:
-  int *p_cluster_port;
-  int socket_send_bufsize;
-  int socket_recv_bufsize;
-  int current_cluster_port;
-  Action *accept_action;
-  Event *periodic_event;
-};
-
-// VC++ 5.0 special
-struct ClusterHandler;
-typedef int (ClusterHandler::*ClusterContHandler)(int, void *);
-
-struct OutgoingControl;
-typedef int (OutgoingControl::*OutgoingCtrlHandler)(int, void *);
-
-struct ClusterVConnection;
-typedef int (ClusterVConnection::*ClusterVConnHandler)(int, void *);
-
-// Library  declarations
-extern void cluster_set_priority(ClusterHandler *, ClusterVConnState *, int);
-extern void cluster_lower_priority(ClusterHandler *, ClusterVConnState *);
-extern void cluster_raise_priority(ClusterHandler *, ClusterVConnState *);
-extern void cluster_schedule(ClusterHandler *, ClusterVConnection *, ClusterVConnState *);
-extern void cluster_reschedule(ClusterHandler *, ClusterVConnection *, ClusterVConnState *);
-extern void cluster_reschedule_offset(ClusterHandler *, ClusterVConnection *, ClusterVConnState *, int);
-extern void cluster_disable(ClusterHandler *, ClusterVConnection *, ClusterVConnState *);
-extern void cluster_update_priority(ClusterHandler *, ClusterVConnection *, ClusterVConnState *, int64_t, int64_t);
-#define CLUSTER_BUMP_NO_REMOVE -1
-extern void cluster_bump(ClusterHandler *, ClusterVConnectionBase *, ClusterVConnState *, int);
-
-extern IOBufferBlock *clone_IOBufferBlockList(IOBufferBlock *, int, int, IOBufferBlock **);
-
-static inline IOBufferBlock *
-clone_IOBufferBlockList(Ptr<IOBufferBlock> &src, int start_off, int b, IOBufferBlock **b_tail)
-{
-  return clone_IOBufferBlockList(src.get(), start_off, b, b_tail);
-}
-
-extern IOBufferBlock *consume_IOBufferBlockList(IOBufferBlock *, int64_t);
-
-static inline IOBufferBlock *
-consume_IOBufferBlockList(Ptr<IOBufferBlock> &b, int64_t n)
-{
-  return consume_IOBufferBlockList(b.get(), n);
-}
-
-extern int64_t bytes_IOBufferBlockList(IOBufferBlock *, int64_t);
-
-static inline int64_t
-bytes_IOBufferBlockList(Ptr<IOBufferBlock> &b, int64_t read_avail_bytes)
-{
-  return bytes_IOBufferBlockList(b.get(), read_avail_bytes);
-}
-
-// ClusterVConnection declarations
-extern void clusterVCAllocator_free(ClusterVConnection *vc);
-extern ClassAllocator<ClusterVConnection> clusterVCAllocator;
-extern ClassAllocator<ByteBankDescriptor> byteBankAllocator;
-
-// Cluster configuration declarations
-extern int cluster_port;
-// extern void * machine_config_change(void *, void *);
-int machine_config_change(const char *, RecDataT, RecData, void *);
-extern void do_machine_config_change(void *, const char *);
-
-// Cluster API support functions
-extern void clusterAPI_init();
-extern void machine_online_APIcallout(int);
-extern void machine_offline_APIcallout(int);
-#endif /* _ClusterInternal_h */
diff --git a/iocore/cluster/P_ClusterLib.h b/iocore/cluster/P_ClusterLib.h
deleted file mode 100644
index 39e510b..0000000
--- a/iocore/cluster/P_ClusterLib.h
+++ /dev/null
@@ -1,70 +0,0 @@
-/** @file
-
-  A brief file description
-
-  @section license License
-
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
- */
-
-/****************************************************************************
-
-  ClusterLib.h
-
-
-****************************************************************************/
-
-#ifndef _P_ClusterLib_h
-#define _P_ClusterLib_h
-
-#include "P_Cluster.h"
-
-extern void cluster_set_priority(ClusterHandler *, ClusterVConnState *, int);
-extern void cluster_lower_priority(ClusterHandler *, ClusterVConnState *);
-extern void cluster_raise_priority(ClusterHandler *, ClusterVConnState *);
-extern void cluster_schedule(ClusterHandler *, ClusterVConnection *, ClusterVConnState *);
-extern void cluster_reschedule(ClusterHandler *, ClusterVConnection *, ClusterVConnState *);
-extern void cluster_disable(ClusterHandler *, ClusterVConnection *, ClusterVConnState *);
-extern void cluster_update_priority(ClusterHandler *, ClusterVConnection *, ClusterVConnState *, int64_t, int64_t);
-extern void cluster_bump(ClusterHandler *, ClusterVConnectionBase *, ClusterVConnState *, int);
-
-#if TEST_PARTIAL_READS
-extern int partial_readv(int, IOVec *, int, int);
-#endif
-
-#if TEST_PARTIAL_WRITES
-extern int partial_writev(int, IOVec *, int, int);
-#endif
-
-extern void dump_time_buckets();
-
-struct GlobalClusterPeriodicEvent;
-typedef int (GlobalClusterPeriodicEvent::*GClusterPEHandler)(int, void *);
-
-struct GlobalClusterPeriodicEvent : public Continuation {
-  GlobalClusterPeriodicEvent();
-  ~GlobalClusterPeriodicEvent();
-  void init();
-  int calloutEvent(Event *e, void *data);
-
-  // Private data
-  Event *_thisCallout;
-};
-
-#endif /* _ClusterLib_h */
-
-// End of ClusterLib.h
diff --git a/iocore/cluster/P_ClusterLoadMonitor.h b/iocore/cluster/P_ClusterLoadMonitor.h
deleted file mode 100644
index c499c07..0000000
--- a/iocore/cluster/P_ClusterLoadMonitor.h
+++ /dev/null
@@ -1,116 +0,0 @@
-/** @file
-
-  A brief file description
-
-  @section license License
-
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
- */
-
-/****************************************************************************
-
-  ClusterLoadMonitor.h
-****************************************************************************/
-
-#ifndef _P_ClusterLoadMonitor_h
-#define _P_ClusterLoadMonitor_h
-
-#include "P_Cluster.h"
-
-//***************************************************************************
-// ClusterLoadMonitor class -- Compute cluster interconnect load metric
-//***************************************************************************
-class ClusterLoadMonitor : public Continuation
-{
-public:
-  /////////////////////////////////////
-  // Defined by records.config
-  /////////////////////////////////////
-  static int cf_monitor_enabled;
-  static int cf_ping_message_send_msec_interval;
-  static int cf_num_ping_response_buckets;
-  static int cf_msecs_per_ping_response_bucket;
-  static int cf_ping_latency_threshold_msecs;
-  static int cf_cluster_load_compute_msec_interval;
-  static int cf_cluster_periodic_msec_interval;
-  static int cf_ping_history_buf_length;
-  static int cf_cluster_load_clear_duration;
-  static int cf_cluster_load_exceed_duration;
-
-  struct cluster_load_ping_msg {
-    int magicno;
-    int version;
-    int sequence_number;
-    ink_hrtime send_time;
-    ClusterLoadMonitor *monitor;
-
-    enum {
-      CL_MSG_MAGICNO = 0x12ABCDEF,
-      CL_MSG_VERSION = 1,
-    };
-    cluster_load_ping_msg(ClusterLoadMonitor *m = 0)
-      : magicno(CL_MSG_MAGICNO), version(CL_MSG_VERSION), sequence_number(0), send_time(0), monitor(m)
-    {
-    }
-  };
-
-  static void cluster_load_ping_rethandler(ClusterHandler *, void *, int);
-
-public:
-  ClusterLoadMonitor(ClusterHandler *ch);
-  void init();
-  ~ClusterLoadMonitor();
-  void cancel_monitor();
-  bool is_cluster_overloaded();
-
-private:
-  void compute_cluster_load();
-  void note_ping_response_time(ink_hrtime, int);
-  void recv_cluster_load_msg(cluster_load_ping_msg *);
-  void send_cluster_load_msg(ink_hrtime);
-  int cluster_load_periodic(int, Event *);
-
-private:
-  ////////////////////////////////////////////////////
-  // Copy of global configuration (records.config)
-  ////////////////////////////////////////////////////
-  int ping_message_send_msec_interval;
-  int num_ping_response_buckets;
-  int msecs_per_ping_response_bucket;
-  int ping_latency_threshold_msecs;
-  int cluster_load_compute_msec_interval;
-  int cluster_periodic_msec_interval;
-  int ping_history_buf_length;
-  int cluster_load_clear_duration;
-  int cluster_load_exceed_duration;
-
-  // Class specific data
-  ClusterHandler *ch;
-  int *ping_response_buckets;
-  ink_hrtime *ping_response_history_buf;
-  int ping_history_buf_head;
-  Action *periodic_action;
-
-  int cluster_overloaded;
-  int cancel_periodic;
-  ink_hrtime last_ping_message_sent;
-  ink_hrtime last_cluster_load_compute;
-  int cluster_load_msg_sequence_number;
-  int cluster_load_msg_start_sequence_number;
-};
-
-#endif /* _ClusterLoadMonitor_h */
diff --git a/iocore/cluster/P_ClusterMachine.h b/iocore/cluster/P_ClusterMachine.h
deleted file mode 100644
index 64bf133..0000000
--- a/iocore/cluster/P_ClusterMachine.h
+++ /dev/null
@@ -1,131 +0,0 @@
-/** @file
-
-  A brief file description
-
-  @section license License
-
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
- */
-
-/****************************************************************************
-
-  Machine.h
- ****************************************************************************/
-
-//
-//  The Machine is the set of processes which share part of an
-//  address space.
-//
-
-#ifndef _P_ClusterMachine_h
-#define _P_ClusterMachine_h
-
-//
-// Timeout the Machine * this amount of time after they
-// fall out of the current configuration that are deleted.
-//
-#define MACHINE_TIMEOUT (HRTIME_DAY * 2)
-
-//
-//  This is the time processors should delay before freeing up resouces
-//  which are shared with other threads in non-long running operations.
-//  For example, a Machine * is returned by the hash and used to do
-//  a remote invoke.  For the pointer to remain valid (or be recognized as
-//  invalid) he resource should not be raclaimed for NO_RACE_DELAY.
-//
-//  Long running operations should use more sophisticated synchronization.
-//
-#define NO_RACE_DELAY HRTIME_HOUR // a long long time
-
-struct ClusterHandler; // Leave this a class - VC++ gets very anal  ~SR -- which version of VC++? ~igalic
-
-struct ClusterMachine : public Server {
-  bool dead;
-  char *hostname;
-  int hostname_len;
-  //
-  // The network address of the current machine,
-  // stored in network byte order
-  //
-  unsigned int ip;
-  int cluster_port;
-  int num_connections;
-  int now_connections;
-  int free_connections;
-  int64_t rr_count;
-
-  Link<ClusterMachine> link;
-
-  // default for localhost
-  ClusterMachine(char *hostname = nullptr, unsigned int ip = 0, int acluster_port = 0);
-  ~ClusterMachine();
-  ClusterHandler *pop_ClusterHandler(int no_rr = 0);
-
-  // Cluster message protocol version
-  uint16_t msg_proto_major;
-  uint16_t msg_proto_minor;
-
-  // Private data for ClusterProcessor
-  //
-  ClusterHandler **clusterHandlers;
-};
-
-struct MachineListElement {
-  unsigned int ip;
-  int port;
-};
-
-struct MachineList {
-  int n;
-  MachineListElement machine[1];
-  MachineListElement *
-  find(unsigned int ip, int port = 0)
-  {
-    for (int i = 0; i < n; i++)
-      if (machine[i].ip == ip && (!port || machine[i].port == port))
-        return &machine[i];
-    return nullptr;
-  }
-};
-
-MachineList *read_MachineList(const char *filename, int test_fd = -1);
-void free_MachineList(MachineList *l);
-
-struct clusterConfigFile {
-  char *
-  parseFile(int fd)
-  {
-    return (char *)read_MachineList(nullptr, fd);
-  }
-};
-
-inkcoreapi ClusterMachine *this_cluster_machine();
-void create_this_cluster_machine();
-
-void free_ClusterMachine(ClusterMachine *m);
-
-MachineList *the_cluster_machines_config();
-MachineList *the_cluster_config();
-extern ProxyMutex *the_cluster_config_mutex;
-
-//
-// Private
-//
-extern MachineList *machines_config;
-extern MachineList *cluster_config;
-
-#endif /* _Machine_h */
diff --git a/iocore/cluster/P_TimeTrace.h b/iocore/cluster/P_TimeTrace.h
deleted file mode 100644
index 1bf3c89..0000000
--- a/iocore/cluster/P_TimeTrace.h
+++ /dev/null
@@ -1,83 +0,0 @@
-/** @file
-
-  A brief file description
-
-  @section license License
-
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
- */
-
-/****************************************************************************
-
-  TimeTrace.h
- ****************************************************************************/
-
-#ifndef _P_TimeTrace_h_
-#define _P_TimeTrace_h_
-
-// #define ENABLE_TIME_TRACE
-
-#define TIME_DIST_BUCKETS 500
-#define TIME_DIST_BUCKETS_SIZE TIME_DIST_BUCKETS + 1
-
-#ifdef ENABLE_TIME_TRACE
-extern int cdb_callback_time_dist[TIME_DIST_BUCKETS_SIZE];
-extern int cdb_cache_callbacks;
-
-extern int callback_time_dist[TIME_DIST_BUCKETS_SIZE];
-extern int cache_callbacks;
-
-extern int rmt_callback_time_dist[TIME_DIST_BUCKETS_SIZE];
-extern int rmt_cache_callbacks;
-
-extern int lkrmt_callback_time_dist[TIME_DIST_BUCKETS_SIZE];
-extern int lkrmt_cache_callbacks;
-
-extern int cntlck_acquire_time_dist[TIME_DIST_BUCKETS_SIZE];
-extern int cntlck_acquire_events;
-
-extern int immediate_events_time_dist[TIME_DIST_BUCKETS_SIZE];
-extern int cnt_immediate_events;
-
-extern int inmsg_time_dist[TIME_DIST_BUCKETS_SIZE];
-extern int inmsg_events;
-
-extern int open_delay_time_dist[TIME_DIST_BUCKETS_SIZE];
-extern int open_delay_events;
-
-extern int cluster_send_time_dist[TIME_DIST_BUCKETS_SIZE];
-extern int cluster_send_events;
-#endif // ENABLE_TIME_TRACE
-
-#ifdef ENABLE_TIME_TRACE
-#define LOG_EVENT_TIME(_start_time, _time_dist, _time_cnt)           \
-  do {                                                               \
-    ink_hrtime now      = ink_get_hrtime();                          \
-    unsigned int bucket = (now - _start_time) / HRTIME_MSECONDS(10); \
-    if (bucket > TIME_DIST_BUCKETS)                                  \
-      bucket = TIME_DIST_BUCKETS;                                    \
-    ink_atomic_increment(&_time_dist[bucket], 1);                    \
-    ink_atomic_increment(&_time_cnt, 1);                             \
-  } while (0)
-
-#else // !ENABLE_TIME_TRACE
-#define LOG_EVENT_TIME(_start_time, _time_dist, _time_cnt)
-#endif // !ENABLE_TIME_TRACE
-
-#endif // _TimeTrace_h_
-
-// End of TimeTrace.h
diff --git a/iocore/cluster/test_I_Cluster.cc b/iocore/cluster/test_I_Cluster.cc
deleted file mode 100644
index c05ef8e..0000000
--- a/iocore/cluster/test_I_Cluster.cc
+++ /dev/null
@@ -1,50 +0,0 @@
-/** @file
-
-  A brief file description
-
-  @section license License
-
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
- */
-
-/*
-  This is a bogus test file because it does not need any cluster
-  functions : the actual test file is in the cache directory.
-  This file is a placeholder because each module is supposed to
-  have a test file
-  */
-
-#include "P_Cluster.h"
-#include <iostream.h>
-#include <fstream.h>
-
-#include "diags.i"
-
-int
-main(int argc, char *argv[])
-{
-  int i;
-  int num_net_threads = ink_number_of_processors();
-  init_diags("", nullptr);
-  RecProcessInit(RECM_STAND_ALONE);
-  ink_event_system_init(EVENT_SYSTEM_MODULE_VERSION);
-  eventProcessor.start(num_net_threads);
-  RecProcessStart();
-  // ink_aio_init(AIO_MODULE_VERSION);
-
-  this_thread()->execute();
-}
diff --git a/iocore/cluster/test_P_Cluster.cc b/iocore/cluster/test_P_Cluster.cc
deleted file mode 100644
index c05ef8e..0000000
--- a/iocore/cluster/test_P_Cluster.cc
+++ /dev/null
@@ -1,50 +0,0 @@
-/** @file
-
-  A brief file description
-
-  @section license License
-
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
- */
-
-/*
-  This is a bogus test file because it does not need any cluster
-  functions : the actual test file is in the cache directory.
-  This file is a placeholder because each module is supposed to
-  have a test file
-  */
-
-#include "P_Cluster.h"
-#include <iostream.h>
-#include <fstream.h>
-
-#include "diags.i"
-
-int
-main(int argc, char *argv[])
-{
-  int i;
-  int num_net_threads = ink_number_of_processors();
-  init_diags("", nullptr);
-  RecProcessInit(RECM_STAND_ALONE);
-  ink_event_system_init(EVENT_SYSTEM_MODULE_VERSION);
-  eventProcessor.start(num_net_threads);
-  RecProcessStart();
-  // ink_aio_init(AIO_MODULE_VERSION);
-
-  this_thread()->execute();
-}
diff --git a/iocore/hostdb/HostDB.cc b/iocore/hostdb/HostDB.cc
index 7ead1e1..d28d760 100644
--- a/iocore/hostdb/HostDB.cc
+++ b/iocore/hostdb/HostDB.cc
@@ -47,8 +47,6 @@ HostDBProcessor::Options const HostDBProcessor::DEFAULT_OPTIONS;
 HostDBContinuation::Options const HostDBContinuation::DEFAULT_OPTIONS;
 int hostdb_enable                              = true;
 int hostdb_migrate_on_demand                   = true;
-int hostdb_cluster                             = false;
-int hostdb_cluster_round_robin                 = false;
 int hostdb_lookup_timeout                      = 30;
 int hostdb_insert_timeout                      = 160;
 int hostdb_re_dns_on_reload                    = false;
@@ -381,8 +379,6 @@ HostDBProcessor::start(int, size_t)
   REC_EstablishStaticConfigInt32(hostdb_migrate_on_demand, "proxy.config.hostdb.migrate_on_demand");
   REC_EstablishStaticConfigInt32(hostdb_strict_round_robin, "proxy.config.hostdb.strict_round_robin");
   REC_EstablishStaticConfigInt32(hostdb_timed_round_robin, "proxy.config.hostdb.timed_round_robin");
-  REC_EstablishStaticConfigInt32(hostdb_cluster, "proxy.config.hostdb.cluster");
-  REC_EstablishStaticConfigInt32(hostdb_cluster_round_robin, "proxy.config.hostdb.cluster.round_robin");
   REC_EstablishStaticConfigInt32(hostdb_lookup_timeout, "proxy.config.hostdb.lookup_timeout");
   REC_EstablishStaticConfigInt32U(hostdb_ip_timeout_interval, "proxy.config.hostdb.timeout");
   REC_EstablishStaticConfigInt32U(hostdb_ip_stale_interval, "proxy.config.hostdb.verify_after");
@@ -543,8 +539,7 @@ probe(ProxyMutex *mutex, HostDBMD5 const &md5, bool ignore_timeout)
   }
 
   // If the record is stale, but we want to revalidate-- lets start that up
-  if ((!ignore_timeout && r->is_ip_stale() && !cluster_machine_at_depth(master_hash(md5.hash)) && !r->reverse_dns) ||
-      (r->is_ip_timeout() && r->serve_stale_but_revalidate())) {
+  if ((!ignore_timeout && r->is_ip_stale() && !r->reverse_dns) || (r->is_ip_timeout() && r->serve_stale_but_revalidate())) {
     Debug("hostdb", "stale %u %u %u, using it and refreshing it", r->ip_interval(), r->ip_timestamp, r->ip_timeout_interval);
     HostDBContinuation *c = hostDBContAllocator.alloc();
     HostDBContinuation::Options copt;
@@ -1052,7 +1047,7 @@ HostDBContinuation::removeEvent(int /* event ATS_UNUSED */, Event *e)
 }
 
 // Lookup done, insert into the local table, return data to the
-// calling continuation or to the calling cluster node.
+// calling continuation.
 // NOTE: if "i" exists it means we already allocated the space etc, just return
 //
 HostDBInfo *
@@ -1150,8 +1145,6 @@ HostDBContinuation::lookup_done(IpAddr const &ip, const char *aname, bool around
   // set the "lookup_done" interval
   r->ip_timestamp = hostdb_current_interval;
 
-  if (from_cont)
-    do_put_response(from, r, from_cont);
   ink_assert(!r->round_robin || !r->reverse_dns);
   return r;
 }
@@ -1425,13 +1418,6 @@ HostDBContinuation::dnsEvent(int event, HostEnt *e)
 
     hostDB.refcountcache->put(md5.hash.fold(), r, allocSize, r->expiry_time());
 
-    // if we are not the owner, put on the owner
-    //
-    ClusterMachine *m = cluster_machine_at_depth(master_hash(md5.hash));
-
-    if (m)
-      do_put_response(m, r, nullptr);
-
     // try to callback the user
     //
     if (action.continuation) {
@@ -1463,136 +1449,6 @@ HostDBContinuation::dnsEvent(int event, HostEnt *e)
   }
 }
 
-//
-// HostDB Get Message
-// Used to lookup host information on a remote node in the cluster
-//
-struct HostDB_get_message {
-  INK_MD5 md5;
-  IpEndpoint ip;
-  Continuation *cont;
-  int namelen;
-  char name[MAXDNAME];
-};
-
-//
-// Make a get message
-//
-int
-HostDBContinuation::make_get_message(char *buf, int size)
-{
-  ink_assert(size >= (int)sizeof(HostDB_get_message));
-
-  HostDB_get_message *msg = reinterpret_cast<HostDB_get_message *>(buf);
-  msg->md5                = md5.hash;
-  ats_ip_set(&msg->ip.sa, md5.ip, htons(md5.port));
-  msg->cont = this;
-
-  // name
-  ink_strlcpy(msg->name, md5.host_name, sizeof(msg->name));
-
-  // length
-  int len = sizeof(HostDB_get_message) - MAXDNAME + md5.host_len + 1;
-
-  return len;
-}
-
-//
-// Make and send a get message
-//
-bool
-HostDBContinuation::do_get_response(Event * /* e ATS_UNUSED */)
-{
-  if (!hostdb_cluster)
-    return false;
-
-  // find an appropriate Machine
-  //
-  ClusterMachine *m = nullptr;
-
-  if (hostdb_migrate_on_demand) {
-    m = cluster_machine_at_depth(master_hash(md5.hash), &probe_depth, past_probes);
-  } else {
-    if (probe_depth)
-      return false;
-    m           = cluster_machine_at_depth(master_hash(md5.hash));
-    probe_depth = 1;
-  }
-
-  if (!m)
-    return false;
-
-  // Make message
-  //
-  HostDB_get_message msg;
-
-  memset(&msg, 0, sizeof(msg));
-  int len = make_get_message((char *)&msg, sizeof(HostDB_get_message));
-
-  // Setup this continuation, with a timeout
-  //
-  hostDB.remoteHostDBQueue[key_partition()].enqueue(this);
-  SET_HANDLER((HostDBContHandler)&HostDBContinuation::clusterEvent);
-  timeout = mutex->thread_holding->schedule_in(this, HOST_DB_CLUSTER_TIMEOUT);
-
-  // Send the message
-  //
-  clusterProcessor.invoke_remote(m->pop_ClusterHandler(), GET_HOSTINFO_CLUSTER_FUNCTION, (char *)&msg, len);
-
-  return true;
-}
-
-//
-// HostDB Put Message
-// This message is used in a response to a cluster node for
-// Host inforamation.
-//
-struct HostDB_put_message {
-  INK_MD5 md5;
-  IpEndpoint ip;
-  unsigned int ttl;
-  unsigned int missing : 1;
-  unsigned int round_robin : 1;
-  Continuation *cont;
-  unsigned int application1;
-  unsigned int application2;
-  int namelen;
-  char name[MAXDNAME];
-};
-
-//
-// Build the put message
-//
-int
-HostDBContinuation::make_put_message(HostDBInfo *r, Continuation *c, char *buf, int size)
-{
-  ink_assert(size >= (int)sizeof(HostDB_put_message));
-
-  HostDB_put_message *msg = reinterpret_cast<HostDB_put_message *>(buf);
-  memset(msg, 0, sizeof(HostDB_put_message));
-
-  msg->md5  = md5.hash;
-  msg->cont = c;
-  if (r) {
-    ats_ip_copy(&msg->ip.sa, r->ip());
-    msg->application1 = r->app.allotment.application1;
-    msg->application2 = r->app.allotment.application2;
-    msg->missing      = false;
-    msg->round_robin  = r->round_robin;
-    msg->ttl          = r->ip_time_remaining();
-  } else {
-    msg->missing = true;
-  }
-
-  // name
-  ink_strlcpy(msg->name, md5.host_name, sizeof(msg->name));
-
-  // length
-  int len = sizeof(HostDB_put_message) - MAXDNAME + md5.host_len + 1;
-
-  return len;
-}
-
 int
 HostDBContinuation::iterateEvent(int event, Event *e)
 {
@@ -1649,23 +1505,6 @@ HostDBContinuation::iterateEvent(int event, Event *e)
 }
 
 //
-// Build the put message and send it
-//
-void
-HostDBContinuation::do_put_response(ClusterMachine *m, HostDBInfo *r, Continuation *c)
-{
-  // don't remote fill round-robin DNS entries
-  // if configured not to cluster them
-  if (!hostdb_cluster || (!c && r->round_robin && !hostdb_cluster_round_robin))
-    return;
-
-  HostDB_put_message msg;
-  int len = make_put_message(r, c, (char *)&msg, sizeof(HostDB_put_message));
-
-  clusterProcessor.invoke_remote(m->pop_ClusterHandler(), PUT_HOSTINFO_CLUSTER_FUNCTION, (char *)&msg, len);
-}
-
-//
 // Probe state
 //
 int
@@ -1688,8 +1527,6 @@ HostDBContinuation::probeEvent(int /* event ATS_UNUSED */, Event *e)
   if (!hostdb_enable || (!*md5.host_name && !md5.ip.isValid())) {
     if (action.continuation)
       action.continuation->handleEvent(EVENT_HOST_DB_LOOKUP, nullptr);
-    if (from)
-      do_put_response(from, nullptr, from_cont);
     hostdb_cont_free(this);
     return EVENT_DONE;
   }
@@ -1705,21 +1542,12 @@ HostDBContinuation::probeEvent(int /* event ATS_UNUSED */, Event *e)
     if (action.continuation && r)
       reply_to_cont(action.continuation, r.get());
 
-    // Respond to any remote node
-    //
-    if (from)
-      do_put_response(from, r.get(), from_cont);
-
     // If it suceeds or it was a remote probe, we are done
     //
     if (r || from) {
       hostdb_cont_free(this);
       return EVENT_DONE;
     }
-    // If it failed, do a remote probe
-    //
-    if (do_get_response(e))
-      return EVENT_CONT;
   }
   // If there are no remote nodes to probe, do a DNS lookup
   //
@@ -1825,192 +1653,6 @@ HostDBContinuation::do_dns()
 }
 
 //
-// Handle the response (put message)
-//
-int
-HostDBContinuation::clusterResponseEvent(int /*  event ATS_UNUSED */, Event *e)
-{
-  if (from_cont) {
-    HostDBContinuation *c;
-    for (c = (HostDBContinuation *)hostDB.remoteHostDBQueue[key_partition()].head; c; c = (HostDBContinuation *)c->link.next)
-      if (c == from_cont)
-        break;
-
-    // Check to see that we have not already timed out
-    //
-    if (c) {
-      action    = c;
-      from_cont = nullptr;
-      MUTEX_TRY_LOCK(lock, c->mutex, e->ethread);
-      MUTEX_TRY_LOCK(lock2, c->action.mutex, e->ethread);
-      if (!lock.is_locked() || !lock2.is_locked()) {
-        e->schedule_in(HOST_DB_RETRY_PERIOD);
-        return EVENT_CONT;
-      }
-      bool failed = missing || (round_robin && !hostdb_cluster_round_robin);
-      action.continuation->handleEvent(EVENT_HOST_DB_GET_RESPONSE, failed ? nullptr : this);
-    }
-  } else {
-    action = nullptr;
-    // just a remote fill
-    ink_assert(!missing);
-    lookup_done(md5.ip, md5.host_name, false, ttl, nullptr);
-  }
-  hostdb_cont_free(this);
-  return EVENT_DONE;
-}
-
-//
-// Wait for the response (put message)
-//
-int
-HostDBContinuation::clusterEvent(int event, Event *e)
-{
-  // remove ourselves from the queue
-  //
-  hostDB.remoteHostDBQueue[key_partition()].remove(this);
-
-  switch (event) {
-  default:
-    ink_assert(!"bad case");
-    hostdb_cont_free(this);
-    return EVENT_DONE;
-
-  // handle the put response, e is really a HostDBContinuation *
-  //
-  case EVENT_HOST_DB_GET_RESPONSE:
-    if (timeout) {
-      timeout->cancel(this);
-      timeout = nullptr;
-    }
-    if (e) {
-      HostDBContinuation *c         = (HostDBContinuation *)e;
-      HostDBInfo *r                 = lookup_done(md5.ip, c->md5.host_name, false, c->ttl, nullptr);
-      r->app.allotment.application1 = c->app.allotment.application1;
-      r->app.allotment.application2 = c->app.allotment.application2;
-
-      HOSTDB_INCREMENT_DYN_STAT(hostdb_total_hits_stat);
-
-      if (!action.cancelled) {
-        if (reply_to_cont(action.continuation, r)) {
-          // if we are not the owner and neither was the sender,
-          // fill the owner
-          //
-          if (hostdb_migrate_on_demand) {
-            ClusterMachine *m = cluster_machine_at_depth(master_hash(md5.hash));
-            if (m && m != c->from)
-              do_put_response(m, r, nullptr);
-          }
-        }
-      }
-      hostdb_cont_free(this);
-      return EVENT_DONE;
-    }
-    return failed_cluster_request(e);
-
-  // did not get the put message in time
-  //
-  case EVENT_INTERVAL: {
-    MUTEX_TRY_LOCK_FOR(lock, action.mutex, e->ethread, action.continuation);
-    if (!lock.is_locked()) {
-      e->schedule_in(HOST_DB_RETRY_PERIOD);
-      return EVENT_CONT;
-    }
-    return failed_cluster_request(e);
-  }
-  }
-}
-
-int
-HostDBContinuation::failed_cluster_request(Event *e)
-{
-  if (action.cancelled) {
-    hostdb_cont_free(this);
-    return EVENT_DONE;
-  }
-  // Attempt another remote probe
-  //
-  if (do_get_response(e))
-    return EVENT_CONT;
-
-  // Otherwise, do a DNS lookup
-  //
-  do_dns();
-  return EVENT_DONE;
-}
-
-void
-get_hostinfo_ClusterFunction(ClusterHandler *ch, void *data, int /* len ATS_UNUSED */)
-{
-  HostDBMD5 md5;
-  HostDB_get_message *msg = (HostDB_get_message *)data;
-
-  md5.host_name = msg->name;
-  md5.host_len  = msg->namelen;
-  md5.ip.assign(&msg->ip.sa);
-  md5.port    = ats_ip_port_host_order(&msg->ip.sa);
-  md5.hash    = msg->md5;
-  md5.db_mark = db_mark_for(&msg->ip.sa);
-#ifdef SPLIT_DNS
-  SplitDNS *pSD  = nullptr;
-  char *hostname = msg->name;
-  if (hostname && SplitDNSConfig::isSplitDNSEnabled()) {
-    pSD = SplitDNSConfig::acquire();
-
-    if (nullptr != pSD) {
-      md5.dns_server = static_cast<DNSServer *>(pSD->getDNSRecord(hostname));
-    }
-    SplitDNSConfig::release(pSD);
-  }
-#endif // SPLIT_DNS
-
-  HostDBContinuation *c = hostDBContAllocator.alloc();
-  HostDBContinuation::Options copt;
-  SET_CONTINUATION_HANDLER(c, (HostDBContHandler)&HostDBContinuation::probeEvent);
-  c->from      = ch->machine;
-  c->from_cont = msg->cont;
-
-  /* -----------------------------------------
-     we make a big assumption here! we presume
-     that all the machines in the cluster are
-     set to use the same configuration for
-     DNS servers
-     ----------------------------------------- */
-
-  copt.host_res_style = host_res_style_for(&msg->ip.sa);
-  c->init(md5, copt);
-  c->mutex        = hostDB.refcountcache->lock_for_key(msg->md5.fold());
-  c->action.mutex = c->mutex;
-  dnsProcessor.thread->schedule_imm(c);
-}
-
-void
-put_hostinfo_ClusterFunction(ClusterHandler *ch, void *data, int /* len ATS_UNUSED */)
-{
-  HostDB_put_message *msg = (HostDB_put_message *)data;
-  HostDBContinuation *c   = hostDBContAllocator.alloc();
-  HostDBContinuation::Options copt;
-  HostDBMD5 md5;
-
-  SET_CONTINUATION_HANDLER(c, (HostDBContHandler)&HostDBContinuation::clusterResponseEvent);
-  md5.host_name = msg->name;
-  md5.host_len  = msg->namelen;
-  md5.ip.assign(&msg->ip.sa);
-  md5.port            = ats_ip_port_host_order(&msg->ip.sa);
-  md5.hash            = msg->md5;
-  md5.db_mark         = db_mark_for(&msg->ip.sa);
-  copt.host_res_style = host_res_style_for(&msg->ip.sa);
-  c->init(md5, copt);
-  c->mutex       = hostDB.refcountcache->lock_for_key(msg->md5.fold());
-  c->from_cont   = msg->cont; // cannot use action if cont freed due to timeout
-  c->missing     = msg->missing;
-  c->round_robin = msg->round_robin;
-  c->ttl         = msg->ttl;
-  c->from        = ch->machine;
-  dnsProcessor.thread->schedule_imm(c);
-}
-
-//
 // Background event
 // Just increment the current_interval.  Might do other stuff
 // here, like move records to the current position in the cluster.
@@ -2093,12 +1735,6 @@ HostDBInfo::rr()
   return (HostDBRoundRobin *)((char *)this + this->app.rr.offset);
 }
 
-ClusterMachine *
-HostDBContinuation::master_machine(ClusterConfiguration *cc)
-{
-  return cc->machine_hash((int)(md5.hash[1] >> 32));
-}
-
 struct ShowHostDB;
 typedef int (ShowHostDB::*ShowHostDBEventHandler)(int event, Event *data);
 struct ShowHostDB : public ShowCont {
diff --git a/iocore/hostdb/P_HostDB.h b/iocore/hostdb/P_HostDB.h
index 3220f41..2ca0925 100644
--- a/iocore/hostdb/P_HostDB.h
+++ b/iocore/hostdb/P_HostDB.h
@@ -35,7 +35,6 @@
 
 #ifdef SPLIT_DNS
 #include "P_SplitDNS.h"
-#include "P_Cluster.h"
 #endif
 
 #include "P_EventSystem.h"
diff --git a/iocore/hostdb/P_HostDBProcessor.h b/iocore/hostdb/P_HostDBProcessor.h
index 17f56a3..f7fbbc6 100644
--- a/iocore/hostdb/P_HostDBProcessor.h
+++ b/iocore/hostdb/P_HostDBProcessor.h
@@ -484,9 +484,6 @@ struct HostDBContinuation : public Continuation {
   }
   HostDBInfo *lookup_done(IpAddr const &ip, const char *aname, bool round_robin, unsigned int attl, SRVHosts *s = nullptr,
                           HostDBInfo *r = nullptr);
-  bool do_get_response(Event *e);
-  void do_put_response(ClusterMachine *m, HostDBInfo *r, Continuation *cont);
-  int failed_cluster_request(Event *e);
   int key_partition();
   void remove_trigger_pending_dns();
   int set_check_pending_dns();
diff --git a/lib/bindings/metrics.cc b/lib/bindings/metrics.cc
index ae2a3a5..873db80 100644
--- a/lib/bindings/metrics.cc
+++ b/lib/bindings/metrics.cc
@@ -230,7 +230,7 @@ int
 lua_metrics_install(lua_State *L)
 {
   int count                = 0;
-  int metrics_type         = RECT_NODE | RECT_PROCESS | RECT_CLUSTER | RECT_PLUGIN;
+  int metrics_type         = RECT_NODE | RECT_PROCESS | RECT_PLUGIN;
   BindingInstance *binding = BindingInstance::self(L);
   std::set<std::string> prefixes;
 
diff --git a/lib/records/I_RecDefs.h b/lib/records/I_RecDefs.h
index 0249512..d2eb2ed 100644
--- a/lib/records/I_RecDefs.h
+++ b/lib/records/I_RecDefs.h
@@ -56,7 +56,6 @@ enum RecT {
   RECT_CONFIG  = 0x01,
   RECT_PROCESS = 0x02,
   RECT_NODE    = 0x04,
-  RECT_CLUSTER = 0x08,
   RECT_LOCAL   = 0x10,
   RECT_PLUGIN  = 0x20,
   RECT_ALL     = 0x3F
diff --git a/lib/records/P_RecCore.cc b/lib/records/P_RecCore.cc
index c4e8040..4e07377 100644
--- a/lib/records/P_RecCore.cc
+++ b/lib/records/P_RecCore.cc
@@ -261,21 +261,20 @@ recv_message_cb(RecMessage *msg, RecMessageT msg_type, void * /* cookie */)
 //-------------------------------------------------------------------------
 // RecRegisterStatXXX
 //-------------------------------------------------------------------------
-#define REC_REGISTER_STAT_XXX(A, B)                                                                                             \
-  ink_assert((rec_type == RECT_NODE) || (rec_type == RECT_CLUSTER) || (rec_type == RECT_PROCESS) || (rec_type == RECT_LOCAL) || \
-             (rec_type == RECT_PLUGIN));                                                                                        \
-  RecRecord *r;                                                                                                                 \
-  RecData my_data_default;                                                                                                      \
-  my_data_default.A = data_default;                                                                                             \
-  if ((r = RecRegisterStat(rec_type, name, B, my_data_default, persist_type)) != nullptr) {                                     \
-    if (i_am_the_record_owner(r->rec_type)) {                                                                                   \
-      r->sync_required = r->sync_required | REC_PEER_SYNC_REQUIRED;                                                             \
-    } else {                                                                                                                    \
-      send_register_message(r);                                                                                                 \
-    }                                                                                                                           \
-    return REC_ERR_OKAY;                                                                                                        \
-  } else {                                                                                                                      \
-    return REC_ERR_FAIL;                                                                                                        \
+#define REC_REGISTER_STAT_XXX(A, B)                                                                                           \
+  ink_assert((rec_type == RECT_NODE) || (rec_type == RECT_PROCESS) || (rec_type == RECT_LOCAL) || (rec_type == RECT_PLUGIN)); \
+  RecRecord *r;                                                                                                               \
+  RecData my_data_default;                                                                                                    \
+  my_data_default.A = data_default;                                                                                           \
+  if ((r = RecRegisterStat(rec_type, name, B, my_data_default, persist_type)) != nullptr) {                                   \
+    if (i_am_the_record_owner(r->rec_type)) {                                                                                 \
+      r->sync_required = r->sync_required | REC_PEER_SYNC_REQUIRED;                                                           \
+    } else {                                                                                                                  \
+      send_register_message(r);                                                                                               \
+    }                                                                                                                         \
+    return REC_ERR_OKAY;                                                                                                      \
+  } else {                                                                                                                    \
+    return REC_ERR_FAIL;                                                                                                      \
   }
 
 RecErrT
@@ -705,9 +704,6 @@ RecSyncConfigToTB(TextBuffer *tb, bool *inc_version)
             case RECT_NODE:
               tb->copyFrom("NODE ", 5);
               break;
-            case RECT_CLUSTER:
-              tb->copyFrom("CLUSTER ", 8);
-              break;
             case RECT_LOCAL:
               tb->copyFrom("LOCAL ", 6);
               break;
diff --git a/lib/records/P_RecUtils.h b/lib/records/P_RecUtils.h
index 151fdd9..9c7a0d0 100644
--- a/lib/records/P_RecUtils.h
+++ b/lib/records/P_RecUtils.h
@@ -33,8 +33,7 @@
 // Macros
 //-------------------------------------------------------------------------
 
-#define REC_TYPE_IS_STAT(rec_type) \
-  (((rec_type) == RECT_PROCESS) || ((rec_type) == RECT_PLUGIN) || ((rec_type) == RECT_NODE) || ((rec_type) == RECT_CLUSTER))
+#define REC_TYPE_IS_STAT(rec_type) (((rec_type) == RECT_PROCESS) || ((rec_type) == RECT_PLUGIN) || ((rec_type) == RECT_NODE))
 
 #define REC_TYPE_IS_CONFIG(rec_type) (((rec_type) == RECT_CONFIG) || ((rec_type) == RECT_LOCAL))
 
diff --git a/lib/records/RecConfigParse.cc b/lib/records/RecConfigParse.cc
index 800ec95..5b52a6e 100644
--- a/lib/records/RecConfigParse.cc
+++ b/lib/records/RecConfigParse.cc
@@ -215,8 +215,6 @@ RecConfigFileParse(const char *path, RecConfigEntryCallback handler, bool inc_ve
       rec_type = RECT_PROCESS;
     } else if (strcmp(rec_type_str, "NODE") == 0) {
       rec_type = RECT_NODE;
-    } else if (strcmp(rec_type_str, "CLUSTER") == 0) {
-      rec_type = RECT_CLUSTER;
     } else if (strcmp(rec_type_str, "LOCAL") == 0) {
       rec_type = RECT_LOCAL;
     } else {
diff --git a/lib/records/RecLocal.cc b/lib/records/RecLocal.cc
index de7cc0a..6ec4b72 100644
--- a/lib/records/RecLocal.cc
+++ b/lib/records/RecLocal.cc
@@ -44,7 +44,6 @@ i_am_the_record_owner(RecT rec_type)
   switch (rec_type) {
   case RECT_CONFIG:
   case RECT_NODE:
-  case RECT_CLUSTER:
   case RECT_LOCAL:
     return true;
   case RECT_PROCESS:
diff --git a/lib/records/RecProcess.cc b/lib/records/RecProcess.cc
index 8d0233e..9d67530 100644
--- a/lib/records/RecProcess.cc
+++ b/lib/records/RecProcess.cc
@@ -60,7 +60,6 @@ i_am_the_record_owner(RecT rec_type)
       return true;
     case RECT_CONFIG:
     case RECT_NODE:
-    case RECT_CLUSTER:
     case RECT_LOCAL:
       return false;
     default:
@@ -72,7 +71,6 @@ i_am_the_record_owner(RecT rec_type)
     case RECT_CONFIG:
     case RECT_PROCESS:
     case RECT_NODE:
-    case RECT_CLUSTER:
     case RECT_LOCAL:
     case RECT_PLUGIN:
       return true;
diff --git a/lib/ts/HostLookup.h b/lib/ts/HostLookup.h
index f987eb6..d8d872d 100644
--- a/lib/ts/HostLookup.h
+++ b/lib/ts/HostLookup.h
@@ -30,6 +30,9 @@
 
 #ifndef _HOST_LOOKUP_H_
 #define _HOST_LOOKUP_H_
+
+#include "DynArray.h"
+
 // HostLookup  constantss
 const int HOST_TABLE_DEPTH = 3; // Controls the max number of levels in the logical tree
 const int HOST_ARRAY_MAX   = 8; // Sets the fixed array size
diff --git a/lib/ts/apidefs.h.in b/lib/ts/apidefs.h.in
index bc23d8e..6f3ee68 100644
--- a/lib/ts/apidefs.h.in
+++ b/lib/ts/apidefs.h.in
@@ -667,7 +667,6 @@ typedef enum {
   TS_CONFIG_HTTP_SERVER_TCP_INIT_CWND,
   TS_CONFIG_HTTP_SEND_HTTP11_REQUESTS,
   TS_CONFIG_HTTP_CACHE_HTTP,
-  TS_CONFIG_HTTP_CACHE_CLUSTER_CACHE_LOCAL,
   TS_CONFIG_HTTP_CACHE_IGNORE_CLIENT_NO_CACHE,
   TS_CONFIG_HTTP_CACHE_IGNORE_CLIENT_CC_MAX_AGE,
   TS_CONFIG_HTTP_CACHE_IMS_ON_CLIENT_NO_CACHE,
diff --git a/mgmt/Alarms.cc b/mgmt/Alarms.cc
index bf549c0..d42678a 100644
--- a/mgmt/Alarms.cc
+++ b/mgmt/Alarms.cc
@@ -26,7 +26,6 @@
 #include "ts/ink_file.h"
 #include "ts/ink_time.h"
 #include "LocalManager.h"
-#include "ClusterCom.h"
 #include "MgmtUtils.h"
 #include "Alarms.h"
 #include "ts/Diags.h"
@@ -150,10 +149,6 @@ Alarms::resolveAlarm(alarm_t a, char *ip)
     char buf2[1024];
 
     snprintf(buf2, sizeof(buf2), "aresolv: %d\n", a);
-    if (!lmgmt->ccom->sendReliableMessage(inet_addr(ip), buf2, strlen(buf2))) {
-      ink_mutex_release(&mutex);
-      return;
-    }
     ink_hash_table_delete(remote_alarms, buf);
     ats_free(hash_value);
   }
@@ -371,68 +366,6 @@ Alarms::clearUnSeen(char *ip)
 } /* End Alarms::clearUnSeen */
 
 /*
- * constructAlarmMessage(...)
- *   This functions builds a message buffer for passing to peers. It basically
- * takes the current list of local alarms and builds an alarm message.
- */
-void
-Alarms::constructAlarmMessage(const AppVersionInfo &version, char *ip, char *message, int max)
-{
-  int n = 0, bsum = 0;
-  char buf[4096];
-  InkHashTableEntry *entry;
-  InkHashTableIteratorState iterator_state;
-
-  if (!ip) {
-    return;
-  }
-  // Insert the standard mcast packet header
-  n = ClusterCom::constructSharedPacketHeader(version, message, ip, max);
-
-  ink_mutex_acquire(&mutex);
-  if (!((n + (int)strlen("type: alarm\n")) < max)) {
-    if (max >= 1) {
-      message[0] = '\0';
-    }
-    ink_mutex_release(&mutex);
-    return;
-  }
-
-  ink_strlcpy(&message[n], "type: alarm\n", max - n);
-  n += strlen("type: alarm\n");
-  bsum = n;
-  for (entry = ink_hash_table_iterator_first(local_alarms, &iterator_state); (entry != nullptr && n < max);
-       entry = ink_hash_table_iterator_next(local_alarms, &iterator_state)) {
-    Alarm *tmp = (Alarm *)ink_hash_table_entry_value(remote_alarms, entry);
-
-    if (tmp->description) {
-      snprintf(buf, sizeof(buf), "alarm: %d %s\n", tmp->type, tmp->description);
-    } else {
-      snprintf(buf, sizeof(buf), "alarm: %d No details available\n", tmp->type);
-    }
-
-    if (!((n + (int)strlen(buf)) < max)) {
-      break;
-    }
-    ink_strlcpy(&message[n], buf, max - n);
-    n += strlen(buf);
-  }
-
-  if (n == bsum) { /* No alarms */
-    if (!((n + (int)strlen("alarm: none\n")) < max)) {
-      if (max >= 1) {
-        message[0] = '\0';
-      }
-      ink_mutex_release(&mutex);
-      return;
-    }
-    ink_strlcpy(&message[n], "alarm: none\n", max - n);
-  }
-  ink_mutex_release(&mutex);
-  return;
-} /* End Alarms::constructAlarmMessage */
-
-/*
  * checkSystemNAlert(...)
  *   This function should test the system and signal local alarms. Sending
  * out remote notification commands if necessary.
diff --git a/mgmt/LocalManager.cc b/mgmt/LocalManager.cc
index 3cebd4b..a45e778 100644
--- a/mgmt/LocalManager.cc
+++ b/mgmt/LocalManager.cc
@@ -31,8 +31,6 @@
 #include "MgmtSocket.h"
 #include "ts/ink_cap.h"
 #include "FileManager.h"
-#include "ClusterCom.h"
-#include "VMap.h"
 
 #if TS_USE_POSIX_CAP
 #include <sys/capability.h>
@@ -48,10 +46,6 @@ LocalManager::mgmtCleanup()
 
   // fix me for librecords
 
-  if (virt_map) {
-    virt_map->rl_downAddrs(); // We are bailing done need to worry about table
-  }
-
   closelog();
   return;
 }
@@ -143,29 +137,6 @@ LocalManager::clearStats(const char *name)
   }
 }
 
-// bool LocalManager::clusterOk()
-//
-//   Returns false if the proxy has been up for more than
-//     30 seconds but is not reporting that it has clustered
-//     with all the nodes in cluster.config
-//
-//   Otherwise returns true
-//
-bool
-LocalManager::clusterOk()
-{
-  bool found  = true;
-  bool result = true;
-
-  if (processRunning() == true && time(nullptr) > (this->proxy_started_at + 30) &&
-      this->ccom->alive_peers_count + 1 != REC_readInteger("proxy.process.cluster.nodes", &found)) {
-    result = false;
-  }
-
-  ink_assert(found);
-  return result;
-}
-
 bool
 LocalManager::processRunning()
 {
@@ -185,7 +156,6 @@ LocalManager::LocalManager(bool proxy_on) : BaseManager(), run_proxy(proxy_on),
 
   syslog_facility = 0;
 
-  ccom                      = nullptr;
   proxy_recoverable         = true;
   proxy_started_at          = -1;
   proxy_launch_count        = 0;
@@ -196,8 +166,6 @@ LocalManager::LocalManager(bool proxy_on) : BaseManager(), run_proxy(proxy_on),
 
   RecRegisterStatInt(RECT_NODE, "proxy.node.proxy_running", 0, RECP_NON_PERSISTENT);
 
-  virt_map = nullptr;
-
   RecInt http_enabled = REC_readInteger("proxy.config.http.enabled", &found);
   ink_assert(found);
   if (http_enabled && found) {
@@ -267,8 +235,6 @@ LocalManager::LocalManager(bool proxy_on) : BaseManager(), run_proxy(proxy_on),
 LocalManager::~LocalManager()
 {
   delete alarm_keeper;
-  delete virt_map;
-  delete ccom;
   ats_free(absolute_proxy_binary);
   ats_free(proxy_name);
   ats_free(proxy_binary);
@@ -283,74 +249,6 @@ LocalManager::initAlarm()
 }
 
 /*
- * initCCom(...)
- *   Function initializes cluster communication structure held by local manager.
- */
-void
-LocalManager::initCCom(const AppVersionInfo &version, FileManager *configFiles, int mcport, char *addr, int rsport)
-{
-  ats_scoped_str rundir(RecConfigReadRuntimeDir());
-  bool found;
-  IpEndpoint cluster_ip;         // ip addr of the cluster interface
-  ip_text_buffer clusterAddrStr; // cluster ip addr as a String
-  char *intrName;                // Name of the interface we are to use
-  char hostname[1024];           // hostname of this machine
-  const char envVar[] = "PROXY_CLUSTER_ADDR=";
-  char *envBuf;
-
-  if (gethostname(hostname, 1024) < 0) {
-    mgmt_fatal(errno, "[LocalManager::initCCom] gethostname failed\n");
-  }
-  // Fetch which interface we are using for clustering
-  intrName = REC_readString("proxy.config.cluster.ethernet_interface", &found);
-  ink_assert(intrName != nullptr);
-
-  found = mgmt_getAddrForIntr(intrName, &cluster_ip.sa);
-  if (found == false) {
-    mgmt_fatal(0, "[LocalManager::initCCom] Unable to find network interface %s.  Exiting...\n", intrName);
-  } else if (!ats_is_ip4(&cluster_ip)) {
-    mgmt_fatal(0, "[LocalManager::initCCom] Unable to find IPv4 network interface %s.  Exiting...\n", intrName);
-  }
-
-  ats_ip_ntop(&cluster_ip, clusterAddrStr, sizeof(clusterAddrStr));
-  Debug("ccom", "Cluster Interconnect is %s : %s", intrName, clusterAddrStr);
-
-  // This an awful hack but I could not come up with a better way to
-  //  pass the cluster address to the proxy
-  //    Set an environment variable so the proxy can find out
-  //    what the cluster address is.  The reason we need this awful
-  //    hack is that the proxy needs this info immediately at startup
-  //    and it is different for every machine in the cluster so using
-  //    a config variable will not work.
-  // The other options are to pass it on the command line to the proxy
-  //    which would require a fair bit of code modification since
-  //    what is passed right now is assumed to be static.  The other
-  //    is to write it to a separate file but that seems like a
-  //    lot of trouble for a 16 character string
-  // Set the cluster ip addr variable so that proxy can read it
-  //    and flush it to disk
-  const size_t envBuf_size = strlen(envVar) + strlen(clusterAddrStr) + 1;
-  envBuf                   = (char *)ats_malloc(envBuf_size);
-  ink_strlcpy(envBuf, envVar, envBuf_size);
-  ink_strlcat(envBuf, clusterAddrStr, envBuf_size);
-  ink_release_assert(putenv(envBuf) == 0);
-
-  ccom     = new ClusterCom(ats_ip4_addr_cast(&cluster_ip), hostname, mcport, addr, rsport, rundir);
-  virt_map = new VMap(intrName, ats_ip4_addr_cast(&cluster_ip), &lmgmt->ccom->mutex);
-
-  ccom->appVersionInfo = version;
-  ccom->configFiles    = configFiles;
-
-  virt_map->appVersionInfo = version;
-
-  virt_map->downAddrs(); // Just to be safe
-  ccom->establishChannels();
-  ats_free(intrName);
-
-  return;
-}
-
-/*
  * initMgmtProcessServer()
  *   sets up the server socket that proxy processes connect to.
  */
@@ -681,9 +579,6 @@ LocalManager::sendMgmtMsgToProcesses(MgmtMessageHdr *mh)
   switch (mh->msg_id) {
   case MGMT_EVENT_SHUTDOWN: {
     run_proxy = false;
-    if (lmgmt->virt_map) {
-      lmgmt->virt_map->downAddrs(); /* Down all known addrs to be safe */
-    }
     this->closeProxyPorts();
     break;
   }
diff --git a/mgmt/LocalManager.h b/mgmt/LocalManager.h
index f82e859..d35fe0d 100644
--- a/mgmt/LocalManager.h
+++ b/mgmt/LocalManager.h
@@ -42,8 +42,6 @@
 #endif
 
 class FileManager;
-class ClusterCom;
-class VMap;
 
 enum ManagementPendingOperation {
   MGMT_PENDING_NONE,         // Do nothing
@@ -88,7 +86,6 @@ public:
   void clearStats(const char *name = NULL);
 
   bool processRunning();
-  bool clusterOk();
 
   volatile bool run_proxy;
   volatile bool proxy_recoverable; // false if traffic_server cannot recover with a reboot
@@ -117,11 +114,8 @@ public:
   volatile pid_t proxy_launch_pid;
 
   Alarms *alarm_keeper;
-  VMap *virt_map;
   FileManager *configFiles;
 
-  ClusterCom *ccom;
-
   volatile pid_t watched_process_pid;
 
   int syslog_facility;
diff --git a/mgmt/Makefile.am b/mgmt/Makefile.am
index 8f05078..ac03023 100644
--- a/mgmt/Makefile.am
+++ b/mgmt/Makefile.am
@@ -17,7 +17,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-SUBDIRS = cluster utils api
+SUBDIRS = utils api
 
 # Decoder ring:
 # 	libmgmt_c.la 	Configuration records (defaults).
@@ -28,7 +28,6 @@ noinst_LTLIBRARIES = libmgmt_c.la libmgmt_p.la libmgmt_lm.la
 AM_CPPFLAGS += \
   $(iocore_include_dirs) \
   -I$(abs_top_srcdir)/mgmt/api/include \
-  -I$(abs_top_srcdir)/mgmt/cluster \
   -I$(abs_top_srcdir)/mgmt/utils \
   -I$(abs_top_srcdir)/proxy \
   -I$(abs_top_srcdir)/proxy/http \
@@ -70,7 +69,6 @@ libmgmt_lm_la_SOURCES = \
 
 libmgmt_lm_la_LIBADD = \
   libmgmt_c.la \
-  $(top_builddir)/mgmt/cluster/libcluster.la \
   $(top_builddir)/mgmt/utils/libutils_lm.la
 
 libmgmt_p_la_LIBADD = \
diff --git a/mgmt/RecordsConfigUtils.cc b/mgmt/RecordsConfigUtils.cc
index 3a591bd..a7f5557 100644
--- a/mgmt/RecordsConfigUtils.cc
+++ b/mgmt/RecordsConfigUtils.cc
@@ -157,6 +157,5 @@ test_librecords()
 {
   RecRegisterStatInt(RECT_PROCESS, "proxy.process.librecords.testing.int", (RecInt)100, RECP_NON_PERSISTENT);
   RecRegisterStatFloat(RECT_NODE, "proxy.node.librecords.testing.float", (RecFloat)100.1, RECP_NON_PERSISTENT);
-  RecRegisterStatString(RECT_CLUSTER, "proxy.cluster.librecords.testing.string", (RecString) "Hello World\n", RECP_NON_PERSISTENT);
   RecRegisterStatCounter(RECT_LOCAL, "proxy.local.librecords.testing.counter", (RecCounter)99, RECP_NON_PERSISTENT);
 }
diff --git a/mgmt/api/APITestCliRemote.cc b/mgmt/api/APITestCliRemote.cc
index b22454b..40e8fe3 100644
--- a/mgmt/api/APITestCliRemote.cc
+++ b/mgmt/api/APITestCliRemote.cc
@@ -2193,7 +2193,7 @@ print_stats()
 void
 reset_stats()
 {
-  TSMgmtError err = TSStatsReset(false, nullptr);
+  TSMgmtError err = TSStatsReset(nullptr);
   print_err("TSStatsReset", err);
   return;
 }
diff --git a/mgmt/api/CoreAPI.cc b/mgmt/api/CoreAPI.cc
index de8f934..6a8a485 100644
--- a/mgmt/api/CoreAPI.cc
+++ b/mgmt/api/CoreAPI.cc
@@ -34,7 +34,6 @@
 #include "ts/ParseRules.h"
 #include "MgmtUtils.h"
 #include "LocalManager.h"
-#include "ClusterCom.h"
 #include "FileManager.h"
 #include "Rollback.h"
 #include "WebMgmtUtils.h"
@@ -403,13 +402,7 @@ Reconfigure()
 TSMgmtError
 Restart(unsigned options)
 {
-  if (options & TS_RESTART_OPT_CLUSTER) {
-    // Enqueue an event to restart the proxies across the cluster
-    // this will kill TM completely;traffic_cop will restart TM/TS
-    lmgmt->ccom->sendClusterMessage(CLUSTER_MSG_SHUTDOWN_MANAGER);
-  } else {
-    lmgmt->mgmt_shutdown_outstanding = (options & TS_RESTART_OPT_DRAIN) ? MGMT_PENDING_IDLE_RESTART : MGMT_PENDING_RESTART;
-  }
+  lmgmt->mgmt_shutdown_outstanding = (options & TS_RESTART_OPT_DRAIN) ? MGMT_PENDING_IDLE_RESTART : MGMT_PENDING_RESTART;
 
   return TS_ERR_OKAY;
 }
@@ -422,11 +415,7 @@ Restart(unsigned options)
 TSMgmtError
 Bounce(unsigned options)
 {
-  if (options & TS_RESTART_OPT_CLUSTER) {
-    lmgmt->ccom->sendClusterMessage(CLUSTER_MSG_BOUNCE_PROCESS);
-  } else {
-    lmgmt->mgmt_shutdown_outstanding = (options & TS_RESTART_OPT_DRAIN) ? MGMT_PENDING_IDLE_BOUNCE : MGMT_PENDING_BOUNCE;
-  }
+  lmgmt->mgmt_shutdown_outstanding = (options & TS_RESTART_OPT_DRAIN) ? MGMT_PENDING_IDLE_BOUNCE : MGMT_PENDING_BOUNCE;
 
   return TS_ERR_OKAY;
 }
@@ -1057,13 +1046,9 @@ SnapshotGetMlt(LLQ *snapshots)
  * stats are set back to defaults successfully.
  */
 TSMgmtError
-StatsReset(bool cluster, const char *name)
+StatsReset(const char *name)
 {
-  if (cluster) {
-    lmgmt->ccom->sendClusterMessage(CLUSTER_MSG_CLEAR_STATS, name);
-  } else {
-    lmgmt->clearStats(name);
-  }
+  lmgmt->clearStats(name);
   return TS_ERR_OKAY;
 }
 
diff --git a/mgmt/api/CoreAPI.h b/mgmt/api/CoreAPI.h
index 66dfad2..be6de7e 100644
--- a/mgmt/api/CoreAPI.h
+++ b/mgmt/api/CoreAPI.h
@@ -91,6 +91,6 @@ TSMgmtError SnapshotRestore(const char *snapshot_name);
 TSMgmtError SnapshotRemove(const char *snapshot_name);
 TSMgmtError SnapshotGetMlt(LLQ *snapshots);
 
-TSMgmtError StatsReset(bool cluster, const char *name = NULL);
+TSMgmtError StatsReset(const char *name = NULL);
 
 #endif
diff --git a/mgmt/api/CoreAPIRemote.cc b/mgmt/api/CoreAPIRemote.cc
index 32e7ac1..73b26ce 100644
--- a/mgmt/api/CoreAPIRemote.cc
+++ b/mgmt/api/CoreAPIRemote.cc
@@ -1120,10 +1120,10 @@ SnapshotGetMlt(LLQ *snapshots)
 }
 
 TSMgmtError
-StatsReset(bool cluster, const char *stat_name)
+StatsReset(const char *stat_name)
 {
   TSMgmtError ret;
-  OpType op               = cluster ? OpType::STATS_RESET_CLUSTER : OpType::STATS_RESET_NODE;
+  OpType op               = OpType::STATS_RESET_NODE;
   OpType optype           = op;
   MgmtMarshallString name = const_cast<MgmtMarshallString>(stat_name);
 
diff --git a/mgmt/api/INKMgmtAPI.cc b/mgmt/api/INKMgmtAPI.cc
index bc05f64..60566ee 100644
--- a/mgmt/api/INKMgmtAPI.cc
+++ b/mgmt/api/INKMgmtAPI.cc
@@ -1339,9 +1339,9 @@ TSVirtIpAddrEleDestroy(TSVirtIpAddrEle *ele)
 
 /*--- statistics operations ----------------------------------------------- */
 tsapi TSMgmtError
-TSStatsReset(bool cluster, const char *name)
+TSStatsReset(const char *name)
 {
-  return StatsReset(cluster, name);
+  return StatsReset(name);
 }
 
 /*--- variable operations ------------------------------------------------- */
diff --git a/mgmt/api/TSControlMain.cc b/mgmt/api/TSControlMain.cc
index 6a336af..f5d3269 100644
--- a/mgmt/api/TSControlMain.cc
+++ b/mgmt/api/TSControlMain.cc
@@ -916,7 +916,7 @@ handle_stats_reset(int fd, void *req, size_t reqlen)
 
   err = recv_mgmt_request(req, reqlen, OpType::STATS_RESET_NODE, &optype, &name);
   if (err == TS_ERR_OKAY) {
-    err = StatsReset(optype == OpType::STATS_RESET_CLUSTER, name);
+    err = StatsReset(name);
   }
 
   ats_free(name);
diff --git a/mgmt/api/include/mgmtapi.h b/mgmt/api/include/mgmtapi.h
index 4691240..09d08d9 100644
--- a/mgmt/api/include/mgmtapi.h
+++ b/mgmt/api/include/mgmtapi.h
@@ -971,10 +971,9 @@ tsapi TSMgmtError TSSnapshotGetMlt(TSStringList snapshots);
 
 /*--- statistics operations -----------------------------------------------*/
 /* TSStatsReset: sets all the statistics variables to their default values
- * Input: cluster - Reset the stats clusterwide or not
  * Outpue: TSErrr
  */
-tsapi TSMgmtError TSStatsReset(bool cluster, const char *name);
+tsapi TSMgmtError TSStatsReset(const char *name);
 
 /*--- variable operations -------------------------------------------------*/
 /* TSRecordGet: gets a record
diff --git a/mgmt/cluster/ClusterCom.cc b/mgmt/cluster/ClusterCom.cc
deleted file mode 100644
index c688603..0000000
--- a/mgmt/cluster/ClusterCom.cc
+++ /dev/null
@@ -1,2414 +0,0 @@
-/** @file
-
-  A brief file description
-
-  @section license License
-
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
- */
-
-/**************************************
- *
- * ClusterCom.cc
- *   Member function definitions for Cluster communication class.
- *
- * $Date: 2008-05-20 17:26:19 $
- *
- *
- */
-
-#include "ts/ink_platform.h"
-#include "ts/ink_sock.h"
-#include "ts/ink_file.h"
-
-#include "ts/I_Version.h"
-
-#include "ts/TextBuffer.h"
-#include "MgmtSocket.h"
-
-#include "LocalManager.h"
-#include "ClusterCom.h"
-#include "VMap.h"
-#include "MgmtUtils.h"
-#include "WebMgmtUtils.h"
-#include "MgmtHashTable.h"
-#include "FileManager.h"
-
-static bool checkBackDoor(int req_fd, char *message);
-
-int MultiCastMessages = 0;
-long LastHighestDelta = -1L;
-
-static void *
-drainIncomingChannel_broadcast(void *arg)
-{
-  char message[61440];
-  ClusterCom *ccom = (ClusterCom *)arg;
-
-  time_t t;
-  time_t last_multicast_receive_time = time(nullptr);
-
-  /* Avert race condition, thread spun during constructor */
-  while (lmgmt->ccom != ccom || !lmgmt->ccom->init) {
-    mgmt_sleep_sec(1);
-  }
-
-  for (;;) { /* Loop draining mgmt network channels */
-    int nevents = 0;
-
-    // It's not clear whether this can happen, but historically, this code was written as if it
-    // could. A hacky little sleep here will prevent this thread spinning on the read timeout.
-    if (ccom->cluster_type == NO_CLUSTER || ccom->receive_fd == ts::NO_FD) {
-      mgmt_sleep_sec(1);
-    }
-
-    ink_zero(message);
-
-    if (ccom->cluster_type != NO_CLUSTER) {
-      nevents = mgmt_read_timeout(ccom->receive_fd, ccom->mc_poll_timeout /* secs */, 0 /* usecs */);
-      if (nevents > 0) {
-        last_multicast_receive_time = time(nullptr); // valid multicast msg
-      } else {
-        t = time(nullptr);
-        if ((t - last_multicast_receive_time) > ccom->mc_poll_timeout) {
-          // Timeout on multicast receive channel, reset channel.
-          if (ccom->receive_fd > 0) {
-            close(ccom->receive_fd);
-          }
-          ccom->receive_fd = -1;
-          Debug("ccom", "Timeout, resetting multicast receive channel");
-          if (lmgmt->ccom->establishReceiveChannel(0)) {
-            Debug("ccom", "establishReceiveChannel failed");
-            ccom->receive_fd = -1;
-          }
-          last_multicast_receive_time = t; // next action at next interval
-        }
-      }
-    }
-
-    /* Broadcast message */
-    if (ccom->cluster_type != NO_CLUSTER && ccom->receive_fd > 0 && nevents > 0 &&
-        (ccom->receiveIncomingMessage(message, sizeof(message)) > 0)) {
-      ccom->handleMultiCastMessage(message);
-    }
-  }
-
-  return nullptr;
-} /* End drainIncomingChannel */
-
-/*
- * drainIncomingChannel
- *   This function is blocking, it never returns. It is meant to allow for
- * continuous draining of the network. It drains and handles requests made on
- * the reliable and multicast channels between all the peers.
- */
-static void *
-drainIncomingChannel(void *arg)
-{
-  char message[61440];
-  ClusterCom *ccom = (ClusterCom *)arg;
-  struct sockaddr_in cli_addr;
-
-  // Fix for INKqa07688: There was a problem at Genuity where if you
-  // pulled out the cable on the cluster interface (or just ifconfig'd
-  // down/up the cluster interface), the fd associated with that
-  // interface would somehow get into a bad state... and the multicast
-  // packets from other nodes wouldn't be received anymore.
-  //
-  // The fix for the problem was to close() and re-open the multicast
-  // socket if we detected that no activity has occurred for 30
-  // seconds.  30 seconds was based on the default peer_timeout
-  // (proxy.config.cluster.peer_timeout) value.  davey showed that
-  // this value worked out well experimentally (though more testing
-  // and experimentation would be beneficial).
-  //
-  // traffic_manager running w/ no cop: In this case, our select()
-  // call will hang if the fd gets into the bad state.  The solution
-  // is to timeout select if nothing comes in off the network for
-  // sometime.. wake up, and close/open the multicast channel.
-  //
-  // traffic_manager running w/ cop: In this case, our select() will
-  // never timeout (since cop will be heartbeating us).  Some
-  // additional logic was added to keep track of the last successful
-  // mulitcast receive.
-  //
-  // after closing the channel, some addition logic was put into place
-  // to reopen the channel (e.g. opening the socket would fail if the
-  // interface was down).  In this case, the ccom->receive_fd is set
-  // to '-1' and the open is retried until it succeeds.
-
-  /* Avert race condition, thread spun during constructor */
-  while (lmgmt->ccom != ccom || !lmgmt->ccom->init) {
-    mgmt_sleep_sec(1);
-  }
-
-  for (;;) { /* Loop draining mgmt network channels */
-    ink_zero(message);
-
-    // It's not clear whether this can happen, but historically, this code was written as if it
-    // could. A hacky little sleep here will prevent this thread spinning on the read timeout.
-    if (ccom->cluster_type == NO_CLUSTER || ccom->reliable_server_fd == ts::NO_FD) {
-      mgmt_sleep_sec(1);
-    }
-
-    if (mgmt_read_timeout(ccom->reliable_server_fd, ccom->mc_poll_timeout /* secs */, 0 /* usecs */) > 0) {
-      /* Reliable(TCP) request */
-      socklen_t clilen = sizeof(cli_addr);
-      int req_fd       = mgmt_accept(ccom->reliable_server_fd, (struct sockaddr *)&cli_addr, &clilen);
-      if (req_fd < 0) {
-        mgmt_elog(errno, "[drainIncomingChannel] error accepting "
-                         "reliable connection\n");
-        continue;
-      }
-      if (fcntl(req_fd, F_SETFD, 1) < 0) {
-        mgmt_elog(errno, "[drainIncomingChannel] Unable to set close "
-                         "on exec flag\n");
-        close(req_fd);
-        continue;
-      }
-
-      // In no cluster mode, the rsport should not be listening.
-      ink_release_assert(ccom->cluster_type != NO_CLUSTER);
-
-      /* Handle Request */
-      if (mgmt_readline(req_fd, message, 61440) > 0) {
-        if (strstr(message, "aresolv: ")) {
-          /* Peer is Resolving our alarm */
-          alarm_t a;
-
-          // coverity[secure_coding]
-          if (sscanf(message, "aresolv: %d", &a) != 1) {
-            close_socket(req_fd);
-            continue;
-          }
-          lmgmt->alarm_keeper->resolveAlarm(a);
-        } else if (strstr(message, "unmap: ")) {
-          /*
-           * Explicit virtual ip unmap request. Note order unmap then
-           * map for strstr.
-           */
-          char msg_ip[80];
-          const char *msg;
-          if (sscanf(message, "unmap: %79s", msg_ip) != 1) {
-            close_socket(req_fd);
-            continue;
-          }
-
-          mgmt_log("[drainIncomingChannel] Got unmap request: '%s'\n", message);
-
-          ink_mutex_acquire(&(ccom->mutex));       /* Grab the lock */
-          if (lmgmt->virt_map->rl_unmap(msg_ip)) { /* Requires lock */
-            msg = "unmap: done";
-          } else {
-            msg = "unmap: failed";
-          }
-          ink_mutex_release(&(ccom->mutex)); /* Release the lock */
-
-          mgmt_writeline(req_fd, msg, strlen(msg));
-
-          /* Wait for peer to read status */
-          if (mgmt_readline(req_fd, message, 61440) != 0) {
-            mgmt_elog(0, "[drainIncomingChannel] Connection not closed\n");
-          }
-        } else if (strstr(message, "map: ")) {
-          /* Explicit virtual ip map request */
-          char msg_ip[80];
-          const char *msg;
-          if (sscanf(message, "map: %79s", msg_ip) != 1) {
-            close_socket(req_fd);
-            continue;
-          }
-
-          mgmt_log("[drainIncomingChannel] Got map request: '%s'\n", message);
-
-          if (lmgmt->run_proxy) {
-            ink_mutex_acquire(&(ccom->mutex));     /* Grab the lock */
-            if (lmgmt->virt_map->rl_map(msg_ip)) { /* Requires the lock */
-              msg = "map: done";
-            } else {
-              msg = "map: failed";
-            }
-            ink_mutex_release(&(ccom->mutex)); /* Release the lock */
-          } else {
-            msg = "map: failed";
-          }
-
-          mgmt_writeline(req_fd, msg, strlen(msg));
-
-          /* Wait for peer to read status */
-          if (mgmt_readline(req_fd, message, 61440) != 0) {
-            mgmt_elog(0, "[drainIncomingChannel] Connection not closed\n");
-          }
-
-        } else if (strstr(message, "file: ")) {
-          /* Requesting a config file from us */
-          bool stat = false;
-          char fname[1024];
-          version_t ver;
-          TextBuffer *buff = nullptr;
-          Rollback *rb;
-
-          /* Get the file and blast it back */
-          if (sscanf(message, "file: %1023s %d", fname, &ver) != 2) {
-            close_socket(req_fd);
-            continue;
-          }
-
-          if (ccom->configFiles->getRollbackObj(fname, &rb) && (rb->getCurrentVersion() == ver) &&
-              (rb->getVersion(ver, &buff) == OK_ROLLBACK)) {
-            size_t bytes_written = 0;
-            stat                 = true;
-            bytes_written        = write_socket(req_fd, buff->bufPtr(), strlen(buff->bufPtr()));
-            if (bytes_written != strlen(buff->bufPtr())) {
-              stat = false;
-              mgmt_log("[drainIncomingChannel] Failed file req: %s v: %d\n", fname, ver);
-            } else {
-              Debug("ccom", "[drainIncomingChannel] file req: %s v: %d bytes: %d", fname, ver, (int)strlen(buff->bufPtr()));
-            }
-          } else {
-            mgmt_elog(0, "[drainIncomingChannel] Error file req: %s ver: %d\n", fname, ver);
-          }
-
-          if (!stat) {
-            const char *msg = "file: failed";
-            mgmt_writeline(req_fd, msg, strlen(msg));
-          }
-          if (buff) {
-            delete buff;
-          }
-        } else if (strstr(message, "cmd: shutdown_manager")) {
-          mgmt_log("[ClusterCom::drainIncomingChannel] Received manager shutdown request\n");
-          lmgmt->mgmt_shutdown_outstanding = MGMT_PENDING_RESTART;
-        } else if (strstr(message, "cmd: shutdown_process")) {
-          mgmt_log("[ClusterCom::drainIncomingChannel] Received process shutdown request\n");
-          lmgmt->processShutdown();
-        } else if (strstr(message, "cmd: restart_process")) {
-          mgmt_log("[ClusterCom::drainIncomingChannel] Received restart process request\n");
-          lmgmt->processRestart();
-        } else if (strstr(message, "cmd: bounce_process")) {
-          mgmt_log("[ClusterCom::drainIncomingChannel] Received bounce process request\n");
-          lmgmt->processBounce();
-        } else if (strstr(message, "cmd: clear_stats")) {
-          char sname[1024];
-          mgmt_log("[ClusterCom::drainIncomingChannel] Received clear stats request\n");
-          if (sscanf(message, "cmd: clear_stats %1023s", sname) != 1) {
-            lmgmt->clearStats(sname);
-          } else {
-            lmgmt->clearStats();
-          }
-        } else if (!checkBackDoor(req_fd, message)) { /* Heh... */
-          mgmt_log("[ClusterCom::drainIncomingChannel] Unexpected message on cluster"
-                   " port.  Possibly an attack\n");
-          Debug("ccom", "Unknown message to rsport received: %s", message);
-          close_socket(req_fd);
-          continue;
-        }
-      }
-      close_socket(req_fd);
-    }
-  }
-
-  return nullptr;
-} /* End drainIncomingChannel */
-
-/*
- * cluster_com_port_watcher(...)
- *   This function watches updates and changes that are made to the
- * cluster port. Reconfiguring it if need be.
- *
- * Note: the cluster port here is the cluster port for the proxy not
- * the manager.
- */
-int
-cluster_com_port_watcher(const char *name, RecDataT /* data_type ATS_UNUSED */, RecData data, void * /* cookie ATS_UNUSED */)
-{
-  ink_assert(!name);
-
-  ink_mutex_acquire(&(lmgmt->ccom->mutex)); /* Grab cluster lock */
-  lmgmt->ccom->cluster_port = (RecInt)data.rec_int;
-  ink_mutex_release(&(lmgmt->ccom->mutex)); /* Release cluster lock */
-
-  return 0;
-} /* End cluster_com_port_watcher */
-
-ClusterCom::ClusterCom(unsigned long oip, char *host, int mcport, char *group, int rsport, char *p)
-  : our_wall_clock(0), alive_peers_count(0), reliable_server_fd(0), broadcast_fd(0), receive_fd(0)
-{
-  int rec_err;
-  bool found = false;
-
-  init = false;
-  if (strlen(host) >= 1024) {
-    mgmt_fatal(0, "[ClusterCom::ClusterCom] Hostname too large: %s\n", host);
-  }
-  // the constructor does a memset() on broadcast_addr and receive_addr, initializing them
-  // coverity[uninit_member]
-  memset(&broadcast_addr, 0, sizeof(broadcast_addr));
-  memset(&receive_addr, 0, sizeof(receive_addr));
-
-  ink_strlcpy(our_host, host, sizeof(our_host));
-  our_ip = oip;
-
-  /* Get the cluster type */
-  cluster_type = NO_CLUSTER;
-
-  /* Get the cluster config file name + path */
-  RecString cluster_file;
-
-  rec_err = RecGetRecordString_Xmalloc("proxy.config.cluster.cluster_configuration", &cluster_file);
-  found   = (rec_err == REC_ERR_OKAY);
-
-  if (!found) {
-    mgmt_fatal(0, "[ClusterCom::ClusterCom] no cluster_configuration filename configured\n");
-  }
-
-  if (strlen(p) + strlen(cluster_file) >= 1024) {
-    mgmt_fatal(0, "[ClusterCom::ClusterCom] path + filename too large\n");
-  }
-  // XXX: This allows to have absolute config cluster_configuration directive.
-  //      If that file must be inside config directory (p) use
-  //      ink_filepath_make
-  ink_filepath_merge(cluster_conf, sizeof(cluster_conf), p, cluster_file, INK_FILEPATH_TRUENAME);
-  // XXX: Shouldn't we pass the cluster_conf to the Rollback ???
-  //
-  Debug("ccom", "[ClusterCom::ClusterCom] Using cluster file: %s", cluster_file);
-  Debug("ccom", "[ClusterCom::ClusterCom] Using cluster conf: %s", cluster_conf);
-  cluster_file_rb = new Rollback(cluster_file, false);
-
-  ats_free(cluster_file);
-
-  if (ink_sys_name_release(sys_name, sizeof(sys_name), sys_release, sizeof(sys_release)) >= 0) {
-    mgmt_log("[ClusterCom::ClusterCom] Node running on OS: '%s' Release: '%s'\n", sys_name, sys_release);
-  } else {
-    sys_name[0] = sys_release[0] = '\0';
-    mgmt_elog(errno, "[ClusterCom::ClusterCom] Unable to determime OS and release info\n");
-  }
-
-  /* Grab the proxy cluster port */
-  cluster_port = REC_readInteger("proxy.config.cluster.cluster_port", &found);
-  RecRegisterConfigUpdateCb("proxy.config.cluster.cluster_port", cluster_com_port_watcher, nullptr);
-
-  if (!(strlen(group) < (MAX_MC_GROUP_LEN - 1))) {
-    mgmt_fatal(0, "[ClusterCom::ClusterCom] mc group length too large!\n");
-  }
-
-  ink_strlcpy(mc_group, group, sizeof(mc_group));
-  mc_port              = mcport;
-  reliable_server_port = rsport;
-
-  mc_ttl = REC_readInteger("proxy.config.cluster.mc_ttl", &found);
-  ink_assert(found);
-
-  log_bogus_mc_msgs = REC_readInteger("proxy.config.cluster.log_bogus_mc_msgs", &found);
-  ink_assert(found);
-
-  /* Timeout between config changes, basically a clock noise filter */
-  delta_thresh = REC_readInteger("proxy.config.cluster.delta_thresh", &found);
-  ink_assert(found);
-
-  /* The timeout before marking a peer as dead */
-  peer_timeout = REC_readInteger("proxy.config.cluster.peer_timeout", &found);
-  ink_assert(found);
-
-  mc_send_interval = REC_readInteger("proxy.config.cluster.mc_send_interval", &found);
-  ink_assert(found);
-
-  mc_poll_timeout = REC_readInteger("proxy.config.cluster.mc_poll_timeout", &found);
-  ink_assert(found);
-
-  /* Launch time */
-  startup_time = time(nullptr);
-
-  /* Timeout before broadcasting virtual ip information */
-  startup_timeout = REC_readInteger("proxy.config.cluster.startup_timeout", &found);
-  ink_assert(found);
-
-  last_shared_send = 0;
-
-  ink_mutex_init(&mutex, "ccom-mutex");
-  peers       = ink_hash_table_create(InkHashTableKeyType_String);
-  mismatchLog = ink_hash_table_create(InkHashTableKeyType_String);
-
-  if (cluster_type != NO_CLUSTER) {
-    ink_thread_create(drainIncomingChannel_broadcast, this, 0, 0, nullptr); /* Spin drainer thread */
-    ink_thread_create(drainIncomingChannel, this, 0, 0, nullptr);           /* Spin drainer thread */
-  }
-  return;
-} /* End ClusterCom::ClusterCom */
-
-/*
- * checkPeers(...)
- *   Function checks on our peers by racing through the peer list(ht) and
- * marking nodes as idle/dead if we have not heard from them in awhile.
- */
-void
-ClusterCom::checkPeers(time_t *ticker)
-{
-  static int number_of_nodes = -1;
-  bool signal_alarm          = false;
-  time_t t                   = time(nullptr);
-  InkHashTableEntry *entry;
-  InkHashTableIteratorState iterator_state;
-
-  // Hack in the file manager in case the rollback needs to send a notification. This is definitely
-  // a hack, but it helps break the dependency on global FileManager in traffic_manager.
-  cluster_file_rb->configFiles = configFiles;
-
-  if (cluster_type == NO_CLUSTER) {
-    return;
-  }
-
-  if ((t - *ticker) > 5) {
-    int num_peers = 0;
-    long idle_since;
-    TextBuffer *buff;
-
-    Debug("ccom", "MultiCast Messages received: %d", MultiCastMessages);
-
-    /*
-     * Need the lock here so that someone doesn't change the peer hash
-     * table out from underneath you.
-     */
-    ink_mutex_acquire(&(mutex)); /* Grab cluster lock */
-    for (entry = ink_hash_table_iterator_first(peers, &iterator_state); entry != nullptr;
-         entry = ink_hash_table_iterator_next(peers, &iterator_state)) {
-      ClusterPeerInfo *tmp = (ClusterPeerInfo *)ink_hash_table_entry_value(peers, entry);
-      if ((idle_since = t - tmp->idle_ticks) > peer_timeout) {
-        char cip[80];
-        struct in_addr addr;
-        addr.s_addr = tmp->inet_address;
-
-        ink_strlcpy(cip, inet_ntoa(addr), sizeof(cip));
-
-        Debug("ccom", "[ClusterCom::checkPeers] DEAD! %s idle since: %ld naddrs: %d", cip, idle_since, tmp->num_virt_addrs);
-
-        if ((idle_since = t - tmp->manager_idle_ticks) > peer_timeout) {
-          if (tmp->manager_alive > 0) {
-            Note("marking manager on node %s as down", cip);
-          }
-          tmp->manager_alive = -1;
-          Debug("ccom", "[ClusterCom::checkPeers] Manager DEAD! %s idle since: %ld", cip, idle_since);
-        }
-
-        if (tmp->num_virt_addrs >= 0) {
-          Note("marking server on node %s as down", cip);
-        }
-
-        tmp->num_virt_addrs = -1; /* This is basically the I'm dead flag */
-        if (tmp->num_virt_addrs) {
-          lmgmt->virt_map->rl_resetSeenFlag(cip);
-          lmgmt->virt_map->rl_clearUnSeen(cip);
-        }
-      }
-    }
-
-    /* Create the base for the cluster file(inserting header comment) */
-    buff = new TextBuffer(strlen(CLUSTER_CONFIG_FILE_BLURB) + 1024);
-    buff->copyFrom(CLUSTER_CONFIG_FILE_BLURB, strlen(CLUSTER_CONFIG_FILE_BLURB));
-
-    if (cluster_type == FULL_CLUSTER) {
-      /*
-       * Two pass loop. First loop over the peers hash table and count
-       * the number of peers. Second construct the new cluster config
-       * file for the proxy.
-       */
-      for (int c = 0; c <= 1; c++) {
-        bool flag = false; /* Used to mark first loop on second pass */
-
-        for (entry = ink_hash_table_iterator_first(peers, &iterator_state); entry != nullptr;
-             entry = ink_hash_table_iterator_next(peers, &iterator_state)) {
-          char str_number[80];
-          char *key            = (char *)ink_hash_table_entry_key(peers, entry);
-          ClusterPeerInfo *tmp = (ClusterPeerInfo *)ink_hash_table_entry_value(peers, entry);
-
-          if (!c) {                          /* First pass */
-            if (tmp->num_virt_addrs != -1) { /* Count if not dead */
-              num_peers++;
-            }
-          } else if (!flag) {
-            /*
-             * First time through second pass, cluster config file needs
-             * to start with the number of nodes in the cluster, so we
-             * stick this in the file
-             */
-            snprintf(str_number, sizeof(str_number), "%d\n", num_peers);
-            buff->copyFrom(str_number, strlen(str_number));
-            flag = true;
-          } else if (num_peers == number_of_nodes) {
-            /*
-             * If the number of peers in the hash_table is the same as the
-             * last time we checked, then we have emitted this file before.
-             *
-             * FIX: there is potentially a case where a node comes in and
-             *      a node leaves(for good) and we don't notice the change
-             *      of guard.
-             */
-            break;
-          }
-
-          if (c && tmp->num_virt_addrs != -1) {
-            /* Second pass, add entry to the file "ip:port" */
-            buff->copyFrom(key, strlen(key));
-            snprintf(str_number, sizeof(str_number), ":%d\n", tmp->port);
-            buff->copyFrom(str_number, strlen(str_number));
-          }
-        }
-
-        if (num_peers == number_of_nodes) {
-          /*
-           * If the number of peers in the hash_table is the same as the
-           * last time we checked, then we have emitted this file before.
-           *
-           * FIX: there is potentially a case where a node comes in and
-           *      a node leaves(for good) and we don't notice the change
-           *      of guard.
-           */
-          break;
-        } else if (num_peers == 0 && !c) {
-          /*
-           *Handle the standalone case, you are on the second pass and
-           * there are no peers. Proxy expects 0 to be in the file to
-           * signify standalone case.
-           */
-          buff->copyFrom("0\n", strlen("0\n"));
-          break;
-        }
-      }
-    } else {
-      // We are not doing full clustering so we also tell the
-      //   proxy there are zero nodes in the cluster
-      buff->copyFrom("0\n", strlen("0\n"));
-    }
-    /*
-     * The number of peers have changed, output the new file, this will
-     * trigger an update callback which will eveutually signal the proxy.
-     */
-    if (num_peers != number_of_nodes) {
-      if (cluster_file_rb->forceUpdate(buff) != OK_ROLLBACK) {
-        mgmt_elog(0, "[ClusterCom::checkPeers] Failed update: cluster.config\n");
-        signal_alarm = true; /* Throw the alarm after releasing the lock */
-      } else {
-        number_of_nodes   = num_peers; /* Update the static count */
-        alive_peers_count = num_peers;
-      }
-    }
-    delete buff;
-    ink_mutex_release(&(mutex)); /* Release cluster lock */
-    if (signal_alarm) {
-      /*
-      lmgmt->alarm_keeper->signalAlarm(MGMT_ALARM_PROXY_SYSTEM_ERROR,
-                                       "[TrafficManager] Unable to write cluster.config, membership unchanged");
-      */
-      mgmt_elog(0, "[TrafficManager] Unable to write cluster.config, membership unchanged");
-    }
-    *ticker = t;
-  }
-
-  /*
-   * Here we aggregate cluster stats from the stored peers stats and our
-   * node stats.
-   */
-  // fix me -- what does aggregated_node_data do?
-
-  ink_mutex_acquire(&(mutex)); /* Grab the cluster lock */
-  for (entry = ink_hash_table_iterator_first(peers, &iterator_state); entry != nullptr;
-       entry = ink_hash_table_iterator_next(peers, &iterator_state)) {
-    ClusterPeerInfo *tmp = (ClusterPeerInfo *)ink_hash_table_entry_value(peers, entry);
-
-    /* Skip nodes we've flagged as dead */
-    if (tmp->num_virt_addrs == -1) {
-      continue;
-    }
-    // fix me -- what does aggregated_node_data do?
-  }
-  ink_mutex_release(&(mutex)); /* Release the cluster lock */
-
-  return;
-} /* End ClusterCom::checkPeers */
-
-void
-ClusterCom::generateClusterDelta()
-{
-  long highest_delta = 0L;
-  InkHashTableEntry *entry;
-  InkHashTableIteratorState iterator_state;
-
-  if (cluster_type == NO_CLUSTER) {
-    return;
-  }
-
-  ink_mutex_acquire(&(mutex));
-  for (entry = ink_hash_table_iterator_first(peers, &iterator_state); entry != nullptr;
-       entry = ink_hash_table_iterator_next(peers, &iterator_state)) {
-    ClusterPeerInfo *tmp = (ClusterPeerInfo *)ink_hash_table_entry_value(peers, entry);
-
-    // is the node alive?
-    if (tmp->num_virt_addrs != -1) {
-      highest_delta = max(highest_delta, tmp->delta);
-    }
-  }
-  ink_mutex_release(&(mutex));
-
-  // only transmit if different
-  if (highest_delta != LastHighestDelta) {
-    LastHighestDelta = highest_delta;
-    char highest_delta_str[32];
-    snprintf(highest_delta_str, 31, "%ld", highest_delta);
-    lmgmt->signalEvent(MGMT_EVENT_HTTP_CLUSTER_DELTA, highest_delta_str);
-  }
-
-} /* End ClusterCom::generateClusterDelta */
-
-/*
- * handleMultCastMessage(...)
- *   Function is called to handle(parse) messages received from the broadcast
- * channel.
- */
-void
-ClusterCom::handleMultiCastMessage(char *message)
-{
-  int peer_cluster_port, ccom_port;
-  char *last, *line, ip[1024], hostname[1024];
-  char tsver[128]         = "Before 2.X";
-  char cluster_name[1024] = "UNKNOWN";
-  RecT type;
-  ClusterPeerInfo *p;
-  time_t peer_wall_clock, t;
-  InkHashTableValue hash_value;
-
-  ++MultiCastMessages;
-
-  t              = time(nullptr); /* Get current time for determining most recent changes */
-  our_wall_clock = t;
-
-  /* Grab the ip address, we need to know this so that we only complain
-     once about a cluster name or traffic server version mismatch */
-  if ((line = strtok_r(message, "\n", &last)) == nullptr) {
-    goto Lbogus; /* IP of sender */
-  }
-
-  // coverity[secure_coding]
-  if (strlen(line) >= sizeof(ip) || sscanf(line, "ip: %s", ip) != 1) {
-    goto Lbogus;
-  }
-
-  // FIX THIS: elam 02/23/1999
-  //   Loopback disable is currently not working on NT.
-  //   We will ignore our own multicast messages.
-  if (inet_addr(ip) == our_ip) {
-    return;
-  }
-
-  /* Make sure this is a message for the cluster we belong to */
-  if ((line = strtok_r(nullptr, "\n", &last)) == nullptr) {
-    goto Lbogus; /* ClusterName of sender */
-  }
-
-  // coverity[secure_coding]
-  if (strlen(line) >= sizeof(cluster_name) || sscanf(line, "cluster: %s", cluster_name) != 1) {
-    goto Lbogus;
-  }
-
-  if (strcmp(cluster_name, lmgmt->proxy_name) != 0) {
-    logClusterMismatch(ip, TS_NAME_MISMATCH, cluster_name);
-    return;
-  }
-
-  /* Make sure this a message from a Traffic Server of the same version */
-  if ((line = strtok_r(nullptr, "\n", &last)) == nullptr) {
-    goto Lbogus; /* TS version of sender */
-  }
-
-  // coverity[secure_coding]
-  if (strlen(line) >= sizeof(tsver) || sscanf(line, "tsver: %s", tsver) != 1 || strcmp(line + 7, appVersionInfo.VersionStr) != 0) {
-    logClusterMismatch(ip, TS_VER_MISMATCH, tsver);
-    return;
-  }
-
-  /* Figure out what type of message this is */
-  if ((line = strtok_r(nullptr, "\n", &last)) == nullptr) {
-    goto Lbogus;
-  }
-  if (strcmp("type: files", line) == 0) { /* Config Files report */
-    handleMultiCastFilePacket(last, ip);
-    return;
-  } else if (strcmp("type: stat", line) == 0) { /* Statistics report */
-    type = RECT_CLUSTER;
-  } else if (strcmp("type: alarm", line) == 0) { /* Alarm report */
-    handleMultiCastAlarmPacket(last, ip);
-    return;
-  } else if (strcmp("type: vmap", line) == 0) { /* Virtual Map report */
-    handleMultiCastVMapPacket(last, ip);
-    return;
-  } else {
-    mgmt_elog(0, "[ClusterCom::handleMultiCastMessage] Invalid type msg: '%s'\n", line);
-    return;
-  }
-
-  /* Check OS and version info */
-  if ((line = strtok_r(nullptr, "\n", &last)) == nullptr) {
-    goto Lbogus; /* OS of sender */
-  }
-  if (!strstr(line, "os: ") || !strstr(line, sys_name)) {
-    /*
-    lmgmt->alarm_keeper->signalAlarm(MGMT_ALARM_PROXY_SYSTEM_ERROR,
-                                     "Received Multicast message from peer running mis-match"
-                                     " Operating system, please investigate");
-    */
-    Debug("ccom", "[ClusterCom::handleMultiCastMessage] Received message from peer "
-                  "running different os/release '%s'(ours os: '%s' rel: '%s'\n",
-          line, sys_name, sys_release);
-  }
-
-  if ((line = strtok_r(nullptr, "\n", &last)) == nullptr) {
-    goto Lbogus; /* OS-Version of sender */
-  }
-  if (!strstr(line, "rel: ") || !strstr(line, sys_release)) {
-    /*
-    lmgmt->alarm_keeper->signalAlarm(MGMT_ALARM_PROXY_SYSTEM_ERROR,
-                                     "Received Multicast message from peer running mis-match"
-                                     " Operating system release, please investigate");
-    */
-    Debug("ccom", "[ClusterCom::handleMultiCastMessage] Received message from peer "
-                  "running different os/release '%s'(ours os: '%s' rel: '%s'\n",
-          line, sys_name, sys_release);
-  }
-
-  if ((line = strtok_r(nullptr, "\n", &last)) == nullptr) {
-    goto Lbogus; /* Hostname of sender */
-  }
-  if (strlen(line) >= sizeof(hostname) || sscanf(line, "hostname: %s", hostname) != 1) {
-    mgmt_elog(0, "[ClusterCom::handleMultiCastMessage] Invalid message-line(%d) '%s'\n", __LINE__, line);
-    return;
-  }
-
-  if ((line = strtok_r(nullptr, "\n", &last)) == nullptr) {
-    goto Lbogus; /* mc_port of sender */
-  }
-  if (sscanf(line, "port: %d", &peer_cluster_port) != 1) {
-    mgmt_elog(0, "[ClusterCom::handleMultiCastMessage] Invalid message-line(%d) '%s'\n", __LINE__, line);
-    return;
-  }
-
-  if ((line = strtok_r(nullptr, "\n", &last)) == nullptr) {
-    goto Lbogus; /* rs_port of sender */
-  }
-  if (sscanf(line, "ccomport: %d", &ccom_port) != 1) {
-    mgmt_elog(0, "[ClusterCom::handleMultiCastMessage] Invalid message-line(%d) '%s'\n", __LINE__, line);
-    return;
-  }
-
-  /* Their wall clock time and last config change time */
-  if ((line = strtok_r(nullptr, "\n", &last)) == nullptr) {
-    goto Lbogus;
-  }
-  int64_t tt;
-  if (sscanf(line, "time: %" PRId64 "", &tt) != 1) {
-    mgmt_elog(0, "[ClusterCom::handleMultiCastMessage] Invalid message-line(%d) '%s'\n", __LINE__, line);
-    return;
-  }
-  peer_wall_clock = (time_t)tt;
-
-  /* Have we see this guy before? */
-  ink_mutex_acquire(&(mutex)); /* Grab cluster lock to access hash table */
-  if (ink_hash_table_lookup(peers, (InkHashTableKey)ip, &hash_value) == 0) {
-    p                 = (ClusterPeerInfo *)ats_malloc(sizeof(ClusterPeerInfo));
-    p->inet_address   = inet_addr(ip);
-    p->num_virt_addrs = 0;
-
-    // Safe since these are completely static
-    // TODO: This might no longer be completely optimal, since we don't keep track of
-    // how many RECT_NODE stats there are. I'm hoping it's negligible though, but worst
-    // case we can reoptimize this later (and more efficiently).
-    int cnt               = 0;
-    p->node_rec_data.recs = (RecRecord *)ats_malloc(sizeof(RecRecord) * g_num_records);
-    for (int j = 0; j < g_num_records; j++) {
-      RecRecord *rec = &(g_records[j]);
-
-      /*
-       * The following code make sence only when RECT_NODE records
-       * defined in mgmt/RecordsConfig.cc are placed continuously.
-       */
-      if (rec->rec_type == RECT_NODE) {
-        p->node_rec_data.recs[cnt].rec_type  = rec->rec_type;
-        p->node_rec_data.recs[cnt].name      = rec->name;
-        p->node_rec_data.recs[cnt].data_type = rec->data_type;
-        memset(&p->node_rec_data.recs[cnt].data, 0, sizeof(rec->data));
-        memset(&p->node_rec_data.recs[cnt].data_default, 0, sizeof(rec->data_default));
-        p->node_rec_data.recs[cnt].lock          = rec->lock;
-        p->node_rec_data.recs[cnt].sync_required = rec->sync_required;
-        ++cnt;
-      }
-    }
-    p->node_rec_data.num_recs = cnt;
-
-    ink_hash_table_insert(peers, (InkHashTableKey)ip, (p));
-    ink_hash_table_delete(mismatchLog, ip);
-
-    Note("adding node %s to the cluster", ip);
-  } else {
-    p = (ClusterPeerInfo *)hash_value;
-    if (p->manager_alive < 0) {
-      Note("marking manager on node %s as up", ip);
-    }
-    if (our_wall_clock - p->idle_ticks > peer_timeout) {
-      Note("marking server on node %s as up", ip);
-    }
-  }
-  p->port       = peer_cluster_port;
-  p->ccom_port  = ccom_port;
-  p->idle_ticks = p->manager_idle_ticks = our_wall_clock;
-  p->last_time_recorded                 = peer_wall_clock;
-  p->delta                              = peer_wall_clock - our_wall_clock;
-  p->manager_alive                      = 1;
-
-  ink_assert(type == RECT_CLUSTER);
-  handleMultiCastStatPacket(last, p);
-  ink_mutex_release(&(mutex)); /* Release cluster lock */
-
-  return;
-
-Lbogus:
-  if (log_bogus_mc_msgs) {
-    mgmt_elog(0, "[ClusterCom::handleMultiCastMessage] Bogus mc message-line\n");
-    if (line) {
-      Debug("ccom", "[ClusterCom::handleMultiCastMessage] Bogus mc message-line %s", line);
-    }
-    return;
-  }
-} /* End ClusterCom::handleMultiCastMessage */
-
-/*
- * handleMultiCastStatPacket(...)
- *   Function groks the stat packets received on the mc channel and updates
- * our local copy of our peers stats.
- */
-void
-ClusterCom::handleMultiCastStatPacket(char *last, ClusterPeerInfo *peer)
-{
-  char *line;
-  RecRecords *rec_ptr = nullptr;
-  int tmp_id          = -1;
-  RecDataT tmp_type   = RECD_NULL;
-  RecRecord *rec      = nullptr;
-
-  /* Loop over records, updating peer copy(summed later) */
-  rec_ptr = &(peer->node_rec_data);
-  for (int i = 0; (line = strtok_r(nullptr, "\n", &last)) && i < rec_ptr->num_recs; i++) {
-    tmp_id   = -1;
-    tmp_type = RECD_NULL;
-    rec      = &(rec_ptr->recs[i]);
-    //    rec      = &(g_records[g_type_records[RECT_NODE][i]]);
-
-    switch (rec->data_type) {
-    case RECD_INT:
-    case RECD_COUNTER: {
-      RecInt tmp_msg_val = -1;
-      tmp_id             = ink_atoi(line);
-      char *v2 = strchr(line, ':'), *v3 = nullptr;
-      if (v2) {
-        tmp_type = (RecDataT)ink_atoi(v2 + 1);
-        v3       = strchr(v2 + 1, ':');
-        if (v3) {
-          tmp_msg_val = ink_atoi64(v3 + 1);
-        }
-      }
-      if (!v2 || !v3) {
-        mgmt_elog(0, "[ClusterCom::handleMultiCastStatPacket] Invalid message-line(%d) '%s'\n", __LINE__, line);
-        return;
-      }
-      ink_assert(i == tmp_id && rec->data_type == tmp_type);
-      ink_release_assert(i == tmp_id && rec->data_type == tmp_type);
-      if (!(i == tmp_id && rec->data_type == tmp_type)) {
-        return;
-      }
-
-      if (rec->data_type == RECD_INT) {
-        rec->data.rec_int = tmp_msg_val;
-      } else {
-        rec->data.rec_counter = tmp_msg_val;
-      }
-      break;
-    }
-    case RECD_FLOAT: {
-      MgmtFloat tmp_msg_val = -1.0;
-      // the types specified are all have a defined constant size
-      // coverity[secure_coding]
-      if (sscanf(line, "%d:%d: %f", &tmp_id, (int *)&tmp_type, &tmp_msg_val) != 3) {
-        mgmt_elog(0, "[ClusterCom::handleMultiCastStatPacket] Invalid message-line(%d) '%s'\n", __LINE__, line);
-        return;
-      }
-      ink_assert(i == tmp_id && rec->data_type == tmp_type);
-      ink_release_assert(i == tmp_id && rec->data_type == tmp_type);
-      if (!(i == tmp_id && rec->data_type == tmp_type)) {
-        return;
-      }
-
-      rec->data.rec_float = tmp_msg_val;
-      break;
-    }
-    case RECD_STRING: { /* String stats not supported for cluster passing */
-      int ccons;
-      char *tmp_msg_val = nullptr;
-
-      // the types specified are all have a defined constant size
-      // coverity[secure_coding]
-      if (sscanf(line, "%d:%d: %n", &tmp_id, (int *)&tmp_type, &ccons) != 2) {
-        mgmt_elog(0, "[ClusterCom::handleMultiCastStatPacket] Invalid message-line(%d) '%s'\n", __LINE__, line);
-        return;
-      }
-      tmp_msg_val = &line[ccons];
-      ink_assert(i == tmp_id && rec->data_type == tmp_type);
-      ink_release_assert(i == tmp_id && rec->data_type == tmp_type);
-      if (!(i == tmp_id && rec->data_type == tmp_type)) {
-        return;
-      }
-
-      if (strcmp(tmp_msg_val, "NULL") == 0 && rec->data.rec_string) {
-        ats_free(rec->data.rec_string);
-        rec->data.rec_string = nullptr;
-      } else if (!(strcmp(tmp_msg_val, "NULL") == 0)) {
-        ats_free(rec->data.rec_string);
-        rec->data.rec_string = (RecString)ats_strdup(tmp_msg_val);
-      }
-      break;
-    }
-    default:
-      ink_assert(false);
-      break;
-    }
-  }
-  return;
-} /* End ClusterCom::handleMultiCastStatPacket */
-
-//-------------------------------------------------------------------------
-// INKqa08381 - These functions are called by
-// ClusterCom::handleMultiCastFilePacket(); required so that we only
-// sync records.config CONFIG values (not LOCAL values) across a
-// records.config cluster syncronize operation.
-//-------------------------------------------------------------------------
-bool
-scan_and_terminate(char *&p, char a, char b)
-{
-  bool eob = false; // 'eob' is end-of-buffer
-  while ((*p != a) && (*p != b) && (*p != '\0')) {
-    p++;
-  }
-  if (*p == '\0') {
-    eob = true;
-  } else {
-    *(p++) = '\0';
-    while ((*p == a) || ((*p == b) && (*p != '\0'))) {
-      p++;
-    }
-    if (*p == '\0') {
-      eob = true;
-    }
-  }
-  return eob;
-}
-
-void
-extract_locals(MgmtHashTable *local_ht, char *record_buffer)
-{
-  char *p, *q, *line, *line_cp, *name;
-  bool eof;
-  p = record_buffer;
-  for (eof = false; !eof;) {
-    line = q = p;
-    eof      = scan_and_terminate(p, '\r', '\n');
-    Debug("ccom_rec", "[extract_locals] %s", line);
-    while ((*q == ' ') || (*q == '\t')) {
-      q++;
-    }
-    // is this line a LOCAL?
-    if (strncmp(q, "LOCAL", strlen("LOCAL")) == 0) {
-      line_cp = ats_strdup(line);
-      q += strlen("LOCAL");
-      while ((*q == ' ') || (*q == '\t')) {
-        q++;
-      }
-      name = q;
-      if (scan_and_terminate(q, ' ', '\t')) {
-        Debug("ccom_rec", "[extract_locals] malformed line: %s", name);
-        ats_free(line_cp);
-        continue;
-      }
-      local_ht->mgmt_hash_table_insert(name, line_cp);
-    }
-  }
-}
-
-bool
-insert_locals(TextBuffer *rec_cfg_new, TextBuffer *rec_cfg, MgmtHashTable *local_ht)
-{
-  char *p, *q, *line, *name;
-  bool eof;
-  InkHashTableEntry *hte;
-  InkHashTableIteratorState htis;
-  MgmtHashTable *local_access_ht = new MgmtHashTable("local_access_ht", false, InkHashTableKeyType_String);
-  p                              = rec_cfg->bufPtr();
-  for (eof = false; !eof;) {
-    line = q = p;
-    eof      = scan_and_terminate(p, '\r', '\n');
-    Debug("ccom_rec", "[insert_locals] %s", line);
-    while ((*q == ' ') || (*q == '\t')) {
-      q++;
-    }
-    // is this line a local?
-    if (strncmp(q, "LOCAL", strlen("LOCAL")) == 0) {
-      q += strlen("LOCAL");
-      while ((*q == ' ') || (*q == '\t')) {
-        q++;
-      }
-      name = q;
-      if (scan_and_terminate(q, ' ', '\t')) {
-        Debug("ccom_rec", "[insert_locals] malformed line: %s", name);
-        continue;
-      }
-      if (local_ht->mgmt_hash_table_lookup(name, (void **)&line)) {
-        // LOCAL found in hash-table; 'line' points to our LOCAL
-        // value; keep track that we accessed this LOCAL already;
-        // later, we need to iterate through all of our un-accessed
-        // LOCALs and add them to the bottom of the remote config
-        local_access_ht->mgmt_hash_table_insert(name, nullptr);
-      } else {
-        // LOCAL didn't exist in our config, don't merge into the
-        // remote config
-        continue;
-      }
-    }
-    // copy the whole line over
-    rec_cfg_new->copyFrom(line, strlen(line));
-    rec_cfg_new->copyFrom("\n", strlen("\n"));
-  }
-  // remove any of our accessed LOCALs from local_ht
-  for (hte = local_access_ht->mgmt_hash_table_iterator_first(&htis); hte != nullptr;
-       hte = local_access_ht->mgmt_hash_table_iterator_next(&htis)) {
-    name = (char *)local_access_ht->mgmt_hash_table_entry_key(hte);
-    local_ht->mgmt_hash_table_delete(name);
-  }
-  // add our un-accessed LOCALs to the remote config
-  for (hte = local_ht->mgmt_hash_table_iterator_first(&htis); hte != nullptr;
-       hte = local_ht->mgmt_hash_table_iterator_next(&htis)) {
-    line = (char *)local_ht->mgmt_hash_table_entry_value(hte);
-    rec_cfg_new->copyFrom(line, strlen(line));
-    rec_cfg_new->copyFrom("\n", strlen("\n"));
-  }
-  // clean-up and return
-  delete local_access_ht;
-  return true;
-}
-
-/*
- * handleMultiCastFilePacket(...)
- *   Functions handles file packets that come over the mc channel. Its
- * basic job is to determine whether or not the timestamps/version
- * numbers that its peers are reporting are newer than the timestamps
- * and versions of the local config files. If there is a mis-match and
- * their files are newer then we initiate a request to get the newer file
- * and then roll that into our own config files.
- */
-void
-ClusterCom::handleMultiCastFilePacket(char *last, char *ip)
-{
-  char *line, file[1024];
-  version_t ver, our_ver;
-  int64_t tt;
-  InkHashTableValue hash_value;
-  bool file_update_failure;
-
-  while ((line = strtok_r(nullptr, "\n", &last))) {
-    Rollback *rb;
-
-    file_update_failure = false;
-    // coverity[secure_coding]
-    if (sscanf(line, "%1023s %d %" PRId64 "\n", file, &ver, &tt) != 3) {
-      mgmt_elog(0, "[ClusterCom::handleMultiCastFilePacket] Invalid message-line(%d) '%s'\n", __LINE__, line);
-      return;
-    }
-
-    if (configFiles->getRollbackObj(file, &rb)) {
-      our_ver = rb->getCurrentVersion();
-      if (ver > our_ver) { /* Their version is newer */
-                           /*
-                            * FIX: we have the timestamp from them as well, should we also
-                            * figure that into this? or are version numbers sufficient?
-                            *
-                            * (mod > rb->versionTimeStamp(our_ver)
-                            *
-                            * When fixing this, watch out for the workaround put in place
-                            * for INKqa08567.  File timestamps aren't sent around the
-                            * cluster anymore.
-                            */
-        char message[1024];
-        TextBuffer *reply = new TextBuffer(2048); /* Start with 2k file size */
-        snprintf(message, sizeof(message), "file: %s %d", file, ver);
-
-        /* Send request, read response, write new file. */
-        if (!(sendReliableMessageReadTillClose(inet_addr(ip), message, strlen(message), reply)) || (reply->spaceUsed() <= 0)) {
-          delete reply;
-          return;
-        }
-
-        if (strstr(reply->bufPtr(), "file: failed")) {
-          file_update_failure = true;
-        }
-        // INKqa08381: Special case for records.config; we only want
-        // to syncronize CONFIG records from remote machine, not LOCAL
-        // records; store our LOCAL records in a hash-table, and then
-        // merge our LOCALs into the newly acquired remote config.
-        if (!file_update_failure && (strcmp(file, "records.config") == 0)) {
-          TextBuffer *our_rec_cfg;
-          char *our_rec_cfg_cp;
-          TextBuffer *reply_new;
-          MgmtHashTable *our_locals_ht;
-
-          if (rb->getVersion(our_ver, &our_rec_cfg) != OK_ROLLBACK) {
-            file_update_failure = true;
-          } else {
-            our_locals_ht  = new MgmtHashTable("our_locals_ht", true, InkHashTableKeyType_String);
-            our_rec_cfg_cp = ats_strdup(our_rec_cfg->bufPtr());
-            extract_locals(our_locals_ht, our_rec_cfg_cp);
-            reply_new = new TextBuffer(reply->spaceUsed());
-            if (!insert_locals(reply_new, reply, our_locals_ht)) {
-              file_update_failure = true;
-              delete reply_new;
-              reply_new = nullptr;
-            } else {
-              delete reply;
-              reply = reply_new;
-            }
-            ats_free(our_rec_cfg_cp);
-            delete our_rec_cfg;
-            delete our_locals_ht;
-          }
-        }
-
-        if (!file_update_failure && (rb->updateVersion(reply, our_ver, ver, true, false) != OK_ROLLBACK)) {
-          file_update_failure = true;
-        }
-
-        if (file_update_failure) {
-          mgmt_elog(0, "[ClusterCom::handleMultiCastFilePacket] Update failed\n");
-        } else {
-          mgmt_log("[ClusterCom::handleMultiCastFilePacket] "
-                   "Updated '%s' o: %d n: %d\n",
-                   file, our_ver, ver);
-        }
-
-        delete reply;
-      }
-    } else {
-      mgmt_elog(0, "[ClusterCom::handleMultiCastFilePacket] Unknown file seen: '%s'\n", file);
-    }
-  }
-
-  ink_mutex_acquire(&(mutex)); /* Grab cluster lock to access hash table */
-  if (ink_hash_table_lookup(peers, (InkHashTableKey)ip, &hash_value) != 0) {
-    ((ClusterPeerInfo *)hash_value)->manager_idle_ticks = time(nullptr);
-    if (((ClusterPeerInfo *)hash_value)->manager_alive < 0) {
-      Note("marking manager on node %s as up", ip);
-    }
-    ((ClusterPeerInfo *)hash_value)->manager_alive = 1;
-  }
-  ink_mutex_release(&(mutex)); /* Release cluster lock */
-
-  return;
-} /* End ClusterCom::handleMultiCastFilePacket */
-
-/*
- * handleMultiCastAlarmPacket(...)
- *   Function receives incoming alarm messages and updates the alarms class.
- */
-void
-ClusterCom::handleMultiCastAlarmPacket(char *last, char *ip)
-{
-  char *line;
-
-  /* Allows us to expire stale alarms */
-  lmgmt->alarm_keeper->resetSeenFlag(ip);
-  while ((line = strtok_r(nullptr, "\n", &last))) {
-    int ccons;
-    alarm_t a;
-    char *desc;
-
-    if (strcmp(line, "alarm: none") == 0) {
-      break;
-    }
-    // both types have a finite size
-    // coverity[secure_coding]
-    if (sscanf(line, "alarm: %d %n", &a, &ccons) != 1) {
-      mgmt_elog(0, "[ClusterCom::handleMultiCastAlarmPacket] Invalid message-line(%d) '%s'\n", __LINE__, line);
-      return;
-    }
-
-    desc = &line[ccons];
-
-    /* Signalling will only re-issue if new */
-    lmgmt->alarm_keeper->signalAlarm(a, desc, ip);
-    Debug("ccom", "[ClusterCom::handleMultiCastAlarmPacket] Alarm: ip: '%s' '%s'", ip, line);
-  }
-  lmgmt->alarm_keeper->clearUnSeen(ip); /* Purge expired alarms */
-  return;
-} /* End ClusterCom::handleMultiCastAlarmPacket */
-
-/*
- * handleMultiCastVMapPacket(...)
- *   Handles incoming reports from peers about which virtual interfaces
- * they are servicing. This then updates the VMap class to indicate who
- * is holding what.
- */
-void
-ClusterCom::handleMultiCastVMapPacket(char *last, char *ip)
-{
-  char *line;
-  InkHashTableValue hash_value;
-
-  ink_mutex_acquire(&(mutex));           /* VMap class uses cluster mutex */
-  lmgmt->virt_map->rl_resetSeenFlag(ip); /* Ala alarms */
-  ink_mutex_release(&(mutex));
-
-  while ((line = strtok_r(nullptr, "\n", &last))) {
-    char vaddr[80];
-
-    if (strcmp(line, "virt: none") == 0) {
-      break;
-    }
-    // coverity[secure_coding]
-    if (sscanf(line, "virt: %79s", vaddr) != 1) {
-      mgmt_elog(0, "[ClusterCom::handleMultiCastVMapPacket] Invalid message-line(%d) '%s'\n", __LINE__, line);
-      return;
-    }
-
-    ink_mutex_acquire(&(mutex));
-    lmgmt->virt_map->rl_map(vaddr, ip); /* Record this nodes map */
-    ink_mutex_release(&(mutex));
-  }
-
-  ink_mutex_acquire(&(mutex));
-  if (ink_hash_table_lookup(peers, ip, &hash_value) != 0) {
-    ((ClusterPeerInfo *)hash_value)->num_virt_addrs = lmgmt->virt_map->rl_clearUnSeen(ip);
-  }
-  ink_mutex_release(&(mutex));
-  return;
-} /* End ClusterCom::handleMultiCastVMapPacket */
-
-/*
- * sendSharedData
- *   Function serves as aggregator of NODE data to be shared with the
- * cluster. It collects the data, formats the message, and finally broadcasts
- * the message.
- */
-bool
-ClusterCom::sendSharedData(bool send_proxy_heart_beat)
-{
-  char message[61440], addr[80];
-  struct in_addr resolved_addr;
-  time_t now;
-
-  if (cluster_type == NO_CLUSTER) {
-    return true;
-  }
-
-  now = time(nullptr);
-  if (now == -1) {
-    // The time call failed
-    last_shared_send = 0;
-  } else {
-    int time_since_last_send = now - last_shared_send;
-    if (last_shared_send != 0 && time_since_last_send > peer_timeout) {
-      Warning("multicast send timeout exceeded.  %d seconds since"
-              " last send.",
-              time_since_last_send);
-    } else if (last_shared_send != 0 && time_since_last_send < mc_send_interval) {
-      return true;
-    }
-    last_shared_send = now;
-  }
-
-  /* Config Files Message */
-  memset(message, 0, 61440);
-  constructSharedFilePacket(message, 61440);
-  sendOutgoingMessage(message, strlen(message));
-
-  /* Alarm Message */
-  memset(message, 0, 61440);
-  resolved_addr.s_addr = our_ip;
-  ink_strlcpy(addr, inet_ntoa(resolved_addr), sizeof(addr));
-  lmgmt->alarm_keeper->constructAlarmMessage(appVersionInfo, addr, message, 61440);
-  sendOutgoingMessage(message, strlen(message));
-
-  /*
-   * Send alarms and file config information always, if we are not running
-   * a proxy we should not send a stat packet(no stats to report since no
-   * proxy is running). The stat packet is used to hearbeat that the node
-   * is alive, since this packet is not sent the master peer will not assign
-   * us ip addresses and if we are the master peer someone will take over our
-   * duties.
-   */
-  if (!send_proxy_heart_beat) {
-    return true;
-  }
-
-  /* Stat Message */
-  memset(message, 0, 61440);
-  constructSharedStatPacket(message, 61440);
-  sendOutgoingMessage(message, strlen(message));
-
-  /* VMap Message */
-  memset(message, 0, 61440);
-  lmgmt->virt_map->lt_constructVMapMessage(addr, message, 61440);
-  sendOutgoingMessage(message, strlen(message));
-
-  return true;
-} /* End ClusterCom::sendSharedData */
-
-/*
- * constructSharedGenericPacket(...)
- *   A generic packet builder that can construct config or stat
- * broadcast packets. Basically the smarts to read the records values.
- */
-void
-ClusterCom::constructSharedGenericPacket(char *message, int max, RecT packet_type)
-{
-  int running_sum = 0; /* Make sure we never go over max */
-  char tmp[1024];
-  struct in_addr resolved_addr;
-
-  /* Insert the standard packet header */
-  resolved_addr.s_addr = our_ip;
-  running_sum          = constructSharedPacketHeader(appVersionInfo, message, inet_ntoa(resolved_addr), max);
-
-  if (packet_type == RECT_NODE) {
-    ink_strlcpy(&message[running_sum], "type: stat\n", (max - running_sum));
-    running_sum += strlen("type: stat\n");
-    ink_release_assert(running_sum < max);
-  } else {
-    mgmt_elog(0, "[ClusterCom::constructSharedGenericPacket] Illegal type seen '%d'\n", packet_type);
-    return;
-  }
-
-  if (sys_name[0]) {
-    snprintf(tmp, sizeof(tmp), "os: %s\n", sys_name);
-  } else {
-    snprintf(tmp, sizeof(tmp), "os: unknown\n");
-  }
-  ink_strlcpy(&message[running_sum], tmp, (max - running_sum));
-  running_sum += strlen(tmp);
-  ink_release_assert(running_sum < max);
-
-  if (sys_release[0]) {
-    snprintf(tmp, sizeof(tmp), "rel: %s\n", sys_release);
-  } else {
-    snprintf(tmp, sizeof(tmp), "rel: unknown\n");
-  }
-  ink_strlcpy(&message[running_sum], tmp, (max - running_sum));
-  running_sum += strlen(tmp);
-  ink_release_assert(running_sum < max);
-
-  snprintf(tmp, sizeof(tmp), "hostname: %s\n", our_host);
-  ink_strlcpy(&message[running_sum], tmp, (max - running_sum));
-  running_sum += strlen(tmp);
-  ink_release_assert(running_sum < max);
-
-  snprintf(tmp, sizeof(tmp), "port: %d\n", cluster_port);
-  ink_strlcpy(&message[running_sum], tmp, (max - running_sum));
-  running_sum += strlen(tmp);
-  ink_release_assert(running_sum < max);
-
-  snprintf(tmp, sizeof(tmp), "ccomport: %d\n", reliable_server_port);
-  ink_strlcpy(&message[running_sum], tmp, (max - running_sum));
-  running_sum += strlen(tmp);
-  ink_release_assert(running_sum < max);
-
-  /* Current time stamp, for xntp like synching */
-  if (time(nullptr) > 0) {
-    snprintf(tmp, sizeof(tmp), "time: %" PRId64 "\n", (int64_t)time(nullptr));
-    ink_strlcpy(&message[running_sum], tmp, (max - running_sum));
-    running_sum += strlen(tmp);
-  } else {
-    mgmt_elog(errno, "[ClusterCom::constructSharedPacket] time failed\n");
-  }
-  ink_release_assert(running_sum < max);
-
-  int cnt = 0;
-  for (int j = 0; j < g_num_records; j++) {
-    RecRecord *rec = &(g_records[j]);
-
-    if (rec->rec_type == RECT_NODE) {
-      switch (rec->data_type) {
-      case RECD_COUNTER:
-        snprintf(tmp, sizeof(tmp), "%d:%d: %" PRId64 "\n", cnt, rec->data_type, rec->data.rec_counter);
-        ink_strlcpy(&message[running_sum], tmp, (max - running_sum));
-        running_sum += strlen(tmp);
-        break;
-      case RECD_INT:
-        snprintf(tmp, sizeof(tmp), "%d:%d: %" PRId64 "\n", cnt, rec->data_type, rec->data.rec_int);
-        ink_strlcpy(&message[running_sum], tmp, (max - running_sum));
-        running_sum += strlen(tmp);
-        break;
-      case RECD_FLOAT:
-        snprintf(tmp, sizeof(tmp), "%d:%d: %f\n", cnt, rec->data_type, rec->data.rec_float);
-        ink_strlcpy(&message[running_sum], tmp, (max - running_sum));
-        running_sum += strlen(tmp);
-        break;
-      case RECD_STRING:
-        if (rec->data.rec_string) {
-          snprintf(tmp, sizeof(tmp), "%d:%d: %s\n", cnt, rec->data_type, rec->data.rec_string);
-        } else {
-          snprintf(tmp, sizeof(tmp), "%d:%d: NULL\n", cnt, rec->data_type);
-        }
-        ink_strlcpy(&message[running_sum], tmp, (max - running_sum));
-        running_sum += strlen(tmp);
-        break;
-      default:
-        break;
-      }
-      ++cnt;
-    }
-    ink_release_assert(running_sum < max);
-  }
-
-  return;
-} /* End ClusterCom::constructSharedGenericPacket */
-
-void
-ClusterCom::constructSharedStatPacket(char *message, int max)
-{
-  constructSharedGenericPacket(message, max, RECT_NODE);
-  return;
-} /* End ClusterCom::constructSharedStatPacket */
-
-/* static int constructSharedPacketHeader(...)
- *   Each multicast packet needs to have the following
- *   header info.  Ip, Cluster Name, TS Version.  This function
- *   Inserts that information.  Returns the nubmer of bytes inserted
- */
-int
-ClusterCom::constructSharedPacketHeader(const AppVersionInfo &version, char *message, char *ip, int max)
-{
-  int running_sum = 0;
-
-  /* Insert the IP Address of this node */
-  /* Insert the name of this cluster for cluster-identification of mc packets */
-  /* Insert the Traffic Server verison */
-  snprintf(message, max, "ip: %s\ncluster: %s\ntsver: %s\n", ip, lmgmt->proxy_name, version.VersionStr);
-  running_sum = strlen(message);
-  ink_release_assert(running_sum < max);
-
-  return running_sum;
-} /* End ClusterCom::constructSharedPacketHeader */
-
-/*
- * constructSharedFilePacket(...)
- *   Foreach of the config files we are holding build a packet that
- * can be used to share the current version and time stamp of the
- * files, so others can tell if ours our newer.
- */
-void
-ClusterCom::constructSharedFilePacket(char *message, int max)
-{
-  int running_sum = 0;
-  char tmp[1024], *files, *line, *last;
-  struct in_addr resolved_addr;
-  TextBuffer *buff;
-
-  /* Insert the standard packet header */
-  resolved_addr.s_addr = our_ip;
-  running_sum          = constructSharedPacketHeader(appVersionInfo, message, inet_ntoa(resolved_addr), max);
-
-  ink_strlcpy(&message[running_sum], "type: files\n", (max - running_sum));
-  running_sum += strlen("type: files\n");
-  ink_release_assert(running_sum < max);
-
-  buff  = configFiles->filesManaged();
-  files = buff->bufPtr();
-  line  = strtok_r(files, "\n", &last);
-  if (line == nullptr) {
-    delete buff;
-    return;
-  }
-
-  /* Loop over the known files building the packet */
-  do {
-    Rollback *rb;
-
-    /* Some files are local only */
-    if (strcmp(line, "storage.config") == 0) {
-      continue;
-    }
-
-    if (configFiles->getRollbackObj(line, &rb)) {
-      version_t ver = rb->getCurrentVersion();
-
-      // Workaround INKqa08567: Calling versionTimeStamp here causes
-      // traffic_manager to periodically switch to be a root user (as
-      // it needs to stat() some snmp related configuration files).
-      // This caused a race-condition to occur if some configuration
-      // were being written to disk (files would be written as owned
-      // by root instead of being owned by inktomi... thus causing
-      // general permissions badness everywhere).  Because the
-      // timestamp isn't actually used by the peer cluster nodes to
-      // determine which config files are newer, the workaround is to
-      // remove the unnecessary call to versionTimeStamp().
-
-      // time_t mod = rb->versionTimeStamp(ver);
-      time_t mod = 0;
-
-      snprintf(tmp, sizeof(tmp), "%s %d %" PRId64 "\n", line, ver, (int64_t)mod);
-      ink_strlcpy(&message[running_sum], tmp, (max - running_sum));
-      running_sum += strlen(tmp);
-      ink_release_assert(running_sum < max);
-    } else {
-      mgmt_elog(0, "[ClusterCom::constructSharedFilePacket] Invalid base name? '%s'\n", line);
-    }
-  } while ((line = strtok_r(nullptr, "\n", &last)));
-
-  delete buff;
-  return;
-} /* End ClusterCom::constructSharedFilePacket */
-
-/*
- * estabilishChannels(...)
- *   Sets up the multi-cast and reliable tcp channels for cluster
- * communication. But only if clustering is enabled in some shape.
- */
-void
-ClusterCom::establishChannels()
-{
-  int one = 1;
-  struct sockaddr_in serv_addr;
-
-  if (cluster_type != NO_CLUSTER) {
-    establishBroadcastChannel();
-    establishReceiveChannel();
-
-    if (reliable_server_port > 0) {
-      /* Setup reliable connection, for large config changes */
-      if ((reliable_server_fd = socket(AF_INET, SOCK_STREAM, 0)) < 0) {
-        mgmt_fatal(errno, "[ClusterCom::establishChannels] Unable to create socket\n");
-      }
-      if (fcntl(reliable_server_fd, F_SETFD, 1) < 0) {
-        mgmt_fatal(errno, "[ClusterCom::establishChannels] Unable to set close-on-exec.\n");
-      }
-
-      if (setsockopt(reliable_server_fd, SOL_SOCKET, SO_REUSEADDR, (char *)&one, sizeof(int)) < 0) {
-        mgmt_fatal(errno, "[ClusterCom::establishChannels] Unable to set socket options.\n");
-      }
-
-      memset(&serv_addr, 0, sizeof(serv_addr));
-      serv_addr.sin_family      = AF_INET;
-      serv_addr.sin_addr.s_addr = htonl(INADDR_ANY);
-      serv_addr.sin_port        = htons(reliable_server_port);
-
-      if ((bind(reliable_server_fd, (struct sockaddr *)&serv_addr, sizeof(serv_addr))) < 0) {
-        mgmt_fatal(errno, "[ClusterCom::establishChannels] Unable to bind socket (port:%d)\n", reliable_server_port);
-      }
-
-      if ((listen(reliable_server_fd, 10)) < 0) {
-        mgmt_fatal(errno, "[ClusterCom::establishChannels] Unable to listen on socket\n");
-      }
-    }
-  }
-
-  Debug("ccom", "[ClusterCom::establishChannels] Channels setup");
-  init = true;
-  return;
-}
-
-/*
- * establishBroadcastChannel()
- *   Setup our multicast channel for broadcasting.
- */
-void
-ClusterCom::establishBroadcastChannel()
-{
-  if ((broadcast_fd = socket(AF_INET, SOCK_DGRAM, 0)) < 0) {
-    mgmt_fatal(errno, "[ClusterCom::establishBroadcastChannel] Unable to open socket.\n");
-  }
-
-  if (fcntl(broadcast_fd, F_SETFD, 1) < 0) {
-    mgmt_fatal(errno, "[ClusterCom::establishBroadcastChannel] Unable to set close-on-exec.\n");
-  }
-
-  int one = 1;
-  if (setsockopt(broadcast_fd, SOL_SOCKET, SO_REUSEADDR, (const char *)&one, sizeof(one)) < 0) {
-    mgmt_fatal(errno, "[ClusterCom::establishBroadcastChannel] Unable to set socket options.\n");
-  }
-
-  memset(&broadcast_addr, 0, sizeof(broadcast_addr));
-  broadcast_addr.sin_family      = AF_INET;
-  broadcast_addr.sin_addr.s_addr = inet_addr(mc_group);
-  broadcast_addr.sin_port        = htons(mc_port);
-
-  u_char ttl = mc_ttl, loop = 0;
-
-  /* Set ttl(max forwards), 1 should be default(same subnetwork). */
-  if (setsockopt(broadcast_fd, IPPROTO_IP, IP_MULTICAST_TTL, (const char *)&ttl, sizeof(ttl)) < 0) {
-    mgmt_fatal(errno, "[ClusterCom::establishBroadcastChannel] Unable to setsocketopt, ttl\n");
-  }
-
-  /* Disable broadcast loopback, that is broadcasting to self */
-  if (setsockopt(broadcast_fd, IPPROTO_IP, IP_MULTICAST_LOOP, (const char *)&loop, sizeof(loop)) < 0) {
-    mgmt_fatal(errno, "[ClusterCom::establishBroadcastChannel] Unable to disable loopback\n");
-  }
-
-  return;
-} /* End ClusterCom::establishBroadcastChannel */
-
-/*
- * establishReceiveChannel()
- *   Setup our multicast channel for receiving incoming broadcasts
- * from other peers.
- */
-
-int
-ClusterCom::establishReceiveChannel(int fatal_on_error)
-{
-  if ((receive_fd = socket(AF_INET, SOCK_DGRAM, 0)) < 0) {
-    if (!fatal_on_error) {
-      Debug("ccom", "establishReceiveChannel: Unable to open socket");
-      return 1;
-    }
-    mgmt_fatal(errno, "[ClusterCom::establishReceiveChannel] Unable to open socket\n");
-  }
-
-  if (fcntl(receive_fd, F_SETFD, 1) < 0) {
-    if (!fatal_on_error) {
-      close(receive_fd);
-      receive_fd = -1;
-      Debug("ccom", "establishReceiveChannel: Unable to set close-on-exec");
-      return 1;
-    }
-    mgmt_fatal(errno, "[ClusterCom::establishReceiveChannel] Unable to set close-on-exec.\n");
-  }
-
-  int one = 1;
-  if (setsockopt(receive_fd, SOL_SOCKET, SO_REUSEADDR, (char *)&one, sizeof(int)) < 0) {
-    if (!fatal_on_error) {
-      close(receive_fd);
-      receive_fd = -1;
-      Debug("ccom", "establishReceiveChannel: Unable to set socket to reuse addr");
-      return 1;
-    }
-    mgmt_fatal(errno, "[ClusterCom::establishReceiveChannel] Unable to set socket to reuse addr.\n");
-  }
-
-  memset(&receive_addr, 0, sizeof(receive_addr));
-  receive_addr.sin_family      = AF_INET;
-  receive_addr.sin_addr.s_addr = htonl(INADDR_ANY);
-  receive_addr.sin_port        = htons(mc_port);
-
-  if (bind(receive_fd, (struct sockaddr *)&receive_addr, sizeof(receive_addr)) < 0) {
-    if (!fatal_on_error) {
-      close(receive_fd);
-      receive_fd = -1;
-      Debug("ccom", "establishReceiveChannel: Unable to bind to socket, port %d", mc_port);
-      return 1;
-    }
-    mgmt_fatal(errno, "[ClusterCom::establishReceiveChannel] Unable to bind to socket, port %d\n", mc_port);
-  }
-  /* Add ourselves to the group */
-  struct ip_mreq mc_request;
-  mc_request.imr_multiaddr.s_addr = inet_addr(mc_group);
-  mc_request.imr_interface.s_addr = htonl(INADDR_ANY);
-  if (setsockopt(receive_fd, IPPROTO_IP, IP_ADD_MEMBERSHIP, (const char *)&mc_request, sizeof(mc_request)) < 0) {
-    if (!fatal_on_error) {
-      close(receive_fd);
-      receive_fd = -1;
-      Debug("ccom", "establishReceiveChannel: Can't add ourselves to multicast group %s", mc_group);
-
-      return 1;
-    }
-    mgmt_fatal(errno, "[ClusterCom::establishReceiveChannel] Can't add ourselves to multicast group %s\n", mc_group);
-  }
-
-  return 0;
-} /* End ClusterCom::establishReceiveChannel */
-
-/*
- * sendOutgoingMessage
- *   Function basically writes a message to the broadcast_fd, it is blocking,
- * but since the drainer thread is constantly draining the network for all
- * local managers it should not block for very long.
- */
-bool
-ClusterCom::sendOutgoingMessage(char *buf, int len)
-{
-  if (mgmt_sendto(broadcast_fd, buf, len, 0, (struct sockaddr *)&broadcast_addr, sizeof(broadcast_addr)) < 0) {
-    mgmt_elog(errno, "[ClusterCom::sendOutgoingMessage] Message send failed\n");
-    return false;
-  }
-  return true;
-} /* End ClusterCom::sendOutgoingMessage */
-
-bool
-ClusterCom::sendClusterMessage(int msg_type, const char *args)
-{
-  bool ret       = true, tmp_ret;
-  char msg[1124] = {0};
-  InkHashTableEntry *entry;
-  InkHashTableIteratorState iterator_state;
-
-  switch (msg_type) {
-  case CLUSTER_MSG_SHUTDOWN_MANAGER:
-    ink_strlcpy(msg, "cmd: shutdown_manager", sizeof(msg));
-    break;
-  case CLUSTER_MSG_SHUTDOWN_PROCESS:
-    ink_strlcpy(msg, "cmd: shutdown_process", sizeof(msg));
-    break;
-  case CLUSTER_MSG_RESTART_PROCESS:
-    ink_strlcpy(msg, "cmd: restart_process", sizeof(msg));
-    break;
-  case CLUSTER_MSG_BOUNCE_PROCESS:
-    ink_strlcpy(msg, "cmd: bounce_process", sizeof(msg));
-    break;
-  case CLUSTER_MSG_CLEAR_STATS:
-    if (args) {
-      snprintf(msg, sizeof(msg), "cmd: clear_stats %1023s\n", args);
-    } else {
-      ink_strlcpy(msg, "cmd: clear_stats", sizeof(msg));
-    }
-    break;
-  default:
-    mgmt_log("[ClusterCom::sendClusterMessage] Invalid message type '%d'\n", msg_type);
-    return false;
-  }
-
-  ink_mutex_acquire(&(mutex)); /* Grab cluster lock */
-  for (entry = ink_hash_table_iterator_first(peers, &iterator_state); entry != nullptr;
-       entry = ink_hash_table_iterator_next(peers, &iterator_state)) {
-    ClusterPeerInfo *tmp = (ClusterPeerInfo *)ink_hash_table_entry_value(peers, entry);
-
-    tmp_ret = rl_sendReliableMessage(tmp->inet_address, msg, strlen(msg));
-    if (tmp->num_virt_addrs != -1) {
-      /* Only change return val if he is not dead, if dead manager could be up. */
-      ret = tmp_ret;
-    }
-  }
-  ink_mutex_release(&(mutex));
-
-  switch (msg_type) {
-  case CLUSTER_MSG_SHUTDOWN_MANAGER:
-    lmgmt->mgmt_shutdown_outstanding = MGMT_PENDING_RESTART;
-    break;
-  case CLUSTER_MSG_SHUTDOWN_PROCESS:
-    lmgmt->processShutdown();
-    break;
-  case CLUSTER_MSG_RESTART_PROCESS:
-    lmgmt->processRestart();
-    break;
-  case CLUSTER_MSG_BOUNCE_PROCESS:
-    lmgmt->processBounce();
-    break;
-  case CLUSTER_MSG_CLEAR_STATS:
-    lmgmt->clearStats(args);
-    break;
-  }
-
-  return ret;
-} /* End ClusterCom::sendClusterMessage */
-
-bool
-ClusterCom::sendReliableMessage(unsigned long addr, char *buf, int len)
-{
-  bool ret = false;
-  ink_mutex_acquire(&mutex);
-  ret = rl_sendReliableMessage(addr, buf, len);
-  ink_mutex_release(&mutex);
-  return ret;
-} /* End ClusterCom::sendReliableMessage */
-
-/*
- * rl_sendReliableMessage(...)
- *   Used to send a string across the reliable fd.
- */
-bool
-ClusterCom::rl_sendReliableMessage(unsigned long addr, const char *buf, int len)
-{
-  int fd, cport;
-  char string_addr[80];
-  struct sockaddr_in serv_addr;
-  struct in_addr address;
-  InkHashTableValue hash_value;
-
-  address.s_addr = addr;
-
-  ink_strlcpy(string_addr, inet_ntoa(address), sizeof(string_addr));
-  if (ink_hash_table_lookup(peers, string_addr, &hash_value) == 0) {
-    return false;
-  }
-  cport = ((ClusterPeerInfo *)hash_value)->ccom_port;
-
-  memset(&serv_addr, 0, sizeof(serv_addr));
-  serv_addr.sin_family      = AF_INET;
-  serv_addr.sin_addr.s_addr = addr;
-  serv_addr.sin_port        = htons(cport);
-
-  if ((fd = mgmt_socket(AF_INET, SOCK_STREAM, 0)) < 0) {
-    mgmt_elog(errno, "[ClusterCom::rl_sendReliableMessage] Unable to create socket\n");
-    return false;
-  }
-  if (fcntl(fd, F_SETFD, 1) < 0) {
-    mgmt_log("[ClusterCom::rl_sendReliableMessage] Unable to set close-on-exec.\n");
-    close(fd);
-    return false;
-  }
-
-  if (connect(fd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) {
-    mgmt_elog(errno, "[ClusterCom::rl_sendReliableMessage] Unable to connect to peer\n");
-    close_socket(fd);
-    return false;
-  }
-
-  if (mgmt_writeline(fd, buf, len) != 0) {
-    mgmt_elog(errno, "[ClusterCom::rl_sendReliableMessage] Write failed\n");
-    close_socket(fd);
-    return false;
-  }
-  close_socket(fd);
-  return true;
-} /* End ClusterCom::rl_sendReliableMessage */
-
-/*
- * sendReliableMessage(...)
- *   Used to send a string across the reliable fd.
- */
-bool
-ClusterCom::sendReliableMessage(unsigned long addr, char *buf, int len, char *reply, int len2, bool take_lock)
-{
-  int fd, cport;
-  char string_addr[80];
-  struct sockaddr_in serv_addr;
-  struct in_addr address;
-  InkHashTableValue hash_value;
-
-  address.s_addr = addr;
-  if (take_lock) {
-    ink_mutex_acquire(&mutex);
-  }
-  ink_strlcpy(string_addr, inet_ntoa(address), sizeof(string_addr));
-  if (ink_hash_table_lookup(peers, string_addr, &hash_value) == 0) {
-    if (take_lock) {
-      ink_mutex_release(&mutex);
-    }
-    return false;
-  }
-  cport = ((ClusterPeerInfo *)hash_value)->ccom_port;
-
-  memset(&serv_addr, 0, sizeof(serv_addr));
-  serv_addr.sin_family      = AF_INET;
-  serv_addr.sin_addr.s_addr = addr;
-  serv_addr.sin_port        = htons(cport);
-
-  if ((fd = mgmt_socket(AF_INET, SOCK_STREAM, 0)) < 0) {
-    mgmt_elog(errno, "[ClusterCom::sendReliableMessage] Unable to create socket\n");
-    if (take_lock) {
-      ink_mutex_release(&mutex);
-    }
-    return false;
-  }
-  if (fcntl(fd, F_SETFD, 1) < 0) {
-    mgmt_elog(errno, "[ClusterCom::sendReliableMessage] Unable to set close-on-exec.\n");
-    if (take_lock) {
-      ink_mutex_release(&mutex);
-    }
-    close(fd);
-    return false;
-  }
-
-  if (connect(fd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) {
-    mgmt_elog(errno, "[ClusterCom::sendReliableMessage] Unable to connect to peer\n");
-    if (take_lock) {
-      ink_mutex_release(&mutex);
-    }
-    close_socket(fd);
-    return false;
-  }
-
-  if (mgmt_writeline(fd, buf, len) != 0) {
-    mgmt_elog(errno, "[ClusterCom::sendReliableMessage] Write failed\n");
-    if (take_lock) {
-      ink_mutex_release(&mutex);
-    }
-    close_socket(fd);
-    return false;
-  }
-
-  if (mgmt_readline(fd, reply, len2) == -1) {
-    mgmt_elog(errno, "[ClusterCom::sendReliableMessage] Read failed\n");
-    perror("ClusterCom::sendReliableMessage");
-    reply[0] = '\0';
-    if (take_lock) {
-      ink_mutex_release(&mutex);
-    }
-    close_socket(fd);
-    return false;
-  }
-
-  close_socket(fd);
-  if (take_lock) {
-    ink_mutex_release(&mutex);
-  }
-  return true;
-} /* End ClusterCom::sendReliableMessage */
-
-/*
- * sendReliableMessage(...)
- *   Used to send a string across the reliable fd.
- */
-bool
-ClusterCom::sendReliableMessageReadTillClose(unsigned long addr, char *buf, int len, TextBuffer *reply)
-{
-  int fd, cport, res;
-  char string_addr[80], tmp_reply[1024];
-  struct sockaddr_in serv_addr;
-  struct in_addr address;
-  InkHashTableValue hash_value;
-
-  address.s_addr = addr;
-  ink_mutex_acquire(&mutex);
-  ink_strlcpy(string_addr, inet_ntoa(address), sizeof(string_addr));
-  if (ink_hash_table_lookup(peers, string_addr, &hash_value) == 0) {
-    ink_mutex_release(&mutex);
-    return false;
-  }
-  cport = ((ClusterPeerInfo *)hash_value)->ccom_port;
-
-  memset(&serv_addr, 0, sizeof(serv_addr));
-  serv_addr.sin_family      = AF_INET;
-  serv_addr.sin_addr.s_addr = addr;
-  serv_addr.sin_port        = htons(cport);
-
-  if ((fd = mgmt_socket(AF_INET, SOCK_STREAM, 0)) < 0) {
-    mgmt_elog(errno, "[ClusterCom::sendReliableMessageReadTillClose] Unable create sock\n");
-    ink_mutex_release(&mutex);
-    return false;
-  }
-  if (fcntl(fd, F_SETFD, 1) < 0) {
-    mgmt_elog(errno, "[ClusterCom::sendReliableMessageReadTillClose] Unable to set close-on-exec.\n");
-    ink_mutex_release(&mutex);
-    close(fd);
-    return false;
-  }
-
-  if (connect(fd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) {
-    mgmt_elog(errno, "[ClusterCom::sendReliableMessageReadTillClose] Unable to connect\n");
-    ink_mutex_release(&mutex);
-    close_socket(fd);
-    return false;
-  }
-
-  if (mgmt_writeline(fd, buf, len) != 0) {
-    mgmt_elog(errno, "[ClusterCom::sendReliableMessageReadTillClose] Write failed\n");
-    ink_mutex_release(&mutex);
-    close_socket(fd);
-    return false;
-  } else {
-    Debug("ccom", "[ClusterCom::sendReliableMessageREadTillClose] Sent '%s' len: %d on fd: %d", buf, len, fd);
-  }
-
-  memset(tmp_reply, 0, 1024);
-  while ((res = read_socket(fd, tmp_reply, 1022)) > 0) {
-    if (tmp_reply[0] == (char)EOF) {
-      break;
-    }
-    reply->copyFrom(tmp_reply, strlen(tmp_reply));
-    memset(tmp_reply, 0, 1024);
-  }
-
-  if (res < 0) {
-    mgmt_elog(errno, "[ClusterCom::sendReliableMessageReadTillClose] Read failed\n");
-    perror("ClusterCom::sendReliableMessageReadTillClose");
-    ink_mutex_release(&mutex);
-    close_socket(fd);
-    return false;
-  }
-
-  close_socket(fd);
-  ink_mutex_release(&mutex);
-  return true;
-} /* End ClusterCom::sendReliableMessageReadTillClose */
-
-/*
- * receiveIncomingMessage
- *   This function reads from the incoming channel. It is blocking,
- * this is ok since the channel is drained by an independent thread.
- */
-int
-ClusterCom::receiveIncomingMessage(char *buf, int max)
-{
-  int nbytes = 0, addr_len = sizeof(receive_addr);
-
-  if ((nbytes = recvfrom(receive_fd, buf, max, 0, (struct sockaddr *)&receive_addr, (socklen_t *)&addr_len)) < 0) {
-    mgmt_elog(errno, "[ClusterCom::receiveIncomingMessage] Receive failed\n");
-  }
-  return nbytes;
-} /* End ClusterCom::processIncomingMessages */
-
-/*
- * isMaster()
- *   Function checks known hosts and decides whether this local manager is
- * the current cluster master.
- */
-bool
-ClusterCom::isMaster()
-{
-  bool init_flag    = false;
-  unsigned long min = 0;
-  InkHashTableEntry *entry;
-  InkHashTableIteratorState iterator_state;
-
-  for (entry = ink_hash_table_iterator_first(peers, &iterator_state); entry != nullptr;
-       entry = ink_hash_table_iterator_next(peers, &iterator_state)) {
-    ClusterPeerInfo *pinfo = (ClusterPeerInfo *)ink_hash_table_entry_value(peers, entry);
-
-    if (!pinfo || pinfo->num_virt_addrs == -1) {
-      continue;
-    } else if (!init_flag) {
-      init_flag = true;
-      min       = pinfo->inet_address;
-    } else if (min > pinfo->inet_address) {
-      min = pinfo->inet_address;
-    }
-  }
-
-  if (our_ip == min || !init_flag || our_ip < min) {
-    return true;
-  }
-  return false;
-} /* End ClusterCom::isMaster */
-
-/*
- * lowestPeer()
- *   Function finds the peer with the lowest number of current virtual
- * interfaces. It returns the ip and sets num to the no. of connections.
- */
-unsigned long
-ClusterCom::lowestPeer(int *no)
-{
-  bool flag            = true;
-  int naddrs           = -1;
-  unsigned long min_ip = 0;
-  InkHashTableEntry *entry;
-  InkHashTableIteratorState iterator_state;
-
-  for (entry = ink_hash_table_iterator_first(peers, &iterator_state); entry != nullptr;
-       entry = ink_hash_table_iterator_next(peers, &iterator_state)) {
-    ClusterPeerInfo *tmp = (ClusterPeerInfo *)ink_hash_table_entry_value(peers, entry);
-    if (tmp->num_virt_addrs == -1) {
-      continue;
-    } else if (flag) {
-      flag   = false;
-      min_ip = tmp->inet_address;
-      naddrs = tmp->num_virt_addrs;
-    } else if (naddrs > tmp->num_virt_addrs) {
-      min_ip = tmp->inet_address;
-      naddrs = tmp->num_virt_addrs;
-    } else if (naddrs == tmp->num_virt_addrs && min_ip > tmp->inet_address) {
-      min_ip = tmp->inet_address;
-      naddrs = tmp->num_virt_addrs;
-    }
-  }
-  *no = naddrs;
-  return min_ip;
-} /* End ClusterCom::lowestPeer */
-
-void
-ClusterCom::logClusterMismatch(const char *ip, ClusterMismatch type, char *data)
-{
-  void *value;
-  ClusterMismatch stored_type;
-
-  // Check to see if we have have already logged a message of time type
-  //   for this node
-  if (ink_hash_table_lookup(mismatchLog, ip, &value)) {
-    stored_type = (ClusterMismatch)(long)value;
-
-    if (type == stored_type) {
-      return;
-    } else {
-      // The message logged is of a different type so delete the entry
-      ink_hash_table_delete(mismatchLog, ip);
-    }
-  }
-  // Log the message and store the that we've logged it in
-  //   our hash table
-  switch (type) {
-  case TS_NAME_MISMATCH:
-    mgmt_log("[ClusterCom::logClusterMismatch] Found node with ip %s.  Ignoring"
-             " since it is part of cluster %s\n",
-             ip, data);
-    break;
-  case TS_VER_MISMATCH:
-    mgmt_log("[ClusterCom::logClusterMismatch] Found node with ip %s.  Ignoring"
-             " since it is version %s (our version: %s)\n",
-             ip, data, appVersionInfo.VersionStr);
-    break;
-  default:
-    ink_assert(0);
-  }
-
-  ink_hash_table_insert(mismatchLog, ip, (void *)type);
-}
-
-/*
- * highestPeer()
- *   Function finds the peer with the highest number of current virtual
- * interfaces. It returns the ip and sets num to the no. of connections.
- */
-unsigned long
-ClusterCom::highestPeer(int *no)
-{
-  bool flag            = true;
-  int naddrs           = -1;
-  unsigned long max_ip = 0;
-  InkHashTableEntry *entry;
-  InkHashTableIteratorState iterator_state;
-
-  for (entry = ink_hash_table_iterator_first(peers, &iterator_state); entry != nullptr;
-       entry = ink_hash_table_iterator_next(peers, &iterator_state)) {
-    ClusterPeerInfo *tmp = (ClusterPeerInfo *)ink_hash_table_entry_value(peers, entry);
-    if (tmp->num_virt_addrs == -1) {
-      continue;
-    } else if (flag) {
-      flag   = false;
-      max_ip = tmp->inet_address;
-      naddrs = tmp->num_virt_addrs;
-    } else if (naddrs < tmp->num_virt_addrs) {
-      max_ip = tmp->inet_address;
-      naddrs = tmp->num_virt_addrs;
-    } else if (naddrs == tmp->num_virt_addrs && max_ip > tmp->inet_address) {
-      max_ip = tmp->inet_address;
-      naddrs = tmp->num_virt_addrs;
-    }
-  }
-  *no = naddrs;
-
-  return max_ip;
-} /* End ClusterCom::highestPeer */
-
-/*
- * checkBackDoor(...)
- *   Function checks for "backdoor" commands on the cluster port.
- */
-bool
-checkBackDoor(int req_fd, char *message)
-{
-  char reply[4096];
-
-  if (strstr(message, "show_map")) {
-    const char *tmp_msg;
-    bool map_empty = true;
-    InkHashTableEntry *entry;
-    InkHashTableIteratorState iterator_state;
-
-    ink_mutex_acquire(&(lmgmt->ccom->mutex));
-    tmp_msg = "\nLocal Map (virtual-ip):\n-----------------------\n";
-    mgmt_writeline(req_fd, tmp_msg, strlen(tmp_msg));
-    for (entry = ink_hash_table_iterator_first(lmgmt->virt_map->our_map, &iterator_state); entry != nullptr;
-         entry = ink_hash_table_iterator_next(lmgmt->virt_map->our_map, &iterator_state)) {
-      char *tmp = (char *)ink_hash_table_entry_key(lmgmt->virt_map->our_map, entry);
-      mgmt_writeline(req_fd, tmp, strlen(tmp));
-      map_empty = false;
-    }
-
-    if (map_empty) {
-      tmp_msg = "(No interfaces mapped)";
-      mgmt_writeline(req_fd, tmp_msg, strlen(tmp_msg));
-    }
-
-    map_empty = true;
-    tmp_msg   = "\nPeer Map (virtual-ip real-ip):\n------------------------------\n";
-    mgmt_writeline(req_fd, tmp_msg, strlen(tmp_msg));
-    for (entry = ink_hash_table_iterator_first(lmgmt->virt_map->ext_map, &iterator_state); entry != nullptr;
-         entry = ink_hash_table_iterator_next(lmgmt->virt_map->ext_map, &iterator_state)) {
-      char *tmp = (char *)ink_hash_table_entry_key(lmgmt->virt_map->ext_map, entry);
-      mgmt_writeline(req_fd, tmp, strlen(tmp));
-      map_empty = false;
-    }
-
-    if (map_empty) {
-      tmp_msg = "(No interfaces mapped)\n";
-      mgmt_writeline(req_fd, tmp_msg, strlen(tmp_msg));
-    } else {
-      tmp_msg = "\n\n";
-      mgmt_writeline(req_fd, tmp_msg, strlen(tmp_msg));
-    }
-    ink_mutex_release(&(lmgmt->ccom->mutex));
-    return true;
-  } else if (strstr(message, "read ")) {
-    char variable[1024];
-
-    // coverity[secure_coding]
-    if (sscanf(message, "read %s\n", variable) != 1) {
-      mgmt_elog(0, "[ClusterCom::CBD] Invalid message-line(%d) '%s'\n", __LINE__, message);
-      return false;
-    }
-
-    RecDataT stype = RECD_NULL;
-    if (RecGetRecordDataType(variable, &stype) == REC_ERR_OKAY) {
-      bool found  = false;
-      int rep_len = 0;
-
-      switch (stype) {
-      case RECD_COUNTER:
-      case RECD_INT: {
-        int64_t val = (stype == RECD_COUNTER ? REC_readCounter(variable, &found) : REC_readInteger(variable, &found));
-        if (found) {
-          rep_len = snprintf(reply, sizeof(reply), "\nRecord '%s' Val: '%" PRId64 "'\n", variable, val);
-        }
-        break;
-      }
-      case RECD_FLOAT: {
-        RecFloat val = REC_readFloat(variable, &found);
-        if (found) {
-          rep_len = snprintf(reply, sizeof(reply), "\nRecord '%s' Val: '%f'\n", variable, val);
-        }
-        break;
-      }
-      case RECD_STRING: {
-        char *val = REC_readString(variable, &found);
-        if (found) {
-          rep_len = snprintf(reply, sizeof(reply), "\nRecord '%s' Val: '%s'\n", variable, val);
-          ats_free(val);
-        }
-        break;
-      }
-      default:
-        break;
-      }
-      if (found) {
-        mgmt_writeline(req_fd, reply, rep_len);
-      } else {
-        mgmt_elog(0, "[checkBackDoor] record not found '%s'\n", variable);
-      }
-    } else {
-      mgmt_elog(0, "[checkBackDoor] Unknown variable requested '%s'\n", variable);
-    }
-    return true;
-  } else if (strstr(message, "write ")) {
-    char variable[1024], value[1024];
-
-    if (sscanf(message, "write %s %s", variable, value) != 2) {
-      mgmt_elog(0, "[ClusterCom::CBD] Invalid message-line(%d) '%s'\n", __LINE__, message);
-      return false;
-    }
-    // TODO: I think this is correct, it used to do lmgmt->record_data-> ...
-    if (RecSetRecordConvert(variable, value, REC_SOURCE_EXPLICIT, true, false) == REC_ERR_OKAY) {
-      ink_strlcpy(reply, "\nRecord Updated\n\n", sizeof(reply));
-      mgmt_writeline(req_fd, reply, strlen(reply));
-    } else {
-      mgmt_elog(0, "[checkBackDoor] Assignment to unknown variable requested '%s'\n", variable);
-    }
-    return true;
-  } else if (strstr(message, "peers")) {
-    InkHashTableEntry *entry;
-    InkHashTableIteratorState iterator_state;
-
-    ink_mutex_acquire(&(lmgmt->ccom->mutex));
-
-    for (entry = ink_hash_table_iterator_first(lmgmt->ccom->peers, &iterator_state); entry != nullptr;
-         entry = ink_hash_table_iterator_next(lmgmt->ccom->peers, &iterator_state)) {
-      const char *tmp_msg;
-      char ip_addr[80];
-      struct in_addr addr;
-
-      ClusterPeerInfo *tmp = (ClusterPeerInfo *)ink_hash_table_entry_value(lmgmt->ccom->peers, entry);
-
-      tmp_msg = "---------------------------";
-      mgmt_writeline(req_fd, tmp_msg, strlen(tmp_msg));
-
-      addr.s_addr = tmp->inet_address;
-      ink_strlcpy(ip_addr, inet_ntoa(addr), sizeof(ip_addr));
-      snprintf(reply, sizeof(reply), "Peer: %s   naddrs: %d", ip_addr, tmp->num_virt_addrs);
-      mgmt_writeline(req_fd, reply, strlen(reply));
-
-      snprintf(reply, sizeof(reply), "Cluster Port(proxy): %d  RS Port(mgmt): %d", tmp->port, tmp->ccom_port);
-      mgmt_writeline(req_fd, reply, strlen(reply));
-
-      snprintf(reply, sizeof(reply),
-               "Idle-Our-WC: %" PRId64 "   Peer-WC-Last-Time: %ld  Delta: %ld Mgmt-Idle: %" PRId64 " M-Alive: %d",
-               (int64_t)tmp->idle_ticks, tmp->last_time_recorded, tmp->delta, (int64_t)tmp->manager_idle_ticks, tmp->manager_alive);
-      mgmt_writeline(req_fd, reply, strlen(reply));
-
-      tmp_msg = "---------------------------\n";
-      mgmt_writeline(req_fd, tmp_msg, strlen(tmp_msg));
-    }
-    ink_mutex_release(&(lmgmt->ccom->mutex));
-    return true;
-  } else if (strstr(message, "dump: lm")) {
-    ink_strlcpy(reply, "---------------------------", sizeof(reply));
-    mgmt_writeline(req_fd, reply, strlen(reply));
-    ink_strlcpy(reply, "Local Manager:\n", sizeof(reply));
-    mgmt_writeline(req_fd, reply, strlen(reply));
-
-    snprintf(reply, sizeof(reply), "\tproxy_running: %s", (lmgmt->proxy_running ? "true" : "false"));
-    mgmt_writeline(req_fd, reply, strlen(reply));
-
-    snprintf(reply, sizeof(reply), "\tproxy_started_at: %" PRId64 "", (int64_t)lmgmt->proxy_started_at);
-    mgmt_writeline(req_fd, reply, strlen(reply));
-
-    snprintf(reply, sizeof(reply), "\trun_proxy: %s", (lmgmt->run_proxy ? "true" : "false"));
-    mgmt_writeline(req_fd, reply, strlen(reply));
-
-    snprintf(reply, sizeof(reply), "\tproxy_launch_oustanding: %s", (lmgmt->proxy_launch_outstanding ? "true" : "false"));
-    mgmt_writeline(req_fd, reply, strlen(reply));
-
-    snprintf(reply, sizeof(reply), "\tmgmt_shutdown_outstanding: %s\n", (lmgmt->mgmt_shutdown_outstanding ? "true" : "false"));
-    mgmt_writeline(req_fd, reply, strlen(reply));
-
-// XXX: Again multiple code caused by misssing PID_T_FMT
-// TODO: Was #if defined(solaris) && (!defined(_FILE_OFFSET_BITS) || _FILE_OFFSET_BITS != 64)
-#if defined(solaris)
-    snprintf(reply, sizeof(reply), "\twatched_process_fd: %d  watched_process_pid: %ld\n", lmgmt->watched_process_fd,
-             (long int)lmgmt->watched_process_pid);
-#else
-    snprintf(reply, sizeof(reply), "\twatched_process_fd: %d  watched_process_pid: %d\n", lmgmt->watched_process_fd,
-             lmgmt->watched_process_pid);
-#endif
-    mgmt_writeline(req_fd, reply, strlen(reply));
-
-    ink_strlcpy(reply, "---------------------------\n", sizeof(reply));
-    mgmt_writeline(req_fd, reply, strlen(reply));
-    return true;
-  } else if (strstr(message, "cluster: ")) {
-    int msg_type;
-    char *args = nullptr;
-
-    if (strstr(message, "cluster: shutdown_manager")) {
-      msg_type = CLUSTER_MSG_SHUTDOWN_MANAGER;
-    } else if (strstr(message, "cluster: shutdown_process")) {
-      msg_type = CLUSTER_MSG_SHUTDOWN_PROCESS;
-    } else if (strstr(message, "cluster: restart_process")) {
-      msg_type = CLUSTER_MSG_RESTART_PROCESS;
-    } else if (strstr(message, "cluster: bounce_process")) {
-      msg_type = CLUSTER_MSG_BOUNCE_PROCESS;
-    } else if (strstr(message, "cluster: clear_stats")) {
-      if (strlen(message) > sizeof("cluster: clear_stats") + 1) {
-        args = message + sizeof("cluster: clear_stats") + 1;
-      }
-      msg_type = CLUSTER_MSG_CLEAR_STATS;
-    } else {
-      return false;
-    }
-    lmgmt->ccom->sendClusterMessage(msg_type, args);
-    return true;
-  }
-  return false;
-} /* End checkBackDoor */
diff --git a/mgmt/cluster/ClusterCom.h b/mgmt/cluster/ClusterCom.h
deleted file mode 100644
index 55f2b26..0000000
--- a/mgmt/cluster/ClusterCom.h
+++ /dev/null
@@ -1,194 +0,0 @@
-/** @file
-
-  A brief file description
-
-  @section license License
-
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
- */
-
-/**************************************
- *
- * ClusterCom.h
- *   Cluster communications class. Wrapper around communication
- * channel used. This class is basically another agent in that it
- * synthezises out bound data and collects incoming(user in the case
- * of other agents, other nodes in this case) data.
- *
- * A thread is spun up at runtime to drain the incoming channel, and
- * integrate it with other node data. Out bound is done by the main
- * mgmt thread. This scheme should insure that the read buffer is always
- * drained.
- *
- * $Date: 2007-10-05 16:56:45 $
- *
- *
- */
-
-#ifndef _CLUSTER_COM_H
-#define _CLUSTER_COM_H
-
-#include "ts/ink_platform.h"
-#include "P_RecDefs.h"
-#include "ts/I_Version.h"
-#include "Rollback.h"
-
-class FileManager;
-
-#define CLUSTER_MSG_SHUTDOWN_MANAGER 1000
-#define CLUSTER_MSG_SHUTDOWN_PROCESS 1001
-#define CLUSTER_MSG_RESTART_PROCESS 1002
-#define CLUSTER_MSG_BOUNCE_PROCESS 1003
-#define CLUSTER_MSG_CLEAR_STATS 1004
-
-#define MAX_MC_GROUP_LEN 20
-#define MAX_NODE_SYSINFO_STRING 32
-
-#define CLUSTER_CONFIG_FILE_BLURB                                                                                                  \
-  "# Cluster Configuration file\n#\n# This file is machine generated and machine parsed.\n# Please do not change this file by "    \
-  "hand.\n#\n# This file designates the machines which make up the cluster\n# proper.  Data and load are distributed among these " \
-  "machines.\n#\n############################################################################\n# Number\n# IP:Port \n# "           \
-  "...\n############################################################################\n# Number = { 0, 1 ... } where 0 is a "       \
-  "stand-alone proxy\n# IP:Port = IP address: cluster accept port number\n#\n# Example 1: stand-alone proxy\n# 0\n#\n# Example "   \
-  "2: 3 machines\n# 3\n# 127.1.2.3:83\n# 127.1.2.4:83\n# 127.1.2.5:83\n#\n"
-
-enum MgmtClusterType {
-  CLUSTER_INVALID = 0,
-  FULL_CLUSTER,
-  MGMT_CLUSTER,
-  NO_CLUSTER,
-};
-
-enum ClusterMismatch {
-  TS_NAME_MISMATCH,
-  TS_VER_MISMATCH,
-};
-
-typedef struct _cluster_peer_info {
-  unsigned long inet_address; /* IP addr of node */
-  int port;                   /* Cluster port */
-  int ccom_port;              /* CCom reliable port */
-  time_t idle_ticks;          /* Number of ticks since last heard */
-  time_t manager_idle_ticks;  /* Time last manager heartbeat received */
-  int manager_alive;
-
-  long last_time_recorded;
-  long delta;
-
-  int num_virt_addrs;
-
-  RecRecords node_rec_data;
-
-} ClusterPeerInfo;
-
-class ClusterCom
-{
-public:
-  ClusterCom(unsigned long oip, char *hname, int mcport, char *group, int rsport, char *p);
-  ~ClusterCom(){};
-
-  void checkPeers(time_t *ticker);
-  void generateClusterDelta(void);
-  void handleMultiCastMessage(char *message);
-  void handleMultiCastFilePacket(char *last, char *from_ip);
-  void handleMultiCastStatPacket(char *last, ClusterPeerInfo *peer);
-  void handleMultiCastAlarmPacket(char *last, char *from_ip);
-  void handleMultiCastVMapPacket(char *last, char *from_ip);
-
-  void establishChannels();
-  void establishBroadcastChannel();
-  int establishReceiveChannel(int fatal_on_error = 1);
-
-  bool sendSharedData(bool send_proxy_heart_beat = true);
-  void constructSharedGenericPacket(char *message, int max, RecT packet_type);
-  void constructSharedStatPacket(char *message, int max);
-  void constructSharedFilePacket(char *message, int max);
-
-  static int constructSharedPacketHeader(const AppVersionInfo &version, char *message, char *ip, int max);
-
-  bool sendClusterMessage(int msg_type, const char *args = NULL);
-  bool sendOutgoingMessage(char *buf, int len);
-  bool sendReliableMessage(unsigned long addr, char *buf, int len);
-
-  bool rl_sendReliableMessage(unsigned long addr, const char *buf, int len);
-  bool sendReliableMessage(unsigned long addr, char *buf, int len, char *reply, int len2, bool take_lock);
-  bool sendReliableMessageReadTillClose(unsigned long addr, char *buf, int len, TextBuffer *reply);
-
-  int receiveIncomingMessage(char *buf, int max);
-
-  bool isMaster();
-  unsigned long lowestPeer(int *no);
-  unsigned long highestPeer(int *no);
-
-  unsigned long
-  getIP()
-  {
-    return our_ip;
-  }
-
-  void logClusterMismatch(const char *ip, ClusterMismatch type, char *data);
-
-  InkHashTable *mismatchLog; // drainer thread use only
-
-  bool init;
-
-  MgmtClusterType cluster_type;
-
-  unsigned long our_ip;
-  char our_host[1024];
-
-  AppVersionInfo appVersionInfo;
-  char sys_name[MAX_NODE_SYSINFO_STRING];
-  char sys_release[MAX_NODE_SYSINFO_STRING];
-
-  time_t delta_thresh;
-  time_t peer_timeout;
-  time_t mc_send_interval;
-  time_t mc_poll_timeout;
-  time_t our_wall_clock;
-  time_t startup_timeout;
-  time_t startup_time;
-  time_t last_shared_send;
-
-  int log_bogus_mc_msgs;
-
-  ink_mutex mutex;
-  InkHashTable *peers;
-  volatile int alive_peers_count;
-
-  char cluster_conf[1024];
-  Rollback *cluster_file_rb;
-  FileManager *configFiles;
-
-  int cluster_port;
-  int reliable_server_port;
-  int reliable_server_fd;
-  int broadcast_fd;
-  int receive_fd;
-
-  int mc_port;
-  int mc_ttl;
-  char mc_group[MAX_MC_GROUP_LEN];
-
-  struct sockaddr_in broadcast_addr;
-  struct sockaddr_in receive_addr;
-
-}; /* End class ClusterCom */
-
-struct in_addr *mgmt_sortipaddrs(int num, struct in_addr **list);
-
-#endif /* _CLUSTER_COM_H */
diff --git a/mgmt/cluster/Makefile.am b/mgmt/cluster/Makefile.am
deleted file mode 100644
index 96ad0d4..0000000
--- a/mgmt/cluster/Makefile.am
+++ /dev/null
@@ -1,38 +0,0 @@
-#
-# Makefile.am for the Enterprise Management module.
-#
-#  Licensed to the Apache Software Foundation (ASF) under one
-#  or more contributor license agreements.  See the NOTICE file
-#  distributed with this work for additional information
-#  regarding copyright ownership.  The ASF licenses this file
-#  to you under the Apache License, Version 2.0 (the
-#  "License"); you may not use this file except in compliance
-#  with the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-include $(top_srcdir)/build/tidy.mk
-
-AM_CPPFLAGS += \
-  -I$(abs_top_srcdir)/lib/records \
-  -I$(abs_top_srcdir)/mgmt \
-  -I$(abs_top_srcdir)/mgmt/api/include \
-  -I$(abs_top_srcdir)/mgmt/utils \
-  -I$(abs_top_srcdir)/lib
-
-noinst_LTLIBRARIES = libcluster.la
-
-libcluster_la_SOURCES = \
-  ClusterCom.cc \
-  ClusterCom.h \
-  VMap.cc \
-  VMap.h
-
-tidy-local: $(DIST_SOURCES)
-	$(CXX_Clang_Tidy)
diff --git a/mgmt/cluster/VMap.cc b/mgmt/cluster/VMap.cc
deleted file mode 100644
index a175c3e..0000000
--- a/mgmt/cluster/VMap.cc
+++ /dev/null
@@ -1,863 +0,0 @@
-/** @file
-
-  A brief file description
-
-  @section license License
-
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
- */
-
-/************************************** */
-/*
- * VMap.c
- *   Function defs for the virtual mapping mgmt stuff.
- *
- *
- */
-#include "ts/ink_platform.h"
-
-#include "LocalManager.h"
-#include "VMap.h"
-#include "ClusterCom.h"
-#include "MgmtUtils.h"
-#include "P_RecLocal.h"
-#include "ts/I_Layout.h"
-
-// for linux and freebsd
-#ifndef C_ISUID
-#define C_ISUID S_ISUID
-#endif
-
-int
-vmapEnableHandler(const char *tok, RecDataT /* data_type ATS_UNUSED */, RecData data, void * /* cookie ATS_UNUSED */)
-{
-  bool before = true;
-  ink_assert(!tok);
-  if (!lmgmt->virt_map->enabled) {
-    before = false;
-  }
-  lmgmt->virt_map->enabled = (RecInt)data.rec_int;
-  if (!lmgmt->virt_map->enabled && before) {
-    lmgmt->virt_map->turning_off = true; // turing VIP from off to on
-    lmgmt->virt_map->downAddrs();
-  }
-  return 0;
-} /* End vmapEnableHandler */
-
-VMap::VMap(char *interface, unsigned long ip, ink_mutex *m)
-{
-  bool found;
-
-  while (!m) {
-    mgmt_sleep_sec(1);
-  } // Wait until mutex pointer is initialized
-  mutex = m;
-
-  our_ip               = ip;
-  num_interfaces       = 0;
-  id_map               = nullptr;
-  interface_realip_map = ink_hash_table_create(InkHashTableKeyType_String);
-  our_map              = ink_hash_table_create(InkHashTableKeyType_String);
-  ext_map              = ink_hash_table_create(InkHashTableKeyType_String);
-  addr_list            = nullptr;
-  num_addrs            = 0;
-  num_nics             = 0;
-
-  this->interface = ats_strdup(interface);
-  turning_off     = false; // we are not turning off VIP
-
-  enabled = REC_readInteger("proxy.config.vmap.enabled", &found);
-  /*
-   * Perpetuating a hack for the cluster interface. Here I want to loop
-   * at startup(before any virtual ips have been brought up) and get the real
-   * IP address for each interface.
-   *
-   * Later this will be used to ping the interfaces that have virtual IP addrs
-   * associated with them to detect if the interface is down.
-   */
-  { /* First Add the cluster interface */
-    RealIPInfo *tmp_realip_info;
-    struct in_addr tmp_addr;
-
-    tmp_addr.s_addr = ip;
-
-    tmp_realip_info                         = (RealIPInfo *)ats_malloc(sizeof(RealIPInfo));
-    tmp_realip_info->real_ip                = tmp_addr;
-    tmp_realip_info->mappings_for_interface = true;
-
-    num_nics++;
-    ink_hash_table_insert(interface_realip_map, interface, (void *)tmp_realip_info);
-    if (enabled) {
-      mgmt_log("[VMap::Vmap] Added cluster interface '%s' real ip: '%s' to known interfaces\n", interface,
-               inet_ntoa(tmp_realip_info->real_ip));
-    }
-  }
-  {
-    int tmp_socket;
-    struct sockaddr_in *tmp;   // a tmp ptr for addresses
-    struct ifconf ifc;         // ifconf information
-    char *ifbuf;               // ifconf buffer
-    struct ifreq *ifr, *ifend; // pointer to individual inferface info
-    int lastlen;
-    int len;
-
-    if ((tmp_socket = socket(AF_INET, SOCK_DGRAM, 0)) < 0) {
-      mgmt_fatal(errno, "[VMap::VMap] Unable to create socket for interface ioctls\n");
-    }
-    // INKqa06739
-    // Fetch the list of network interfaces
-    // . from Stevens, Unix Network Prog., pg 434-435
-    ifbuf   = nullptr;
-    lastlen = 0;
-    len     = 128 * sizeof(struct ifreq); // initial buffer size guess
-    for (;;) {
-      ifbuf = (char *)ats_malloc(len);
-      memset(ifbuf, 0, len); // prevent UMRs
-      ifc.ifc_len = len;
-      ifc.ifc_buf = ifbuf;
-      if (ioctl(tmp_socket, SIOCGIFCONF, &ifc) < 0) {
-        if (errno != EINVAL || lastlen != 0) {
-          mgmt_fatal(errno, "[VMap::VMap] Unable to read network interface configuration\n");
-        }
-      } else {
-        if (ifc.ifc_len == lastlen) {
-          break;
-        }
-        lastlen = ifc.ifc_len;
-      }
-      len *= 2;
-      ats_free(ifbuf);
-    }
-
-    ifend = (struct ifreq *)(ifc.ifc_buf + ifc.ifc_len);
-    // Loop through the list of interfaces
-    for (ifr = ifc.ifc_req; ifr < ifend;) {
-      if (ifr->ifr_addr.sa_family == AF_INET && strcmp(ifr->ifr_name, "lo0") != 0 &&
-          !strstr(ifr->ifr_name, ":")) { // Don't count the loopback interface
-
-        // Get the address of the interface
-        if (ioctl(tmp_socket, SIOCGIFADDR, (char *)ifr) < 0) {
-          mgmt_log("[VMap::VMap] Unable obtain address for network interface %s, presuming unused\n", ifr->ifr_name);
-        } else {
-          InkHashTableValue hash_value;
-
-          // Only look at the address if it an internet address
-          if (ifr->ifr_ifru.ifru_addr.sa_family == AF_INET) {
-            RealIPInfo *tmp_realip_info;
-
-            tmp = (struct sockaddr_in *)&ifr->ifr_ifru.ifru_addr;
-
-            tmp_realip_info                         = (RealIPInfo *)ats_malloc(sizeof(RealIPInfo));
-            tmp_realip_info->real_ip                = tmp->sin_addr;
-            tmp_realip_info->mappings_for_interface = false;
-
-            if (ink_hash_table_lookup(interface_realip_map, ifr->ifr_name, &hash_value) != 0) {
-              if (enabled) {
-                mgmt_log("[VMap::VMap] Already added interface '%s'. Not adding for"
-                         " real IP '%s'\n",
-                         ifr->ifr_name, inet_ntoa(tmp_realip_info->real_ip));
-              }
-              ats_free(tmp_realip_info);
-            } else {
-              ink_hash_table_insert(interface_realip_map, ifr->ifr_name, (void *)tmp_realip_info);
-              num_nics++;
-              if (enabled) {
-                mgmt_log("[VMap::Vmap] Added interface '%s' real ip: '%s' to known interfaces\n", ifr->ifr_name,
-                         inet_ntoa(tmp_realip_info->real_ip));
-              }
-            }
-          } else {
-            if (enabled) {
-              mgmt_log("[VMap::VMap] Interface %s is not configured for IP.\n", ifr->ifr_name);
-            }
-          }
-        }
-      }
-#if defined(freebsd) || defined(darwin)
-      ifr = (struct ifreq *)((char *)&ifr->ifr_addr + ifr->ifr_addr.sa_len);
-#else
-      ifr = (struct ifreq *)(((char *)ifr) + sizeof(*ifr));
-#endif
-    }
-    ats_free(ifbuf);
-    close(tmp_socket);
-  }
-
-  RecRegisterConfigUpdateCb("proxy.config.vmap.enabled", vmapEnableHandler, nullptr);
-
-  down_up_timeout = REC_readInteger("proxy.config.vmap.down_up_timeout", &found);
-  addr_list_fname = REC_readString("proxy.config.vmap.addr_file", &found);
-  lt_readAListFile(addr_list_fname);
-
-  map_change_thresh = 10;
-  last_map_change   = time(nullptr);
-
-  return;
-
-} /* End VMap::VMap */
-
-VMap::~VMap()
-{
-  if (id_map) {
-    ink_hash_table_destroy_and_free_values(id_map);
-  }
-
-  ink_hash_table_destroy_and_free_values(interface_realip_map);
-  ink_hash_table_destroy(our_map);
-  ink_hash_table_destroy(ext_map);
-  ats_free(this->interface);
-  ats_free(addr_list);
-} /* End VMap::~VMap */
-
-/*
- * lt_runGambit()
- *   Function basically runs the virtual ip assignment gambit. If you are
- * the current cluster master you will check for rebalancing or un-assigned
- * interfaces and distribute them. Both cluster master and other nodes will
- * check for conflicts between their virtual interfaces and other nodes in the
- * cluster.
- */
-void
-VMap::lt_runGambit()
-{
-  int i, no = 0;
-  char vaddr[80], raddr[80], *conf_addr = nullptr;
-  bool init = false;
-  struct in_addr virtual_addr, real_addr;
-
-  if (!enabled) {
-    return;
-  }
-  if (!((time(nullptr) - lmgmt->ccom->startup_time) > lmgmt->ccom->startup_timeout)) {
-    return;
-  }
-  if (num_addrs == 0) {
-    return;
-  }
-
-  ink_mutex_acquire(mutex);
-  if (lmgmt->ccom->isMaster()) { /* Are we the cluster master? */
-
-    for (i = 0; i < num_addrs; i++) { /* See if there is an unbound interface */
-      virtual_addr.s_addr = addr_list[i];
-      ink_strlcpy(vaddr, inet_ntoa(virtual_addr), sizeof(vaddr));
-      if (rl_boundAddr(vaddr) == 0) {
-        mgmt_log("[VMap::lt_runGambit] Unmapped vaddr: '%s'\n", vaddr);
-        break;
-      }
-    }
-
-    if (i != num_addrs) { /* Got one to map, find a candidate and map it */
-      real_addr.s_addr = lmgmt->ccom->lowestPeer(&no);
-      if (no != -1) { /* Make sure we have peers to map interfaces to */
-        init = true;
-      }
-
-      if (init && ((no < num_interfaces) || (no == num_interfaces && real_addr.s_addr < our_ip))) {
-        ink_strlcpy(raddr, inet_ntoa(real_addr), sizeof(raddr));
-        rl_remote_map(vaddr, raddr);
-      } else if (!rl_map(vaddr)) { /* We are the winner, map it to us */
-        mgmt_elog(0, "[VMap::lt_runGambit] Map failed for vaddr: %s\n", vaddr);
-      } else {
-        mgmt_log("[VMap::lt_runGambit] Map succeeded for vaddr: %s\n", vaddr);
-      }
-    }
-  }
-
-  for (i = 0; i < num_addrs; i++) { /* Check for conflicts with your interfaces */
-    virtual_addr.s_addr = addr_list[i];
-    ink_strlcpy(vaddr, inet_ntoa(virtual_addr), sizeof(vaddr));
-
-    if ((conf_addr = rl_checkConflict(vaddr))) {
-      mgmt_log("[VMap::lt_runGambit] Conflict w/addr: '%s' - Unable to use virtual address.\n", vaddr);
-      ats_free(conf_addr);
-      break;
-    }
-  }
-
-  ink_mutex_release(mutex);
-  return;
-} /* End VMap::lt_runGambit */
-
-/*
- * lt_readAListFile(...)
- *   Function reads in the virtual ip list, basically a parsing routine for the
- * vaddr file.
- */
-void
-VMap::lt_readAListFile(const char *fname)
-{
-  int tmp_num_addrs = 0;
-  char buf[1024];
-  char tmp_addr[1024], tmp_interface[1024];
-  FILE *fin;
-  char tmp_id[1024];
-  ats_scoped_str vaddr_path(RecConfigReadConfigPath(nullptr, fname));
-
-  if (!(fin = fopen(vaddr_path, "r"))) {
-    mgmt_log("[VMap::lt_readAListFile] Unable to open file: %s, addr list unchanged\n", (const char *)vaddr_path);
-    return;
-  }
-
-  ink_mutex_acquire(mutex);
-  rl_downAddrs(); /* Down everything before we re-init */
-  if (id_map) {
-    ink_hash_table_destroy_and_free_values(id_map);
-  }
-
-  id_map = ink_hash_table_create(InkHashTableKeyType_String);
-  while (fgets(buf, 1024, fin)) {
-    // since each of the tmp_addr, tmp_interface, tmp_id has length 1024 which is not less than buf
-    // so here we don't need to worry about overflow, disable coverity check for this line
-    // coverity[secure_coding]
-    if (buf[0] != '#' && isascii(buf[0]) && isdigit(buf[0]) && (sscanf(buf, "%s %s %s\n", tmp_addr, tmp_interface, tmp_id) == 3)) {
-      tmp_num_addrs++;
-    }
-  }
-
-  num_addrs = tmp_num_addrs;
-  if (num_addrs) {
-    addr_list = (unsigned long *)ats_malloc(sizeof(unsigned long) * num_addrs);
-  } else { /* Handle the case where there are no addrs in the file */
-    addr_list = nullptr;
-    fclose(fin);
-    ink_mutex_release(mutex);
-    return;
-  }
-
-  tmp_num_addrs = 0;
-  rewind(fin);
-  while (fgets(buf, 1024, fin)) {
-    VIPInfo *tmp_val;
-    InkHashTableValue hash_value;
-
-    /* Make sure we have a valid line and its not commented */
-    // since each of the tmp_addr, tmp_interface, tmp_id has length 1024 which is not less than buf
-    // so here we don't need to worry about overflow, disable coverity check for this line
-    // coverity[secure_coding]
-    if (!isascii(buf[0]) || !isdigit(buf[0]) || (sscanf(buf, "%s %s %s\n", tmp_addr, tmp_interface, tmp_id) != 3)) {
-      continue;
-    }
-    mgmt_log("[VMap::lt_readAListFile] Adding virtual address '%s' interface: '%s'"
-             " sub-interface-id '%s'\n",
-             tmp_addr, tmp_interface, tmp_id);
-
-    addr_list[tmp_num_addrs++] = inet_addr(tmp_addr);
-
-    tmp_val = (VIPInfo *)ats_malloc(sizeof(VIPInfo));
-
-    ink_strlcpy(tmp_val->interface, tmp_interface, sizeof(tmp_val->interface));
-    ink_strlcpy(tmp_val->sub_interface_id, tmp_id, sizeof(tmp_val->sub_interface_id));
-    ink_hash_table_insert(id_map, tmp_addr, (void *)tmp_val);
-
-    // we don't need to do mgmt ping stuff on NT the way its done on UNIX
-    if (ink_hash_table_lookup(interface_realip_map, tmp_interface, &hash_value) != 0) {
-      if (!((RealIPInfo *)hash_value)->mappings_for_interface) {
-        ((RealIPInfo *)hash_value)->mappings_for_interface = true;
-        mgmt_log("[VMap::lt_readAListFile] Interface '%s' marked as having potential"
-                 " virtual ips\n",
-                 tmp_interface);
-      }
-    } else {
-      mgmt_elog(0, "[VMap::lt_readAListFile] VIP in config file but no interface"
-                   " '%s' present on node.\n",
-                tmp_interface);
-    }
-  }
-  fclose(fin);
-
-  ink_mutex_release(mutex);
-  return;
-} /* End VMap::lt_readAListFile */
-
-/*
- * rl_resetSeenFlag(...)
- *   Function resets the "seen" flag for a given peer's mapped addrs.
- */
-void
-VMap::rl_resetSeenFlag(char *ip)
-{
-  InkHashTableEntry *entry;
-  InkHashTableIteratorState iterator_state;
-
-  for (entry = ink_hash_table_iterator_first(ext_map, &iterator_state); entry != nullptr;
-       entry = ink_hash_table_iterator_next(ext_map, &iterator_state)) {
-    char *key = (char *)ink_hash_table_entry_key(ext_map, entry);
-    bool *tmp = (bool *)ink_hash_table_entry_value(ext_map, entry);
-
-    if (strstr(key, ip)) {
-      *tmp = false;
-    }
-  }
-  return;
-} /* End VMap::rl_resetSeenFlag */
-
-/*
- * rl_clearUnSeen(...)
- *   This function is a sweeper function to clean up the map.
- */
-int
-VMap::rl_clearUnSeen(char *ip)
-{
-  int numAddrs = 0;
-  InkHashTableEntry *entry;
-  InkHashTableIteratorState iterator_state;
-
-  for (entry = ink_hash_table_iterator_first(ext_map, &iterator_state); entry != nullptr;
-       entry = ink_hash_table_iterator_next(ext_map, &iterator_state)) {
-    char *key = (char *)ink_hash_table_entry_key(ext_map, entry);
-    bool *tmp = (bool *)ink_hash_table_entry_value(ext_map, entry);
-
-    if (strstr(key, ip)) {
-      if (!*tmp) {
-        ink_hash_table_delete(ext_map, key); /* Safe in iterator? */
-        ats_free(tmp);
-      } else {
-        numAddrs++;
-      }
-    }
-  }
-  return numAddrs;
-} /* End VMap::rl_clearUnSeen */
-
-/*
- * rl_remote_map(...)
- *   Function sends the up interface command to a remote node.
- */
-bool
-VMap::rl_remote_map(char *virt_ip, char *real_ip)
-{
-  char buf[1024], reply[4096];
-
-  snprintf((char *)buf, sizeof(buf), "map: %s", virt_ip);
-  if (!(lmgmt->ccom->sendReliableMessage(inet_addr(real_ip), buf, strlen(buf), reply, 4096, false))) {
-    mgmt_elog(errno, "[VMap::rl_remote_map] Reliable send failed\n");
-    return false;
-  } else if (strcmp(reply, "map: failed") == 0) {
-    mgmt_log("[VMap::rl_remote_map] Mapping failed\n");
-    return false;
-  }
-  rl_map(virt_ip, real_ip);
-  return true;
-} /* End VMap::rl_remote_map */
-
-/*
- * rl_remote_unmap(...)
- *   Function sends the up interface command to a remote node.
- */
-bool
-VMap::rl_remote_unmap(char *virt_ip, char *real_ip)
-{
-  char buf[1024], reply[4096];
-
-  snprintf((char *)buf, sizeof(buf), "unmap: %s", virt_ip);
-  if (!(lmgmt->ccom->sendReliableMessage(inet_addr(real_ip), buf, strlen(buf), reply, 4096, false))) {
-    mgmt_elog(errno, "[VMap::rl_remote_unmap] Reliable send failed\n");
-    return false;
-  } else if (strcmp((char *)reply, "unmap: failed") == 0) {
-    mgmt_log("[VMap::rl_remote_unmap] Mapping failed\n");
-    return false;
-  }
-  return true;
-} /* End VMap::rl_remote_unmap */
-
-/*
- * rl_map(...)
- *   Function maps a virt_ip to a real_ip, if real_ip is NULL it maps it
- *   to the local node itself. Otherwise it means another node within
- *   the multicast cluster has it.
- */
-bool
-VMap::rl_map(char *virt_ip, char *real_ip)
-{
-  char buf[80];
-  bool *entry;
-  InkHashTable *tmp;
-  InkHashTableValue hash_value;
-
-  if (real_ip) {
-    tmp = ext_map;
-    snprintf(buf, sizeof(buf), "%s %s", virt_ip, real_ip);
-  } else {
-    tmp = our_map;
-    ink_strlcpy(buf, virt_ip, sizeof(buf));
-  }
-
-  if (ink_hash_table_lookup(tmp, buf, &hash_value) != 0) {
-    *((bool *)hash_value) = true;
-    return false;
-  }
-
-  entry  = (bool *)ats_malloc(sizeof(bool));
-  *entry = true;
-
-  if (!real_ip) {
-    mgmt_elog(0, "[VMap::rl_map] no real ip associated with virtual ip %s, mapping to local\n", buf);
-    last_map_change = time(nullptr);
-  }
-  ink_hash_table_insert(tmp, buf, (void *)entry);
-  return true;
-} /* End VMap::rl_map */
-
-bool
-VMap::rl_unmap(char *virt_ip, char *real_ip)
-{
-  char buf[80];
-  InkHashTable *tmp;
-  InkHashTableValue hash_value;
-
-  if (real_ip) {
-    tmp = ext_map;
-    snprintf(buf, sizeof(buf), "%s %s", virt_ip, real_ip);
-  } else {
-    tmp = our_map;
-    ink_strlcpy(buf, virt_ip, sizeof(buf));
-  }
-
-  if (ink_hash_table_lookup(tmp, buf, &hash_value) == 0) {
-    return false;
-  }
-
-  if (!real_ip) {
-    last_map_change = time(nullptr);
-  }
-  ink_hash_table_delete(tmp, buf);
-  ats_free(hash_value);
-  return true;
-} /* End VMap::rl_unmap */
-
-/*
- * rl_checkConflict(...)
- *   This function checks for virt conflicts between the local node and
- * any peers. It returns NULL on no conflict or the real ip of the peer
- * the local node is in contention with.
- */
-char *
-VMap::rl_checkConflict(char *virt_ip)
-{
-  char *key       = nullptr;
-  bool in_our_map = false, in_ext_map = false;
-  InkHashTableValue hash_value;
-  InkHashTableEntry *entry;
-  InkHashTableIteratorState iterator_state;
-
-  if ((time(nullptr) - last_map_change) < map_change_thresh) {
-    return nullptr;
-  }
-
-  for (entry = ink_hash_table_iterator_first(ext_map, &iterator_state); (entry != nullptr && !in_ext_map);
-       entry = ink_hash_table_iterator_next(ext_map, &iterator_state)) {
-    key = (char *)ink_hash_table_entry_key(ext_map, entry);
-    if (strstr(key, virt_ip)) {
-      in_ext_map = true;
-    }
-  }
-
-  if (ink_hash_table_lookup(our_map, virt_ip, &hash_value) != 0) {
-    in_our_map = true;
-  }
-
-  if (in_our_map && in_ext_map) {
-    char *buf, buf2[80];
-
-    if ((buf = strstr(key, " ")) != nullptr) {
-      buf++;
-      ink_strlcpy(buf2, buf, sizeof(buf2));
-    } else {
-      mgmt_fatal(0, "[VMap::rl_checkConflict] Corrupt VMap entry('%s'), bailing\n", key);
-    }
-    return ats_strdup(buf2);
-  }
-  return nullptr;
-} /* End VMap::rl_checkConflict */
-
-/*
- * checkGlobConflict(...)
- *   This function checks for conflict in the local map as well as the
- * global map. It returns false on no conflict and true on conflict.
- */
-bool
-VMap::rl_checkGlobConflict(char *virt_ip)
-{
-  bool in_ext_map = false, ret = false;
-  InkHashTableValue hash_value;
-  InkHashTableEntry *entry;
-  InkHashTableIteratorState iterator_state;
-
-  for (entry = ink_hash_table_iterator_first(ext_map, &iterator_state); (entry != nullptr && !in_ext_map);
-       entry = ink_hash_table_iterator_next(ext_map, &iterator_state)) {
-    char *key = (char *)ink_hash_table_entry_key(ext_map, entry);
-    if (strstr(key, virt_ip)) {
-      if (!in_ext_map) {
-        in_ext_map = true;
-      }
-    }
-  }
-
-  if (in_ext_map) {
-    ret = true;
-  }
-
-  if (ret) {
-    return ret;
-  }
-
-  if (ink_hash_table_lookup(our_map, virt_ip, &hash_value) != 0) {
-    ret = false;
-  }
-
-  return ret;
-} /* End VMap::rl_checkGlobConflict */
-
-/*
- * remap(...)
- *   Function attempts to remap virt_ip. If dest is NULL, test to see if local
- * node should grab the addr.
- */
-bool
-VMap::rl_remap(char *virt_ip, char *cur_ip, char *dest_ip, int cur_naddr, int dest_naddr)
-{
-  char buf[1024];
-  InkHashTable *tmp;
-  InkHashTableValue hash_value;
-
-  if (inet_addr(cur_ip) != our_ip) {
-    snprintf(buf, sizeof(buf), "%s %s", cur_ip, dest_ip);
-    tmp = ext_map;
-  } else {
-    struct in_addr addr;
-    addr.s_addr = our_ip;
-
-    ink_strlcpy(buf, inet_ntoa(addr), sizeof(buf));
-    tmp = our_map;
-  }
-
-  /* Verify the map is correct and has not changed */
-  if (ink_hash_table_lookup(tmp, buf, &hash_value) != 0) {
-    mgmt_log("[VMap::rl_remap] Map has changed for ip: '%s' virt: '%s'\n", cur_ip, virt_ip);
-    return false;
-  } else if (cur_naddr <= dest_naddr) {
-    return false;
-  }
-
-  /* Down interface on cur_ip, wait for completion */
-  if (inet_addr(cur_ip) == our_ip) {
-    if (!rl_unmap(virt_ip)) {
-      return false;
-    }
-  } else {
-    if (!rl_remote_unmap(virt_ip, cur_ip)) {
-      return false;
-    }
-  }
-
-  /* Up interface on dest_ip, again wait for completion */
-  if (inet_addr(dest_ip) == our_ip) {
-    if (!rl_map(virt_ip)) {
-      return false;
-    }
-  } else {
-    if (!rl_remote_map(virt_ip, dest_ip)) {
-      return false;
-    }
-  }
-  return true;
-} /* End VMap::rl_remap */
-
-/*
- * boundAddr(...)
- *   Function tests whether or not the addr is bound. Returns 0(not bound),
- * 1(bound locally), 2(bound in cluster).
- */
-int
-VMap::rl_boundAddr(char *virt_ip)
-{
-  InkHashTableValue hash_value;
-  InkHashTableEntry *entry;
-  InkHashTableIteratorState iterator_state;
-
-  if (ink_hash_table_lookup(our_map, virt_ip, &hash_value) != 0) {
-    return 1;
-  }
-
-  for (entry = ink_hash_table_iterator_first(ext_map, &iterator_state); entry != nullptr;
-       entry = ink_hash_table_iterator_next(ext_map, &iterator_state)) {
-    char *key = (char *)ink_hash_table_entry_key(ext_map, entry);
-
-    if (strstr(key, virt_ip)) {
-      return 2;
-    }
-  }
-  return 0;
-} /* End VMap::rl_boundAddr */
-
-/*
- * boundTo(...)
- *   Function returns ip addr(string form) of the node that the virt address
- * is bound to. Returning NULL if it is unbound.
- */
-unsigned long
-VMap::rl_boundTo(char *virt_ip)
-{
-  InkHashTableValue hash_value;
-  InkHashTableEntry *entry;
-  InkHashTableIteratorState iterator_state;
-
-  if (ink_hash_table_lookup(our_map, virt_ip, &hash_value) != 0) {
-    return our_ip;
-  }
-
-  for (entry = ink_hash_table_iterator_first(ext_map, &iterator_state); entry != nullptr;
-       entry = ink_hash_table_iterator_next(ext_map, &iterator_state)) {
-    char *key = (char *)ink_hash_table_entry_key(ext_map, entry);
-
-    if (strstr(key, virt_ip)) {
-      char *buf, buf2[80];
-      if ((buf = strstr(key, " ")) != nullptr) {
-        buf++;
-        ink_strlcpy(buf2, buf, sizeof(buf2));
-      } else {
-        mgmt_fatal(0, "[VMap::rl_boundTo] Corrupt VMap entry('%s'), bailing\n", key);
-      }
-      return (inet_addr(buf2));
-    }
-  }
-  return 0;
-} /* End VMap::rl_boundTo */
-
-/*
- * constructVMapMessage(...)
- *   Constructs the broadcast message of the local nodes virtual ip map.
- */
-void
-VMap::lt_constructVMapMessage(char *ip, char *message, int max)
-{
-  int n = 0, bsum = 0;
-  InkHashTableEntry *entry;
-  InkHashTableIteratorState iterator_state;
-
-  if (!ip) {
-    return;
-  }
-  // Insert the standard mcast packet header
-  n = ClusterCom::constructSharedPacketHeader(appVersionInfo, message, ip, max);
-
-  if (!((n + (int)strlen("type: vmap\n")) < max)) {
-    if (max >= 1) {
-      message[0] = '\0';
-    }
-    return;
-  }
-
-  ink_strlcpy(&message[n], "type: vmap\n", max - n);
-  n += strlen("type: vmap\n");
-  bsum = n;
-
-  ink_mutex_acquire(mutex);
-  for (entry = ink_hash_table_iterator_first(our_map, &iterator_state); (entry != nullptr && n < max);
-       entry = ink_hash_table_iterator_next(our_map, &iterator_state)) {
-    char buf[1024];
-    char *key = (char *)ink_hash_table_entry_key(our_map, entry);
-
-    snprintf(buf, sizeof(buf), "virt: %s\n", key);
-    if (!((n + (int)strlen(buf)) < max)) {
-      break;
-    }
-    ink_strlcpy(&message[n], buf, max - n);
-    n += strlen(buf);
-  }
-  ink_mutex_release(mutex);
-
-  if (n == bsum) { /* No alarms */
-    if (!((n + (int)strlen("virt: none\n")) < max)) {
-      if (max >= 1) {
-        message[0] = '\0';
-      }
-      return;
-    }
-    ink_strlcpy(&message[n], "virt: none\n", max - n);
-  }
-  return;
-} /* End VMap::constructVMapMessage */
-
-void
-VMap::rl_downAddrs()
-{
-  for (int i = 0; i < num_addrs; i++) {
-    char str_addr[1024];
-    struct in_addr address;
-    address.s_addr = addr_list[i];
-    ink_strlcpy(str_addr, inet_ntoa(address), sizeof(str_addr));
-    rl_unmap(str_addr);
-  }
-  return;
-} /* End VMap::rl_downAddrs */
-
-void
-VMap::downAddrs()
-{
-  ink_mutex_acquire(mutex);
-  for (int i = 0; i < num_addrs; i++) {
-    removeAddressMapping(i);
-  }
-
-  // Reset On->Off flag
-  turning_off = false;
-
-  num_interfaces = 0;
-  ink_mutex_release(mutex);
-  return;
-} /* End VMap::downAddrs */
-
-void
-VMap::downOurAddrs()
-{
-  bool some_address_mapped = false;
-  InkHashTableEntry *entry;
-  InkHashTableIteratorState iterator_state;
-
-  ink_mutex_acquire(mutex);
-  for (entry = ink_hash_table_iterator_first(our_map, &iterator_state); entry != nullptr;
-       entry = ink_hash_table_iterator_next(our_map, &iterator_state)) {
-    some_address_mapped = true;
-  }
-
-  /*
-   * If any address was mapped, take no chance and down
-   * everything we know about
-   */
-  if (some_address_mapped) {
-    for (int i = 0; i < num_addrs; i++) {
-      removeAddressMapping(i);
-    }
-  }
-  num_interfaces = 0;
-  ink_mutex_release(mutex);
-} /* End VMap::downOurAddrs */
-
-/*
- * Remove a virtual address from our map
- */
-void
-VMap::removeAddressMapping(int i)
-{
-  char str_addr[1024];
-  struct in_addr address;
-  address.s_addr = addr_list[i];
-  ink_strlcpy(str_addr, inet_ntoa(address), sizeof(str_addr));
-  ink_hash_table_delete(our_map, str_addr); /* Make sure removed */
-}
diff --git a/mgmt/cluster/VMap.h b/mgmt/cluster/VMap.h
deleted file mode 100644
index 2a59d23..0000000
--- a/mgmt/cluster/VMap.h
+++ /dev/null
@@ -1,121 +0,0 @@
-/** @file
-
-  Class definition for virtual ip cluster mgmt. Maintains the virtual
-  map for the cluster and provides support for mapping operations.
-
-  @section license License
-
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
- */
-#ifndef _VMAP_H
-#define _VMAP_H
-
-#include "ts/ink_platform.h"
-#include "ts/I_Version.h"
-#include "ts/ink_string.h"
-
-#define MAX_INTERFACE 16
-#define MAX_SUB_ID 8
-
-typedef struct _vip_info {
-  char interface[MAX_INTERFACE];
-  char sub_interface_id[MAX_SUB_ID];
-} VIPInfo;
-
-typedef struct _realip_info {
-  struct in_addr real_ip;
-  bool mappings_for_interface;
-} RealIPInfo;
-
-/*
- * class VMap
- *   Class implements the protocol and support functions for mapping the
- * clusters virtual addresses. Member functions names are important here,
- * since this class shares a lock with the ClusterCom class:
- *
- *      lt_   "Lock Taken"   -- release the lock prior to invoking
- *      rl_   "Require Lock" -- acquire the lock prior to invoking
- *
- *   Care should also be taken when accessing any of the member variables,
- * a lock is generally required before modification should be made to them.
- *
- */
-class VMap
-{
-public:
-  VMap(char *interface, unsigned long ip, ink_mutex *m);
-  ~VMap();
-
-  void downAddrs();
-  void downOurAddrs();
-  void rl_downAddrs();
-  void removeAddressMapping(int i);
-  void lt_runGambit();
-  void lt_readAListFile(const char *data);
-  void lt_constructVMapMessage(char *ip, char *message, int max);
-
-  bool rl_remote_map(char *virt_ip, char *real_ip);
-  bool rl_remote_unmap(char *virt_ip, char *real_ip);
-
-  bool rl_map(char *virt_ip, char *real_ip = NULL);
-  bool rl_unmap(char *virt_ip, char *real_ip = NULL);
-  bool rl_remap(char *virt_ip, char *cur_ip, char *dest_ip, int cur_naddr, int dest_naddr);
-
-  char *rl_checkConflict(char *virt_ip);
-  bool rl_checkGlobConflict(char *virt_ip);
-
-  int rl_boundAddr(char *virt_ip);
-  unsigned long rl_boundTo(char *virt_ip);
-  int rl_clearUnSeen(char *ip);
-  void rl_resetSeenFlag(char *ip);
-
-  char vip_conf[PATH_NAME_MAX];
-  char absolute_vipconf_binary[PATH_NAME_MAX];
-
-  AppVersionInfo appVersionInfo;
-
-  int enabled;
-  bool turning_off; /* are we turning off VIP but haven't down'd the addr? */
-  /* map_init has never been used, remove it to avoid coverity complain */
-  int map_change_thresh;
-  time_t last_map_change;
-  time_t down_up_timeout;
-
-  char *addr_list_fname;
-
-  int num_addrs;
-  int num_nics;
-  unsigned long *addr_list;
-
-  int num_interfaces;
-  unsigned long our_ip;
-
-  ink_mutex *mutex;
-  // Map of virtual ip addresses assigned to the local node
-  InkHashTable *our_map;
-  // Map of virtual ip addresses assigned to other nodes; as indicated through multicast messages; used
-  // to detect conflicts
-  InkHashTable *ext_map;
-  InkHashTable *id_map;
-  InkHashTable *interface_realip_map;
-
-  char *interface; /* used to passed the interface from VMap::VMap to VMap::init */
-
-private:
-}; /* End class VMap */
-
-#endif /* _VMAP_H */
diff --git a/plugins/experimental/ts_lua/ts_lua_http_config.c b/plugins/experimental/ts_lua/ts_lua_http_config.c
index 5917f35..435fc75 100644
--- a/plugins/experimental/ts_lua/ts_lua_http_config.c
+++ b/plugins/experimental/ts_lua/ts_lua_http_config.c
@@ -43,7 +43,6 @@ typedef enum {
   TS_LUA_CONFIG_HTTP_SERVER_TCP_INIT_CWND                     = TS_CONFIG_HTTP_SERVER_TCP_INIT_CWND,
   TS_LUA_CONFIG_HTTP_SEND_HTTP11_REQUESTS                     = TS_CONFIG_HTTP_SEND_HTTP11_REQUESTS,
   TS_LUA_CONFIG_HTTP_CACHE_HTTP                               = TS_CONFIG_HTTP_CACHE_HTTP,
-  TS_LUA_CONFIG_HTTP_CACHE_CLUSTER_CACHE_LOCAL                = TS_CONFIG_HTTP_CACHE_CLUSTER_CACHE_LOCAL,
   TS_LUA_CONFIG_HTTP_CACHE_IGNORE_CLIENT_NO_CACHE             = TS_CONFIG_HTTP_CACHE_IGNORE_CLIENT_NO_CACHE,
   TS_LUA_CONFIG_HTTP_CACHE_IGNORE_CLIENT_CC_MAX_AGE           = TS_CONFIG_HTTP_CACHE_IGNORE_CLIENT_CC_MAX_AGE,
   TS_LUA_CONFIG_HTTP_CACHE_IMS_ON_CLIENT_NO_CACHE             = TS_CONFIG_HTTP_CACHE_IMS_ON_CLIENT_NO_CACHE,
@@ -164,7 +163,6 @@ ts_lua_var_item ts_lua_http_config_vars[] = {
   TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_SERVER_TCP_INIT_CWND),
   TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_SEND_HTTP11_REQUESTS),
   TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_CACHE_HTTP),
-  TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_CACHE_CLUSTER_CACHE_LOCAL),
   TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_CACHE_IGNORE_CLIENT_NO_CACHE),
   TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_CACHE_IGNORE_CLIENT_CC_MAX_AGE),
   TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_CACHE_IMS_ON_CLIENT_NO_CACHE),
diff --git a/proxy/EventName.cc b/proxy/EventName.cc
index 341b44b..3d95417 100644
--- a/proxy/EventName.cc
+++ b/proxy/EventName.cc
@@ -29,7 +29,6 @@
 // #include "I_Disk.h" unused
 #include "I_Cache.h"
 #include "I_Net.h"
-//#include "P_Cluster.h"
 #include "I_HostDB.h"
 #include "BaseManager.h"
 #include "P_RefCountCache.h"
@@ -75,18 +74,6 @@ event_int_to_string(int event, int blen, char *buffer)
   case NET_EVENT_ACCEPT_FAILED:
     return "NET_EVENT_ACCEPT_FAILED";
 
-#ifdef CLUSTER_CACHE
-  case CLUSTER_EVENT_CHANGE:
-    return "CLUSTER_EVENT_CHANGE";
-  case CLUSTER_EVENT_CONFIGURATION:
-    return "CLUSTER_EVENT_CONFIGURATION";
-  case CLUSTER_EVENT_OPEN:
-    return "CLUSTER_EVENT_OPEN";
-  case CLUSTER_EVENT_OPEN_FAILED:
-    return "CLUSTER_EVENT_OPEN_FAILED";
-  case CLUSTER_EVENT_STEAL_THREAD:
-    return "CLUSTER_EVENT_STEAL_THREAD";
-#endif
   case EVENT_HOST_DB_LOOKUP:
     return "EVENT_HOST_DB_LOOKUP";
   case EVENT_HOST_DB_GET_RESPONSE:
diff --git a/proxy/IPAllow.cc b/proxy/IPAllow.cc
index b5edc17..6f5c46e 100644
--- a/proxy/IPAllow.cc
+++ b/proxy/IPAllow.cc
@@ -35,6 +35,7 @@
 #include "P_EventSystem.h"
 #include "P_Cache.h"
 #include "hdrs/HdrToken.h"
+#include "ControlMatcher.h"
 
 #include <sstream>
 
diff --git a/proxy/InkAPI.cc b/proxy/InkAPI.cc
index 0d7dc9f..3a24f04 100644
--- a/proxy/InkAPI.cc
+++ b/proxy/InkAPI.cc
@@ -4404,9 +4404,6 @@ TSContSchedule(TSCont contp, ink_hrtime timeout, TSThreadPool tp)
   case TS_THREAD_POOL_REMAP:
     etype = ET_TASK; // Should be ET_REMAP
     break;
-  case TS_THREAD_POOL_CLUSTER:
-    etype = ET_CLUSTER;
-    break;
   case TS_THREAD_POOL_UDP:
     etype = ET_UDP;
     break;
@@ -7867,9 +7864,6 @@ _conf_to_memberp(TSOverridableConfigKey conf, OverridableHttpConfigParams *overr
   case TS_CONFIG_HTTP_CACHE_HTTP:
     ret = _memberp_to_generic(&overridableHttpConfig->cache_http, typep);
     break;
-  case TS_CONFIG_HTTP_CACHE_CLUSTER_CACHE_LOCAL:
-    ret = _memberp_to_generic(&overridableHttpConfig->cache_cluster_cache_local, typep);
-    break;
   case TS_CONFIG_HTTP_CACHE_IGNORE_CLIENT_NO_CACHE:
     ret = _memberp_to_generic(&overridableHttpConfig->cache_ignore_client_no_cache, typep);
     break;
@@ -8610,11 +8604,6 @@ TSHttpTxnConfigFind(const char *name, int length, TSOverridableConfigKey *conf,
         cnf = TS_CONFIG_HTTP_DEFAULT_BUFFER_WATER_MARK;
       }
       break;
-    case 'l':
-      if (!strncmp(name, "proxy.config.http.cache.cluster_cache_local", length)) {
-        cnf = TS_CONFIG_HTTP_CACHE_CLUSTER_CACHE_LOCAL;
-      }
-      break;
     case 'r':
       if (!strncmp(name, "proxy.config.http.cache.heuristic_lm_factor", length)) {
         typ = TS_RECORDDATATYPE_FLOAT;
diff --git a/proxy/InkAPITest.cc b/proxy/InkAPITest.cc
index 25770aa..0553b44 100644
--- a/proxy/InkAPITest.cc
+++ b/proxy/InkAPITest.cc
@@ -7542,7 +7542,6 @@ const char *SDK_Overridable_Configs[TS_CONFIG_LAST_ENTRY] = {
   "proxy.config.http.server_tcp_init_cwnd",
   "proxy.config.http.send_http11_requests",
   "proxy.config.http.cache.http",
-  "proxy.config.http.cache.cluster_cache_local",
   "proxy.config.http.cache.ignore_client_no_cache",
   "proxy.config.http.cache.ignore_client_cc_max_age",
   "proxy.config.http.cache.ims_on_client_no_cache",
diff --git a/proxy/Main.cc b/proxy/Main.cc
index e534f79..02d0e28 100644
--- a/proxy/Main.cc
+++ b/proxy/Main.cc
@@ -64,7 +64,6 @@ extern "C" int plock(int);
 #include "P_UDPNet.h"
 #include "P_DNS.h"
 #include "P_SplitDNS.h"
-#include "P_Cluster.h"
 #include "P_HostDB.h"
 #include "P_Cache.h"
 #include "ts/I_Layout.h"
@@ -78,7 +77,6 @@ extern "C" int plock(int);
 #include "HttpBodyFactory.h"
 #include "logging/Log.h"
 #include "ICPProcessor.h"
-//#include "ClusterTest.h"
 #include "CacheControl.h"
 #include "IPAllow.h"
 #include "ParentSelection.h"
@@ -133,8 +131,6 @@ static int num_of_udp_threads = 0;
 static int num_accept_threads = 0;
 static int num_task_threads   = 0;
 
-extern int num_of_cluster_threads;
-
 static char *http_accept_port_descriptor;
 int http_accept_file_descriptor = NO_FD;
 static char core_file[255]      = "";
@@ -151,11 +147,7 @@ static int regression_level       = REGRESSION_TEST_NONE;
 #endif
 int auto_clear_hostdb_flag = 0;
 extern int fds_limit;
-extern int cluster_port_number;
-extern int cache_clustering_enabled;
-char cluster_host[MAXDNAME + 1] = DEFAULT_CLUSTER_HOST;
 
-//         = DEFAULT_CLUSTER_PORT_NUMBER;
 static char command_string[512] = "";
 static char conf_dir[512]       = "";
 int remote_management_flag      = DEFAULT_REMOTE_MANAGEMENT_FLAG;
@@ -185,12 +177,10 @@ AppVersionInfo appVersionInfo; // Build info for this application
 
 static ArgumentDescription argument_descriptions[] = {
   {"net_threads", 'n', "Number of Net Threads", "I", &num_of_net_threads, "PROXY_NET_THREADS", nullptr},
-  {"cluster_threads", 'Z', "Number of Cluster Threads", "I", &num_of_cluster_threads, "PROXY_CLUSTER_THREADS", nullptr},
   {"udp_threads", 'U', "Number of UDP Threads", "I", &num_of_udp_threads, "PROXY_UDP_THREADS", nullptr},
   {"accept_thread", 'a', "Use an Accept Thread", "T", &num_accept_threads, "PROXY_ACCEPT_THREAD", nullptr},
   {"accept_till_done", 'b', "Accept Till Done", "T", &accept_till_done, "PROXY_ACCEPT_TILL_DONE", nullptr},
   {"httpport", 'p', "Port descriptor for HTTP Accept", "S*", &http_accept_port_descriptor, "PROXY_HTTP_ACCEPT_PORT", nullptr},
-  {"cluster_port", 'P', "Cluster Port Number", "I", &cluster_port_number, "PROXY_CLUSTER_PORT", nullptr},
   {"dprintf_level", 'o', "Debug output level", "I", &cmd_line_dprintf_level, "PROXY_DPRINTF_LEVEL", nullptr},
   {"disable_freelist", 'f', "Disable the freelist memory allocator", "T", &cmd_disable_freelist, "PROXY_DPRINTF_LEVEL", nullptr},
 
@@ -1212,13 +1202,6 @@ struct ShowStats : public Continuation {
     }
     fprintf(fp, "\nopen_delay_events=%d\n", open_delay_events);
 
-    fprintf(fp, "cluster_send_time_dist\n");
-    for (i = 0; i < TIME_DIST_BUCKETS_SIZE; i++) {
-      if ((i % 10) == 0)
-        fprintf(fp, "\n");
-      fprintf(fp, "%5d ", cluster_send_time_dist[i]);
-    }
-    fprintf(fp, "\ncluster_send_events=%d\n", cluster_send_events);
     fflush(fp);
 #endif
     return EVENT_CONT;
@@ -1740,15 +1723,6 @@ main(int /* argc ATS_UNUSED */, const char **argv)
   // Initialize the stat pages manager
   statPagesManager.init();
 
-  //////////////////////////////////////////////////////////////////////
-  // Determine if Cache Clustering is enabled, since the transaction
-  // on a thread changes require special consideration to allow
-  // minimal Cache Clustering functionality.
-  //////////////////////////////////////////////////////////////////////
-  cache_clustering_enabled = 0;
-
-  Note("cache clustering %s", cache_clustering_enabled ? "enabled" : "disabled");
-
   num_of_net_threads = adjust_num_of_net_threads(num_of_net_threads);
 
   size_t stacksize;
@@ -1834,7 +1808,6 @@ main(int /* argc ATS_UNUSED */, const char **argv)
     dnsProcessor.start(0, stacksize);
     if (hostDBProcessor.start() < 0)
       SignalWarning(MGMT_SIGNAL_SYSTEM_ERROR, "bad hostdb or storage configuration, hostdb disabled");
-    clusterProcessor.init();
 
     // initialize logging (after event and net processor)
     Log::init(remote_management_flag ? 0 : Log::NO_REMOTE_MANAGEMENT);
diff --git a/proxy/Makefile.am b/proxy/Makefile.am
index 5fec96a..a46e19d 100644
--- a/proxy/Makefile.am
+++ b/proxy/Makefile.am
@@ -195,11 +195,9 @@ traffic_server_LDADD = \
   shared/libdiagsconfig.a  \
   $(top_builddir)/mgmt/libmgmt_p.la \
   $(top_builddir)/iocore/utils/libinkutils.a \
-  $(top_builddir)/iocore/cluster/libinkcluster.a \
   $(top_builddir)/iocore/dns/libinkdns.a \
   $(top_builddir)/iocore/hostdb/libinkhostdb.a \
   $(top_builddir)/iocore/dns/libinkdns.a \
-  $(top_builddir)/iocore/cluster/libinkcluster.a \
   $(top_builddir)/iocore/cache/libinkcache.a \
   $(top_builddir)/iocore/aio/libinkaio.a \
   $(top_builddir)/lib/ts/libtsutil.la \

-- 
To stop receiving notification emails like this one, please contact
['"commits@trafficserver.apache.org" <commits@trafficserver.apache.org>'].

Mime
View raw message