trafficserver-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a..@apache.org
Subject trafficserver git commit: TS-3088: Use /etc/hosts for HostDB.
Date Tue, 16 Dec 2014 02:21:47 GMT
Repository: trafficserver
Updated Branches:
  refs/heads/master 04871a974 -> a2affb991


TS-3088: Use /etc/hosts for HostDB.


Project: http://git-wip-us.apache.org/repos/asf/trafficserver/repo
Commit: http://git-wip-us.apache.org/repos/asf/trafficserver/commit/a2affb99
Tree: http://git-wip-us.apache.org/repos/asf/trafficserver/tree/a2affb99
Diff: http://git-wip-us.apache.org/repos/asf/trafficserver/diff/a2affb99

Branch: refs/heads/master
Commit: a2affb9916edc53aeb8b7a5d6d223146be9efdc4
Parents: 04871a9
Author: Alan M. Carroll <amc@apache.org>
Authored: Sun Sep 21 17:18:19 2014 -0500
Committer: Alan M. Carroll <amc@apache.org>
Committed: Mon Dec 15 20:21:23 2014 -0600

----------------------------------------------------------------------
 CHANGES                                         |   2 +
 .../configuration/records.config.en.rst         |  17 +
 iocore/hostdb/HostDB.cc                         | 502 ++++++++++++++++---
 iocore/hostdb/I_HostDBProcessor.h               |  16 +-
 iocore/hostdb/P_HostDBProcessor.h               |  11 +-
 lib/ts/ink_inet.cc                              |   8 +-
 lib/ts/ink_inet.h                               |  11 +-
 mgmt/RecordsConfig.cc                           |   4 +
 proxy/http/HttpSM.cc                            |   1 +
 9 files changed, 482 insertions(+), 90 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/trafficserver/blob/a2affb99/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index cd2adb9..200c5ad 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,6 +1,8 @@
                                                          -*- coding: utf-8 -*-
 Changes with Apache Traffic Server 5.3.0
 
+  *) [TS-3088] Enable /etc/hosts resolution.
+
   *) [TS-3240] Add the `dstaddr` hash key to the balancer plugin.
 
   *) [TS-3239] Add a new `generator` plugin.

http://git-wip-us.apache.org/repos/asf/trafficserver/blob/a2affb99/doc/reference/configuration/records.config.en.rst
----------------------------------------------------------------------
diff --git a/doc/reference/configuration/records.config.en.rst b/doc/reference/configuration/records.config.en.rst
index 72bb3d3..2d9bd40 100644
--- a/doc/reference/configuration/records.config.en.rst
+++ b/doc/reference/configuration/records.config.en.rst
@@ -1611,6 +1611,23 @@ uses the same origin server for the same client, for as long as the
origin serve
 set to :arg:`N` the IP address is rotated if more than :arg:`N` seconds have past since the
first time the
 current address was used.
 
+.. ts:cv:: CONFIG proxy.config.hostdb.host_file.path STRING /etc/hosts
+
+   Set the file path for an external host file.
+
+If this is set (non-empty) then the file is presumed to be a hosts file in the standard `host
file format <http://tools.ietf.org/html/rfc1123#page-13>`_. It is read and the entries
there added to the HostDB. The file is periodically checked for a more recent modification
date in which case it is reloaded. The interval is set by the value :ts:cv:`proxy.config.hostdb.host_file.interval`.
+
+While not technically reloadable, the value is read every time the file is to be checked
so that if changed the new
+value will be used on the next check and the file will be treated as modified.
+
+.. ts:cv:: CONFIG proxy.config.hostdb.host_file.interval INT 86400
+   :metric: seconds
+   :reloadable:
+
+   Set the file changed check timer for :ts:cv:`proxy.config.hostdb.host_file.path`.
+
+The file is checked every this many seconds to see if it has changed. If so the HostDB is
updated with the new values in the file.
+
 .. ts:cv:: CONFIG proxy.config.hostdb.ip_resolve STRING NULL
 
    Set the host resolution style.

http://git-wip-us.apache.org/repos/asf/trafficserver/blob/a2affb99/iocore/hostdb/HostDB.cc
----------------------------------------------------------------------
diff --git a/iocore/hostdb/HostDB.cc b/iocore/hostdb/HostDB.cc
index 1e54bcd..5f066d2 100644
--- a/iocore/hostdb/HostDB.cc
+++ b/iocore/hostdb/HostDB.cc
@@ -27,6 +27,9 @@
 #include "I_Layout.h"
 #include "Show.h"
 
+#include <vector>
+#include <algorithm>
+
 // dxu: turn off all Diags.h 's function.
 //#define Debug
 //#define Warning
@@ -52,8 +55,12 @@ unsigned int hostdb_ip_stale_interval = HOST_DB_IP_STALE;
 unsigned int hostdb_ip_timeout_interval = HOST_DB_IP_TIMEOUT;
 unsigned int hostdb_ip_fail_timeout_interval = HOST_DB_IP_FAIL_TIMEOUT;
 unsigned int hostdb_serve_stale_but_revalidate = 0;
+unsigned int hostdb_hostfile_check_interval = 86400; // 1 day
+unsigned int hostdb_hostfile_update_timestamp = 0;
+unsigned int hostdb_hostfile_check_timestamp = 0;
 char hostdb_filename[PATH_NAME_MAX + 1] = DEFAULT_HOST_DB_FILENAME;
 int hostdb_size = DEFAULT_HOST_DB_SIZE;
+char hostdb_hostfile_path[PATH_NAME_MAX + 1] = "";
 int hostdb_sync_frequency = 120;
 int hostdb_srv_enabled = 0;
 int hostdb_disable_reverse_lookup = 0;
@@ -64,6 +71,8 @@ ClassAllocator<HostDBContinuation> hostDBContAllocator("hostDBContAllocator");
 
 HostDBCache hostDB;
 
+void ParseHostFile(char const* path);
+
 static  Queue <HostDBContinuation > remoteHostDBQueue[MULTI_CACHE_PARTITIONS];
 
 char *
@@ -161,31 +170,68 @@ string_for(HostDBMark mark) {
 static Action *
 register_ShowHostDB(Continuation * c, HTTPHdr * h);
 
+HostDBMD5&
+HostDBMD5::set_host(char const* name, int len)
+{
+  host_name = name;
+  host_len = len;
+#ifdef SPLIT_DNS
+  if (host_name && SplitDNSConfig::isSplitDNSEnabled()) {
+    const char *scan;
+    // I think this is checking for a hostname that is just an address.
+    for (scan = host_name ; *scan != '\0' && (ParseRules::is_digit(*scan) || '.'
== *scan || ':' == *scan); ++scan)
+      ;
+    if ('\0' != *scan) {
+      // config is released in the destructor, because we must make sure values we
+      // get out of it don't evaporate while @a this is still around.
+      if (!pSD) pSD = SplitDNSConfig::acquire();
+      if (pSD) {
+        dns_server = static_cast<DNSServer*>(pSD->getDNSRecord(host_name));
+      }
+    } else {
+      dns_server = 0;
+    }
+  }
+#endif // SPLIT_DNS
+  return *this;
+}
+
 void
 HostDBMD5::refresh() {
+  MD5Context ctx;
+
   if (host_name) {
     char const* server_line = dns_server ? dns_server->x_dns_ip_line : 0;
-    make_md5(hash, host_name, host_len, port, server_line, db_mark);
+    uint8_t m = static_cast<uint8_t>(db_mark); // be sure of the type.
+
+    ctx.update(host_name, host_len);
+    ctx.update(reinterpret_cast<uint8_t*>(&port), sizeof(port));
+    ctx.update(&m, sizeof(m));
+    if (server_line) ctx.update(server_line, strlen(server_line));
   } else {
     // INK_MD5 the ip, pad on both sizes with 0's
     // so that it does not intersect the string space
     //
-    uint8_t buff[TS_IP6_SIZE+4];
+    char buff[TS_IP6_SIZE+4];
     int n = ip.isIp6() ? sizeof(in6_addr) : sizeof(in_addr_t);
     memset(buff, 0, 2);
     memcpy(buff+2, ip._addr._byte, n);
     memset(buff + 2 + n , 0, 2);
-    MD5Context().hash_immediate(hash, buff, n+4);
-//    hash.encodeBuffer(buff, n+4);
+    ctx.update(buff, n+4);
   }
+  ctx.finalize(hash);
 }
 
 HostDBMD5::HostDBMD5()
   : host_name(0), host_len(0), port(0),
-    dns_server(0), db_mark(HOSTDB_MARK_GENERIC)
+    dns_server(0), pSD(0), db_mark(HOSTDB_MARK_GENERIC)
 {
 }
 
+HostDBMD5::~HostDBMD5() {
+  if (pSD) SplitDNSConfig::release(pSD);
+}
+
 HostDBCache::HostDBCache()
 {
   tag_bits = HOST_DB_TAG_BITS;
@@ -472,6 +518,7 @@ HostDBProcessor::start(int, size_t)
   REC_EstablishStaticConfigInt32U(hostdb_ip_fail_timeout_interval, "proxy.config.hostdb.fail.timeout");
   REC_EstablishStaticConfigInt32U(hostdb_serve_stale_but_revalidate, "proxy.config.hostdb.serve_stale_for");
   REC_EstablishStaticConfigInt32(hostdb_sync_frequency, "proxy.config.cache.hostdb.sync_frequency");
+  REC_EstablishStaticConfigInt32U(hostdb_hostfile_check_interval, "proxy.config.hostdb.host_file.interval");
 
   //
   // Set up hostdb_current_interval
@@ -530,22 +577,6 @@ HostDBContinuation::refresh_MD5() {
     mutex = hostDB.lock_for_bucket((int) (fold_md5(md5.hash) % hostDB.buckets));
 }
 
-void
-make_md5(INK_MD5 & md5, const char *hostname, int len, int port, char const* pDNSServers,
HostDBMark mark)
-{
-  INK_DIGEST_CTX ctx;
-  ink_code_incr_md5_init(&ctx);
-  ink_code_incr_md5_update(&ctx, hostname, len);
-  unsigned short p = port;
-  p = htons(p);
-  ink_code_incr_md5_update(&ctx, (char *) &p, 2);
-  uint8_t m = static_cast<uint8_t>(mark);
-  ink_code_incr_md5_update(&ctx, (char *) &m, sizeof(m));     /* FIXME: check this
*/
-  if (pDNSServers)
-    ink_code_incr_md5_update(&ctx, pDNSServers, strlen(pDNSServers));
-  ink_code_incr_md5_final((char *) &md5, &ctx);
-}
-
 static bool
 reply_to_cont(Continuation * cont, HostDBInfo * r, bool is_srv = false)
 {
@@ -616,6 +647,11 @@ db_mark_for(sockaddr const* ip) {
   return ats_is_ip6(ip) ? HOSTDB_MARK_IPV6 : HOSTDB_MARK_IPV4;
 }
 
+inline HostDBMark
+db_mark_for(IpAddr const& ip) {
+  return ip.isIp6() ? HOSTDB_MARK_IPV6 : HOSTDB_MARK_IPV4;
+}
+
 HostDBInfo *
 probe(ProxyMutex *mutex, HostDBMD5 const& md5, bool ignore_timeout)
 {
@@ -732,24 +768,10 @@ HostDBProcessor::getby(Continuation * cont,
   }
 
   // Load the MD5 data.
-  md5.host_name = hostname;
-  md5.host_len = hostname ? (len ? len : strlen(hostname)) : 0;
+  md5.set_host(hostname, hostname ? (len ? len : strlen(hostname)) : 0);
   md5.ip.assign(ip);
   md5.port = ip ? ats_ip_port_host_order(ip) : 0;
   md5.db_mark = db_mark_for(host_res_style);
-#ifdef SPLIT_DNS
-  if (hostname && SplitDNSConfig::isSplitDNSEnabled()) {
-    const char *scan = hostname;
-    // Is this a check for IPv4 address? Add check for IPv6?
-    for (; *scan != '\0' && (ParseRules::is_digit(*scan) || '.' == *scan); scan++);
-    if ('\0' != *scan) {
-      SplitDNS* pSD = SplitDNSConfig::acquire();
-      if (0 != pSD)
-        md5.dns_server = static_cast<DNSServer*>(pSD->getDNSRecord(hostname));
-      SplitDNSConfig::release(pSD);
-    }
-  }
-#endif // SPLIT_DNS
   md5.refresh();
 
   // Attempt to find the result in-line, for level 1 hits
@@ -782,7 +804,7 @@ HostDBProcessor::getby(Continuation * cont,
             reply_to_cont(cont, r);
             return ACTION_RESULT_DONE;
           }
-          md5.refresh();
+          md5.refresh(); // only on reloop, because we've changed the family.
         }
       }
     } while (loop);
@@ -938,22 +960,9 @@ HostDBProcessor::getbyname_imm(Continuation * cont, process_hostdb_info_pfn
proc
     return ACTION_RESULT_DONE;
   }
 
-  md5.host_name = hostname;
-  md5.host_len = hostname ? (len ? len : strlen(hostname)) : 0;
+  md5.set_host(hostname, hostname ? (len ? len : strlen(hostname)) : 0);
   md5.port = opt.port;
   md5.db_mark = db_mark_for(opt.host_res_style);
-#ifdef SPLIT_DNS
-  if (SplitDNSConfig::isSplitDNSEnabled()) {
-    const char *scan = hostname;
-    for (; *scan != '\0' && (ParseRules::is_digit(*scan) || '.' == *scan); scan++);
-    if ('\0' != *scan) {
-      SplitDNS* pSD = SplitDNSConfig::acquire();
-      if (0 != pSD)
-        md5.dns_server = static_cast<DNSServer*>(pSD->getDNSRecord(md5.host_name));
-      SplitDNSConfig::release(pSD);
-    }
-  }
-#endif // SPLIT_DNS
   md5.refresh();
 
   // Attempt to find the result in-line, for level 1 hits
@@ -1036,7 +1045,6 @@ do_setby(HostDBInfo * r, HostDBApplicationInfo * app, const char *hostname,
IpAd
   }
 }
 
-
 void
 HostDBProcessor::setby(const char *hostname, int len, sockaddr const* ip, HostDBApplicationInfo
* app)
 {
@@ -1044,8 +1052,7 @@ HostDBProcessor::setby(const char *hostname, int len, sockaddr const*
ip, HostDB
     return;
 
   HostDBMD5 md5;
-  md5.host_name = hostname;
-  md5.host_len = hostname ? (len ? len : strlen(hostname)) : 0;
+  md5.set_host(hostname, hostname ? (len ? len : strlen(hostname)) : 0);
   md5.ip.assign(ip);
   md5.port = ip ? ats_ip_port_host_order(ip) : 0;
   md5.db_mark = db_mark_for(ip);
@@ -1080,8 +1087,7 @@ HostDBProcessor::setby_srv(const char *hostname, int len, const char
*target, Ho
       return;
 
   HostDBMD5 md5;
-  md5.host_name = hostname;
-  md5.host_len = len ? len : strlen(hostname);
+  md5.set_host(hostname, len ? len : strlen(hostname));
   md5.port = 0;
   md5.db_mark = HOSTDB_MARK_SRV;
   md5.refresh();
@@ -1156,21 +1162,10 @@ Action *
 HostDBProcessor::failed_connect_on_ip_for_name(Continuation * cont, sockaddr const* ip, const
char *hostname, int len)
 {
   HostDBMD5 md5;
-  md5.host_name = hostname;
-  md5.host_len = hostname ? (len ? len : strlen(hostname)) : 0;
+  md5.set_host(hostname, hostname ? (len ? len : strlen(hostname)) : 0);
   md5.ip.assign(ip);
   md5.port = ip ? ats_ip_port_host_order(ip) : 0;
   md5.db_mark = db_mark_for(ip);
-#ifdef SPLIT_DNS
-  SplitDNS *pSD = 0;
-  if (hostname && SplitDNSConfig::isSplitDNSEnabled()) {
-    pSD = SplitDNSConfig::acquire();
-
-    if (0 != pSD)
-      md5.dns_server = static_cast<DNSServer*>(pSD->getDNSRecord(hostname));
-    SplitDNSConfig::release(pSD);
-  }
-#endif // SPLIT_DNS
   md5.refresh();
 
   ProxyMutex *mutex = hostDB.lock_for_bucket((int) (fold_md5(md5.hash) % hostDB.buckets));
@@ -1270,8 +1265,12 @@ HostDBContinuation::lookup_done(IpAddr const& ip, char const* aname,
bool around
       break;
     }
     HOSTDB_SUM_DYN_STAT(hostdb_ttl_stat, ttl_seconds);
-    if (!ttl_seconds)
-      ttl_seconds = 1;          // www.barnsandnobel.com is lame
+
+    // Not sure about this - it seems wrong but I can't be sure. If we got a fail
+    // in the DNS event, 0 is passed in which we then change to 1 here. Do we need this
+    // to be non-zero to avoid an infinite timeout?
+    if (0 == ttl_seconds) ttl_seconds = 1;
+
     i = insert(ttl_seconds);
     if (is_byname()) {
       ip_text_buffer b;
@@ -1560,7 +1559,6 @@ HostDBContinuation::dnsEvent(int event, HostEnt * e)
             if (is_addr_valid(af, e->ent.h_addr_list[ii])) {
               HostDBInfo& item = rr_data->info[i];
               ip_addr_set(item.ip(), af, e->ent.h_addr_list[ii]);
-//              rr_data->info[i].ip() = *(unsigned int *) e->ent.h_addr_list[ii];
               item.full = 1;
               item.round_robin = 0;
               item.reverse_dns = 0;
@@ -1600,7 +1598,7 @@ HostDBContinuation::dnsEvent(int event, HostEnt * e)
     if (action.continuation) {
       // Check for IP family failover
       if (failed && check_for_retry(md5.db_mark, host_res_style)) {
-        this->refresh_MD5();
+        this->refresh_MD5(); // family changed if we're doing a retry.
         SET_CONTINUATION_HANDLER(this, (HostDBContHandler) & HostDBContinuation::probeEvent);
         thread->schedule_in(this, MUTEX_RETRY_DELAY);
         return EVENT_CONT;
@@ -2128,7 +2126,32 @@ put_hostinfo_ClusterFunction(ClusterHandler *ch, void *data, int /*
len ATS_UNUS
 int
 HostDBContinuation::backgroundEvent(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */)
 {
-  hostdb_current_interval++;
+  ++hostdb_current_interval;
+
+  // hostdb_current_interval is bumped every HOST_DB_TIMEOUT_INTERVAL seconds
+  // so we need to scale that so the user config value is in seconds.
+  if (hostdb_hostfile_check_interval && // enabled
+      (hostdb_current_interval - hostdb_hostfile_check_timestamp) * (HOST_DB_TIMEOUT_INTERVAL
/ HRTIME_SECOND) > hostdb_hostfile_check_interval
+  ) {
+    struct stat info;
+    char path[sizeof(hostdb_hostfile_path)];
+
+    REC_ReadConfigString(path, "proxy.config.hostdb.host_file.path", PATH_NAME_MAX);
+    if (0 != strcasecmp(hostdb_hostfile_path, path)) {
+      Debug("amc", "Update host file '%s' <- '%s'", path, hostdb_hostfile_path);
+      // path to hostfile changed
+      hostdb_hostfile_update_timestamp = 0; // never updated from this file
+      memcpy(hostdb_hostfile_path, path, sizeof(hostdb_hostfile_path));
+    }
+    hostdb_hostfile_check_timestamp = hostdb_current_interval;
+    if (0 == stat(hostdb_hostfile_path, &info)) {
+      if (info.st_mtime > hostdb_hostfile_update_timestamp) {
+        ParseHostFile(hostdb_hostfile_path);
+      }
+    } else {
+      Debug("hostdb", "Failed to stat host file '%s'", hostdb_hostfile_path);
+    }
+  }
 
   return EVENT_CONT;
 }
@@ -2481,3 +2504,336 @@ ink_hostdb_init(ModuleVersion v)
 
   ts_host_res_global_init();
 }
+
+
+/// Pair of IP address and host name from a host file.
+struct HostFilePair {
+  typedef HostFilePair self;
+  IpAddr ip;
+  char const* name;
+};
+
+struct HostDBFileContinuation : public Continuation {
+  typedef HostDBFileContinuation self;
+
+  int idx; ///< Working index.
+  char const* name; ///< Host name (just for debugging)
+  INK_MD5 md5; ///< Key for entry.
+  typedef std::vector<INK_MD5> Keys;
+  Keys* keys; ///< Entries from file.
+  ats_scoped_str path; ///< Used to keep the host file name around.
+
+  HostDBFileContinuation()
+    : Continuation(0)
+    {
+    }
+
+  int insertEvent(int event, void* data);
+  int removeEvent(int event, void* data);
+
+  /// Set the current entry in the HostDB.
+  HostDBInfo* setDBEntry();
+  /// Create and schedule an update continuation.
+  static void scheduleUpdate(
+    int idx, ///< Pair index to process.
+    Keys* keys = 0 ///< Key table if any.
+    );
+  /// Create and schedule a remove continuation.
+  static void scheduleRemove(
+    int idx, ///< Index of current new key to check against.
+    Keys* keys ///< new valid keys
+    );
+  /// Finish update
+  static void finish(
+    Keys* keys ///< Valid keys from update.
+    );
+  /// Clean up this instance.
+  void destroy();
+};
+
+ClassAllocator<HostDBFileContinuation> hostDBFileContAllocator("hostDBFileContAllocator");
+
+void HostDBFileContinuation::destroy() {
+  this->~HostDBFileContinuation();
+  hostDBFileContAllocator.free(this);
+}
+
+// Host file processing globals.
+
+// We can't allow more than one update to be
+// proceeding at a time in any case so we might as well make these
+// globals.
+int HostDBFileUpdateActive = 0;
+// Contents of the host file. We keep this around because other data
+// points in to it (to minimize allocations).
+ats_scoped_str HostFileText;
+// Accumulated pairs of <IP address, host>
+std::vector<HostFilePair> HostFilePairs;
+// Entries from last update.
+HostDBFileContinuation::Keys HostFileKeys;
+/// Ordering operator for HostFilePair.
+/// We want to group first by name and then by address family.
+bool CmpHostFilePair(HostFilePair const& lhs, HostFilePair const& rhs) {
+  int zret = strcasecmp(lhs.name, rhs.name);
+  return zret < 0 || (0 == zret && lhs.ip < rhs.ip);
+}
+// Actual ordering doesn't matter as long as it's consistent.
+bool CmpMD5(INK_MD5 const& lhs, INK_MD5 const& rhs) {
+  return lhs[0] < rhs[0] || 
+    (lhs[0] == rhs[0] && lhs[1] < rhs[1])
+    ;
+}
+
+/// Finish current update.
+void
+HostDBFileContinuation::finish(Keys* keys) {
+  HostFilePairs.clear();
+  HostFileKeys.clear();
+  if (keys) {
+    std::swap(*keys, HostFileKeys); // put the new keys in place and dump the previous.
+    delete keys;
+  }
+
+  HostFileText = 0;
+  HostDBFileUpdateActive = 0;
+}
+
+HostDBInfo*
+HostDBFileContinuation::setDBEntry() {
+  HostDBInfo* r;
+  uint64_t folded_md5 = fold_md5(md5);
+
+  // remove the old one to prevent buildup
+  if (0 != (r = hostDB.lookup_block(folded_md5, 3))) {
+    hostDB.delete_block(r);
+    Debug("hostdb", "Update host file entry %s - %" PRIx64 ".%" PRIx64,
+          name, md5[0], md5[1] );
+  } else {
+    Debug("hostdb", "Add host file entry %s - %" PRIx64 ".%" PRIx64,
+          name, md5[0],md5[1] );
+  }
+
+  r = hostDB.insert_block(folded_md5, NULL, 0);
+  r->md5_high = md5[1];
+  r->ip_timeout_interval = 0; // special value - no timeout.
+  r->ip_timestamp = hostdb_current_interval;
+  keys->push_back(md5);
+  return r;
+}
+
+int
+HostDBFileContinuation::insertEvent(int, void*) {
+  int n = HostFilePairs.size();
+  HostFilePair const& first = HostFilePairs[idx];
+  int last = idx + 1;
+  HostDBInfo* r = 0;
+
+  ink_assert(idx < n);
+
+  // Get the set of addresses for this name.
+  while (last < n && 0 == strcasecmp(first.name, HostFilePairs[last].name) &&
first.ip.family() == HostFilePairs[last].ip.family())
+    ++last;
+  // Address set is now (idx, last].
+
+  int k = last - idx; // # of addrs for this name
+  ink_assert(k > 0);
+
+  r = this->setDBEntry();
+  r->reverse_dns = false;
+  r->is_srv = false;
+  if (k > 1) {
+    // multiple entries, need round robin
+    int s = HostDBRoundRobin::size(k, false);
+    HostDBRoundRobin* rr_data = static_cast<HostDBRoundRobin*>(hostDB.alloc(&r->app.rr.offset,
s));
+    if (rr_data) {
+      int dst = 0; // index of destination RR item.
+      for (int src = idx ; src < last ; ++src, ++dst ) {
+        HostDBInfo& item = rr_data->info[dst];
+        item.data.ip.assign(HostFilePairs[src].ip);
+        item.full = 1;
+        item.round_robin = false;
+        item.reverse_dns = false;
+        item.is_srv = false;
+        item.md5_high = r->md5_high;
+        item.md5_low = r->md5_low;
+        item.md5_low_low = r->md5_low_low;
+      }
+      r->round_robin = true;
+      rr_data->good = k;
+      rr_data->current = 0;
+    }
+  } else {
+    r->data.ip.assign(HostFilePairs[idx].ip);
+    r->round_robin = false;
+  }
+
+  if (last < n) { // more entries to process
+    // We create a new continuation rather than using this one to
+    // avoid mutex issues - it is critical that the continuation use
+    // the bucket mutex to guarantee a data lock for update. It's
+    // unclear if changing the mutex to this continuation will do that
+    // properly. Because we use a class allocator we should achieve
+    // efficient allocations quickly even for large host files.
+    this->scheduleUpdate(last, keys);
+  } else {
+    std::sort(keys->begin(), keys->end(), &CmpMD5);
+//    keys->qsort(&CmpMD5);
+    // Switch to removing dead entries.
+    this->scheduleRemove(keys->size() - 1, keys);
+  }
+  // This continuation is done.
+  this->destroy();
+  return EVENT_DONE;
+}
+
+int
+HostDBFileContinuation::removeEvent(int, void*) {
+  HostDBInfo* r;
+  uint64_t folded_md5 = fold_md5(md5);
+  Debug("hostdb", "Remove host file entry %" PRIx64 ".%" PRIx64, md5[0],md5[1] );
+  if (0 != (r = hostDB.lookup_block(folded_md5, 3)))
+    hostDB.delete_block(r);
+  this->scheduleRemove(idx, keys);
+  this->destroy();
+  return EVENT_DONE;
+}
+
+void
+HostDBFileContinuation::scheduleUpdate(int idx, Keys* keys) {
+  HostDBFileContinuation* c = hostDBFileContAllocator.alloc();
+  HostFilePair& pair = HostFilePairs[idx];
+  HostDBMD5 md5;
+
+  md5.set_host(pair.name, strlen(pair.name));
+  md5.db_mark = db_mark_for(pair.ip);
+  md5.refresh();
+
+  SET_CONTINUATION_HANDLER(c, &HostDBFileContinuation::insertEvent);
+  c->idx = idx;
+  c->name = pair.name;
+  c->keys = keys ? keys : new Keys;
+  c->md5 = md5.hash;
+  c->mutex = hostDB.lock_for_bucket(fold_md5(md5.hash) % hostDB.buckets);
+  eventProcessor.schedule_imm(c, ET_CALL, EVENT_IMMEDIATE, 0);
+
+}
+
+void
+HostDBFileContinuation::scheduleRemove(int idx, Keys* keys) {
+  bool targetp = false; // Have a valid target?
+  INK_MD5 md5;
+
+  // See if we have a target.
+  if (HostFileKeys.size() > 0) {
+    md5 = HostFileKeys.back();
+    HostFileKeys.pop_back();
+    targetp = true;
+    // Move backwards through the valid items and the previous
+    // keys, removing if we have a prev key that's bigger than
+    // the current valid key (which means it's not currently valid).
+    while (idx >= 0) {
+      if (CmpMD5((*keys)[idx], md5)) { // prev is not valid, remove
+        break;
+      } else if (CmpMD5(md5, (*keys)[idx])) {
+        --idx; // prev is smaller, skip current and keep checking
+      } else if (!HostFileKeys.empty()) {
+        md5 = HostFileKeys.back();
+        HostFileKeys.pop_back(); // match, move to next potential target
+        --idx;
+      } else { // ran out of things to remove.
+        targetp = false;
+        break;
+      }
+    }
+  }
+
+  if (targetp) {
+    HostDBFileContinuation* c = hostDBFileContAllocator.alloc();
+    SET_CONTINUATION_HANDLER(c, &HostDBFileContinuation::removeEvent);
+    c->md5 = md5;
+    c->idx = idx;
+    c->keys = keys;
+    c->mutex = hostDB.lock_for_bucket(fold_md5(c->md5) % hostDB.buckets);
+    eventProcessor.schedule_imm(c, ET_CALL, EVENT_IMMEDIATE, 0);
+  } else {
+    self::finish(keys);
+  }
+}
+
+void
+ParseHostLine(char* l) {
+  Tokenizer elts(" \t");
+  int n_elts = elts.Initialize(l, SHARE_TOKS);
+  // Elements should be the address then a list of host names.
+  // Don't use RecHttpLoadIp because the address *must* be literal.
+  HostFilePair item;
+  if (n_elts > 1 && 0 == item.ip.load(elts[0]) && !item.ip.isLoopback())
{
+    for ( int i = 1 ; i < n_elts ; ++i ) {
+      item.name = elts[i];
+      HostFilePairs.push_back(item);
+    }
+  }
+}
+
+
+void
+ParseHostFile(char const* path) {
+  bool success = false;
+  // Test and set for update in progress.
+  if (0 != ink_atomic_swap(&HostDBFileUpdateActive,1)) {
+    Debug("hostdb", "Skipped load of host file because update already in progress");
+    return;
+  }
+  Debug("hostdb", "Loading host file '%s'", path);
+
+  ats_scoped_fd fd(open(path, O_RDONLY));
+  if (fd >= 0) {
+    struct stat info;
+    if (0 == fstat(fd, &info)) {
+      // +1 in case no terminating newline
+      int64_t size = info.st_size+1;
+      HostFileText = static_cast<char*>(ats_malloc(size));
+      if (HostFileText) {
+        char* base = HostFileText;
+        char* limit;
+
+        size = read(fd, HostFileText, info.st_size);
+        limit = HostFileText + size;
+        *limit = 0;
+
+        // We need to get a list of all name/addr pairs so that we can
+        // group names for round robin records. Also note that the
+        // pairs have pointer back in to the text storage for the file
+        // so we need to keep that until we're done with @a pairs.
+        while (base < limit) {
+          char *spot = strchr(base, '\n');
+
+          // terminate the line.
+          if (0 == spot) spot = limit; // no trailing EOL, grab remaining
+          else *spot = 0;
+
+          while (base < spot && isspace(*base)) ++base; // skip leading ws
+          if (*base != '#' && base < spot) // non-empty non-comment line
+            ParseHostLine(base);
+          base = spot + 1;
+        }
+
+        hostdb_hostfile_update_timestamp = hostdb_current_interval;
+        success = true;
+        if (!HostFilePairs.empty()) {
+          // Need to sort by name so multiple address hosts are
+          // contiguous.
+          std::sort(HostFilePairs.begin(), HostFilePairs.end(), &CmpHostFilePair);
+          HostDBFileContinuation::scheduleUpdate(0);
+        } else if (!HostFileKeys.empty()) {
+          HostDBFileContinuation::scheduleRemove(-1, 0);
+        } else {
+          // Nothing in new data, nothing in old data, just clean up.
+          HostDBFileContinuation::finish(0);
+        }
+      }
+    }
+  }
+  if (!success)
+    HostDBFileUpdateActive = 0;
+}

http://git-wip-us.apache.org/repos/asf/trafficserver/blob/a2affb99/iocore/hostdb/I_HostDBProcessor.h
----------------------------------------------------------------------
diff --git a/iocore/hostdb/I_HostDBProcessor.h b/iocore/hostdb/I_HostDBProcessor.h
index be48457..c26a403 100644
--- a/iocore/hostdb/I_HostDBProcessor.h
+++ b/iocore/hostdb/I_HostDBProcessor.h
@@ -176,18 +176,17 @@ struct HostDBInfo
 
   int ip_time_remaining()
   {
-    return (int) ip_timeout_interval - (int) ((hostdb_current_interval - ip_timestamp) &
0x7FFFFFFF);
+    return static_cast<int>(ip_timeout_interval) - static_cast<int>(this->ip_interval());
   }
 
   bool is_ip_stale() {
-    if (ip_timeout_interval >= 2 * hostdb_ip_stale_interval)
-      return ip_interval() >= hostdb_ip_stale_interval;
-    else
-      return false;
+    return ip_timeout_interval >= 2 * hostdb_ip_stale_interval &&
+      ip_interval() >= hostdb_ip_stale_interval
+      ;
   }
 
   bool is_ip_timeout() {
-    return ip_interval() >= ip_timeout_interval;
+    return ip_timeout_interval && ip_interval() >= ip_timeout_interval;
   }
 
   bool is_ip_fail_timeout() {
@@ -228,8 +227,9 @@ struct HostDBInfo
   } data;
 
   unsigned int ip_timestamp;
-  // limited to 0x1FFFFF (24 days)
-  unsigned int ip_timeout_interval:31;
+  // limited to HOST_DB_MAX_TTL (0x1FFFFF, 24 days)
+  // if this is 0 then no timeout.
+  unsigned int ip_timeout_interval;
 
   unsigned int full:1;
   unsigned int backed:1;        // duplicated in lower level

http://git-wip-us.apache.org/repos/asf/trafficserver/blob/a2affb99/iocore/hostdb/P_HostDBProcessor.h
----------------------------------------------------------------------
diff --git a/iocore/hostdb/P_HostDBProcessor.h b/iocore/hostdb/P_HostDBProcessor.h
index f3ee41e..e67b0f8 100644
--- a/iocore/hostdb/P_HostDBProcessor.h
+++ b/iocore/hostdb/P_HostDBProcessor.h
@@ -416,17 +416,24 @@ struct HostDBMD5 {
   char const* host_name; ///< Host name.
   int host_len; ///< Length of @a _host_name
   IpAddr ip; ///< IP address.
-  int port; ///< IP port (host order).
+  in_port_t port; ///< IP port (host order).
   /// DNS server. Not strictly part of the MD5 data but
   /// it's both used by @c HostDBContinuation and provides access to
   /// MD5 data. It's just handier to store it here for both uses.
   DNSServer* dns_server;
+  SplitDNS* pSD; ///< Hold the container for @a dns_server.
   HostDBMark db_mark; ///< Mark / type of record.
 
   /// Default constructor.
   HostDBMD5();
+  /// Destructor.
+  ~HostDBMD5();
   /// Recompute and update the MD5 hash.
   void refresh();
+  /** Assign a hostname.
+      This updates the split DNS data as well.
+  */
+  self& set_host(char const* name, int len);
 };
 
 //
@@ -458,7 +465,7 @@ struct HostDBContinuation: public Continuation
   //  int namelen;
   char md5_host_name_store[MAXDNAME+1]; // used as backing store for @a md5
   char srv_target_name[MAXDNAME];
-  void *m_pDS;
+  //  void *m_pDS;
   Action *pending_action;
 
   unsigned int missing:1;

http://git-wip-us.apache.org/repos/asf/trafficserver/blob/a2affb99/lib/ts/ink_inet.cc
----------------------------------------------------------------------
diff --git a/lib/ts/ink_inet.cc b/lib/ts/ink_inet.cc
index 0104981..d84fea1 100644
--- a/lib/ts/ink_inet.cc
+++ b/lib/ts/ink_inet.cc
@@ -400,11 +400,8 @@ operator == (IpAddr const& lhs, sockaddr const* rhs) {
       - -1 if @a lhs is less than @a rhs.
       - 0 if @a lhs is identical to @a rhs.
       - 1 if @a lhs is greater than @a rhs.
-
-    @internal This looks like a lot of code for an inline but I think it
-    should compile down to something reasonable.
 */
-inline int
+int
 IpAddr::cmp(self const& that) const {
   int zret = 0;
   uint16_t rtype = that._family;
@@ -433,8 +430,7 @@ IpAddr::cmp(self const& that) const {
   } else if (AF_INET == rtype || AF_INET6 == rtype) {
     // ltype is non-IP so it's less than either IP type.
     zret = -1;
-  } else {
-    // Both types are non-IP so they're equal.
+  } else { // Both types are non-IP so they're equal.
     zret = 0;
   }
 

http://git-wip-us.apache.org/repos/asf/trafficserver/blob/a2affb99/lib/ts/ink_inet.h
----------------------------------------------------------------------
diff --git a/lib/ts/ink_inet.h b/lib/ts/ink_inet.h
index 388ce3c..5e4c5d4 100644
--- a/lib/ts/ink_inet.h
+++ b/lib/ts/ink_inet.h
@@ -1176,6 +1176,8 @@ struct IpAddr {
   self& invalidate() { _family = AF_UNSPEC; return *this; }
   /// Test for multicast
   bool isMulticast() const;
+  /// Test for loopback
+  bool isLoopback() const;
 
   uint16_t _family; ///< Protocol family.
   /// Address data.
@@ -1214,7 +1216,14 @@ IpAddr::isCompatibleWith(self const& that) {
 
 inline bool IpAddr::isIp4() const { return AF_INET == _family; }
 inline bool IpAddr::isIp6() const { return AF_INET6 == _family; }
-  /// Assign sockaddr storage.
+
+inline bool IpAddr::isLoopback() const {
+  return (AF_INET == _family && 0x7F == _addr._byte[0]) ||
+    (AF_INET6 == _family && IN6_IS_ADDR_LOOPBACK(&_addr._ip6))
+    ;
+}
+
+/// Assign sockaddr storage.
 inline IpAddr&
 IpAddr::assign(sockaddr const* addr) {
   if (addr) {

http://git-wip-us.apache.org/repos/asf/trafficserver/blob/a2affb99/mgmt/RecordsConfig.cc
----------------------------------------------------------------------
diff --git a/mgmt/RecordsConfig.cc b/mgmt/RecordsConfig.cc
index 80b7647..0c71255 100644
--- a/mgmt/RecordsConfig.cc
+++ b/mgmt/RecordsConfig.cc
@@ -1037,6 +1037,10 @@ static const RecordElement RecordsConfig[] =
   //       # how often should the hostdb be synced (seconds)
   {RECT_CONFIG, "proxy.config.cache.hostdb.sync_frequency", RECD_INT, "120", RECU_DYNAMIC,
RR_NULL, RECC_NULL, NULL, RECA_NULL}
   ,
+  {RECT_CONFIG, "proxy.config.hostdb.host_file.path", RECD_STRING, "/etc/hosts", RECU_DYNAMIC,
RR_NULL, RECC_STR, NULL, RECA_NULL}
+  ,
+  {RECT_CONFIG, "proxy.config.hostdb.host_file.interval", RECD_INT, "86400", RECU_DYNAMIC,
RR_NULL, RECC_NULL, NULL, RECA_NULL}
+  ,
 
   //##########################################################################
   //#

http://git-wip-us.apache.org/repos/asf/trafficserver/blob/a2affb99/proxy/http/HttpSM.cc
----------------------------------------------------------------------
diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc
index 7c13ebb..e2e6080 100644
--- a/proxy/http/HttpSM.cc
+++ b/proxy/http/HttpSM.cc
@@ -4016,6 +4016,7 @@ HttpSM::do_hostdb_lookup()
     }
 
     HostDBProcessor::Options opt;
+
     opt.flags = (t_state.cache_info.directives.does_client_permit_dns_storing)
       ? HostDBProcessor::HOSTDB_DO_NOT_FORCE_DNS
       : HostDBProcessor::HOSTDB_FORCE_DNS_RELOAD


Mime
View raw message