trafficserver-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bc...@apache.org
Subject [trafficserver] branch master updated: brotli support
Date Tue, 04 Apr 2017 17:20:37 GMT
This is an automated email from the ASF dual-hosted git repository.

bcall pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/trafficserver.git

The following commit(s) were added to refs/heads/master by this push:
       new  39d045e   brotli support
39d045e is described below

commit 39d045ecd4037ac2b8d62944c9a08d20667a23c5
Author: myraid <sandeep.davu@gmail.com>
AuthorDate: Tue Mar 7 15:52:20 2017 -0800

    brotli support
    
    remove enfore-brotli from readme
---
 configure.ac                        |   6 +
 doc/admin-guide/plugins/gzip.en.rst |  24 +++
 plugins/gzip/Makefile.inc           |   4 +
 plugins/gzip/README                 |   6 +
 plugins/gzip/configuration.cc       |  38 +++-
 plugins/gzip/configuration.h        |  18 +-
 plugins/gzip/gzip.cc                | 412 ++++++++++++++++++++++++++----------
 plugins/gzip/misc.cc                |  12 +-
 plugins/gzip/misc.h                 |  30 ++-
 9 files changed, 430 insertions(+), 120 deletions(-)

diff --git a/configure.ac b/configure.ac
index 116ff4c..05a1264 100644
--- a/configure.ac
+++ b/configure.ac
@@ -449,6 +449,12 @@ AC_ARG_WITH([max-threads-per-type],
 )
 AC_SUBST(max_threads_per_type)
 
+# Check Brotli
+AC_CHECK_HEADERS([brotli/encode.h], [has_brotli=1],[has_brotli=0])
+AC_CHECK_LIB([brotlienc],[BrotliEncoderCreateInstance],[AC_SUBST([LIB_BROTLIENC],["-lbrotlienc"])],[has_brotli=0])
+AC_SUBST(has_brotli)
+AM_CONDITIONAL([HAS_BROTLI], [ test "x${has_brotli}" = "x1" ])
+
 #
 # Experimental plugins
 #
diff --git a/doc/admin-guide/plugins/gzip.en.rst b/doc/admin-guide/plugins/gzip.en.rst
index 5ba8363..cbf52be 100644
--- a/doc/admin-guide/plugins/gzip.en.rst
+++ b/doc/admin-guide/plugins/gzip.en.rst
@@ -141,6 +141,14 @@ will leave the header intact if the client provided it.
 - For when the proxy parses responses, and the resulting compression and
   decompression is wasteful.
 
+supported-algorithms
+----------------------
+
+Provides the compression algorithms that are supported. This will allow the proxy to selectively
+support certain compressions. The default is gzip. Multiple algorthims can be selected using
',' delimiter
+
+-- To selectively support only certain compression algorithms.
+
 Examples
 ========
 
@@ -162,6 +170,22 @@ might create a configuration with the following options::
     disallow /notthis/*.js
     flush true
 
+    # Allows brotli encoded response from origin but is not capable of brotli compression
+    [brotli.allowed.com]
+    enabled true
+    compressible-content-type text/*
+    compressible-content-type application/json
+    flush true
+    supported-algorithms gzip,deflate
+
+    # Supports brotli compression
+    [brotli.compress.com]
+    enabled true
+    compressible-content-type text/*
+    compressible-content-type application/json
+    flush true
+    supported-algorithms br, gzip
+
     # This origin does it all
     [bar.example.com]
     enabled false
diff --git a/plugins/gzip/Makefile.inc b/plugins/gzip/Makefile.inc
index 3cd845c..8762e21 100644
--- a/plugins/gzip/Makefile.inc
+++ b/plugins/gzip/Makefile.inc
@@ -16,3 +16,7 @@
 
 pkglib_LTLIBRARIES += gzip/gzip.la
 gzip_gzip_la_SOURCES = gzip/gzip.cc gzip/configuration.cc gzip/misc.cc
+
+gzip_gzip_la_LDFLAGS = \
+  $(AM_LDFLAGS) $(LIB_BROTLIENC)
+
diff --git a/plugins/gzip/README b/plugins/gzip/README
index 601c0a6..1d42b00 100644
--- a/plugins/gzip/README
+++ b/plugins/gzip/README
@@ -44,6 +44,9 @@ a sample configuration (sample.gzip.config):
 # compressible-content-type: wildcard pattern for matching compressible content types
 #
 # disallow: wildcard pattern for disablign compression on urls
+#
+# supported-algorithms: compression algorithms supported. comma separated algorithms. Default
is gzip
+#
 ######################################################################
 
 #first, we configure the default/global plugin behaviour
@@ -60,6 +63,9 @@ disallow /notthis/*.js
 disallow /notthat*
 disallow */bla*
 
+#supported algorithms
+supported-algorithms br,gzip
+
 #override the global configuration for a host.
 #www.foo.nl does NOT inherit anything
 [www.foo.nl]
diff --git a/plugins/gzip/configuration.cc b/plugins/gzip/configuration.cc
index 80d8cf4..e0101a0 100644
--- a/plugins/gzip/configuration.cc
+++ b/plugins/gzip/configuration.cc
@@ -25,6 +25,7 @@
 #include <fstream>
 #include <algorithm>
 #include <vector>
+#include <sstream>
 #include <fnmatch.h>
 
 namespace Gzip
@@ -93,7 +94,8 @@ enum ParserState {
   kParseEnable,
   kParseCache,
   kParseDisallow,
-  kParseFlush
+  kParseFlush,
+  kParseAlgorithms
 };
 
 void
@@ -181,6 +183,34 @@ HostConfiguration::is_content_type_compressible(const char *content_type,
int co
   return is_match;
 }
 
+void
+HostConfiguration::add_compression_algorithms(const string &algorithms)
+{
+  istringstream compress_algo(algorithms);
+  string token;
+  compression_algorithms_ = ALGORITHM_DEFAULT; // remove the default gzip.
+  while (getline(compress_algo, token, ',')) {
+    if (token.find("br") != string::npos) {
+#ifdef HAVE_BROTLI_ENCODE_H
+      compression_algorithms_ |= ALGORITHM_BROTLI;
+#else
+      error("supported-algorithms: brotli support not compiled in.");
+#endif
+    } else if (token.find("gzip") != string::npos)
+      compression_algorithms_ |= ALGORITHM_GZIP;
+    else if (token.find("deflate") != string::npos)
+      compression_algorithms_ |= ALGORITHM_DEFLATE;
+    else
+      error("Unknown compression type. Supported compression-algorithms <br,gzip,deflate>.");
+  }
+}
+
+int
+HostConfiguration::compression_algorithms()
+{
+  return compression_algorithms_;
+}
+
 Configuration *
 Configuration::Parse(const char *path)
 {
@@ -263,6 +293,8 @@ Configuration::Parse(const char *path)
           state = kParseDisallow;
         } else if (token == "flush") {
           state = kParseFlush;
+        } else if (token == "supported-algorithms") {
+          state = kParseAlgorithms;
         } else {
           warning("failed to interpret \"%s\" at line %zu", token.c_str(), lineno);
         }
@@ -291,6 +323,10 @@ Configuration::Parse(const char *path)
         current_host_configuration->set_flush(token == "true");
         state = kParseStart;
         break;
+      case kParseAlgorithms:
+        current_host_configuration->add_compression_algorithms(token);
+        state = kParseStart;
+        break;
       }
     }
   }
diff --git a/plugins/gzip/configuration.h b/plugins/gzip/configuration.h
index 04ff023..09c9559 100644
--- a/plugins/gzip/configuration.h
+++ b/plugins/gzip/configuration.h
@@ -33,11 +33,24 @@ namespace Gzip
 {
 typedef std::vector<std::string> StringContainer;
 
+enum CompressionAlgorithm {
+  ALGORITHM_DEFAULT = 0,
+  ALGORITHM_DEFLATE = 1,
+  ALGORITHM_GZIP    = 2,
+  ALGORITHM_BROTLI  = 4 // For bit manipulations
+};
+
 class HostConfiguration
 {
 public:
   explicit HostConfiguration(const std::string &host)
-    : host_(host), enabled_(true), cache_(true), remove_accept_encoding_(false), flush_(false),
ref_count_(0)
+    : host_(host),
+      enabled_(true),
+      cache_(true),
+      remove_accept_encoding_(false),
+      flush_(false),
+      compression_algorithms_(ALGORITHM_GZIP),
+      ref_count_(0)
   {
   }
 
@@ -96,6 +109,8 @@ public:
   void add_compressible_content_type(const std::string &content_type);
   bool is_url_allowed(const char *url, int url_len);
   bool is_content_type_compressible(const char *content_type, int content_type_length);
+  void add_compression_algorithms(const std::string &algorithms);
+  int compression_algorithms();
 
   // Ref-counting these host configuration objects
   void
@@ -118,6 +133,7 @@ private:
   bool cache_;
   bool remove_accept_encoding_;
   bool flush_;
+  int compression_algorithms_;
   volatile int ref_count_;
 
   StringContainer compressible_content_types_;
diff --git a/plugins/gzip/gzip.cc b/plugins/gzip/gzip.cc
index 51110b1..ad6bbe0 100644
--- a/plugins/gzip/gzip.cc
+++ b/plugins/gzip/gzip.cc
@@ -24,6 +24,10 @@
 #include <string.h>
 #include <zlib.h>
 
+#if HAVE_BROTLI_ENCODE_H
+#include <brotli/encode.h>
+#endif
+
 #include "ts/ts.h"
 #include "ts/ink_defs.h"
 
@@ -36,7 +40,6 @@ using namespace std;
 using namespace Gzip;
 
 // FIXME: custom dictionaries would be nice. configurable/content-type?
-// FIXME: look into autoscaling the compression level based on connection speed
 // a gprs device might benefit from a higher compression ratio, whereas a desktop w. high
bandwith
 // might be served better with little or no compression at all
 // FIXME: look into compressing from the task thread pool
@@ -50,37 +53,48 @@ using namespace Gzip;
 
 const int ZLIB_COMPRESSION_LEVEL = 6;
 const char *global_hidden_header_name;
-const char *dictionary = nullptr;
+const char *dictionary           = nullptr;
+const char *TS_HTTP_VALUE_BROTLI = "br";
+const int TS_HTTP_LEN_BROTLI     = 2;
+
+// brotli compression quality 1-11. Testing proved level '6'
+#if HAVE_BROTLI_ENCODE_H
+const int BROTLI_COMPRESSION_LEVEL = 6;
+const int BROTLI_LGW               = 16;
+#endif
 
 // Current global configuration, and the previous one (for cleanup)
 Configuration *cur_config  = nullptr;
 Configuration *prev_config = nullptr;
 
-static GzipData *
-gzip_data_alloc(int compression_type)
+static Data *
+data_alloc(int compression_type, int compression_algorithms)
 {
-  GzipData *data;
+  Data *data;
   int err;
 
-  data                    = (GzipData *)TSmalloc(sizeof(GzipData));
-  data->downstream_vio    = nullptr;
-  data->downstream_buffer = nullptr;
-  data->downstream_reader = nullptr;
-  data->downstream_length = 0;
-  data->state             = transform_state_initialized;
-  data->compression_type  = compression_type;
-  data->zstrm.next_in     = Z_NULL;
-  data->zstrm.avail_in    = 0;
-  data->zstrm.total_in    = 0;
-  data->zstrm.next_out    = Z_NULL;
-  data->zstrm.avail_out   = 0;
-  data->zstrm.total_out   = 0;
-  data->zstrm.zalloc      = gzip_alloc;
-  data->zstrm.zfree       = gzip_free;
-  data->zstrm.opaque      = (voidpf) nullptr;
-  data->zstrm.data_type   = Z_ASCII;
-
-  int window_bits = (compression_type == COMPRESSION_TYPE_GZIP) ? WINDOW_BITS_GZIP : WINDOW_BITS_DEFLATE;
+  data                         = (Data *)TSmalloc(sizeof(Data));
+  data->downstream_vio         = nullptr;
+  data->downstream_buffer      = nullptr;
+  data->downstream_reader      = nullptr;
+  data->downstream_length      = 0;
+  data->state                  = transform_state_initialized;
+  data->compression_type       = compression_type;
+  data->compression_algorithms = compression_algorithms;
+  data->zstrm.next_in          = Z_NULL;
+  data->zstrm.avail_in         = 0;
+  data->zstrm.total_in         = 0;
+  data->zstrm.next_out         = Z_NULL;
+  data->zstrm.avail_out        = 0;
+  data->zstrm.total_out        = 0;
+  data->zstrm.zalloc           = gzip_alloc;
+  data->zstrm.zfree            = gzip_free;
+  data->zstrm.opaque           = (voidpf) nullptr;
+  data->zstrm.data_type        = Z_ASCII;
+
+  int window_bits = WINDOW_BITS_GZIP;
+  if (compression_type & COMPRESSION_TYPE_DEFLATE)
+    window_bits = WINDOW_BITS_DEFLATE;
 
   err = deflateInit2(&data->zstrm, ZLIB_COMPRESSION_LEVEL, Z_DEFLATED, window_bits,
ZLIB_MEMLEVEL, Z_DEFAULT_STRATEGY);
 
@@ -94,12 +108,28 @@ gzip_data_alloc(int compression_type)
       fatal("gzip-transform: ERROR: deflateSetDictionary (%d)!", err);
     }
   }
-
+#if HAVE_BROTLI_ENCODE_H
+  if (compression_type & COMPRESSION_TYPE_BROTLI) {
+    debug("gzip-transform: brotli compression. Create Brotli Encoder Instance.");
+    data->bstrm.br = BrotliEncoderCreateInstance(0, 0, 0);
+    if (!data->bstrm.br) {
+      fatal("gzip-transform: ERROR: Brotli Encoder Instance Failed");
+    }
+    BrotliEncoderSetParameter(data->bstrm.br, BROTLI_PARAM_QUALITY, BROTLI_COMPRESSION_LEVEL);
+    BrotliEncoderSetParameter(data->bstrm.br, BROTLI_PARAM_LGWIN, BROTLI_LGW);
+    data->bstrm.next_in   = nullptr;
+    data->bstrm.avail_in  = 0;
+    data->bstrm.total_in  = 0;
+    data->bstrm.next_out  = nullptr;
+    data->bstrm.avail_out = 0;
+    data->bstrm.total_out = 0;
+  }
+#endif
   return data;
 }
 
 static void
-gzip_data_destroy(GzipData *data)
+data_destroy(Data *data)
 {
   TSReleaseAssert(data);
 
@@ -111,24 +141,41 @@ gzip_data_destroy(GzipData *data)
     TSIOBufferDestroy(data->downstream_buffer);
   }
 
+// brotlidestory
+#if HAVE_BROTLI_ENCODE_H
+  BrotliEncoderDestroyInstance(data->bstrm.br);
+#endif
+
   TSfree(data);
 }
 
 static TSReturnCode
-gzip_content_encoding_header(TSMBuffer bufp, TSMLoc hdr_loc, const int compression_type)
+content_encoding_header(TSMBuffer bufp, TSMLoc hdr_loc, const int compression_type, int algorithm)
 {
   TSReturnCode ret;
   TSMLoc ce_loc;
-
+  const char *value = nullptr;
+  int value_len     = 0;
   // Delete Content-Encoding if present???
+  if (compression_type & COMPRESSION_TYPE_BROTLI && (algorithm & ALGORITHM_BROTLI))
{
+    value     = TS_HTTP_VALUE_BROTLI;
+    value_len = TS_HTTP_LEN_BROTLI;
+  } else if (compression_type & COMPRESSION_TYPE_GZIP && (algorithm & ALGORITHM_GZIP))
{
+    value     = TS_HTTP_VALUE_GZIP;
+    value_len = TS_HTTP_LEN_GZIP;
+  } else if (compression_type & COMPRESSION_TYPE_DEFLATE && (algorithm &
ALGORITHM_DEFLATE)) {
+    value     = TS_HTTP_VALUE_DEFLATE;
+    value_len = TS_HTTP_LEN_DEFLATE;
+  }
+
+  if (value_len == 0) {
+    error("no need to add Content-Encoding header");
+    return TS_SUCCESS;
+  }
 
   if ((ret = TSMimeHdrFieldCreateNamed(bufp, hdr_loc, TS_MIME_FIELD_CONTENT_ENCODING, TS_MIME_LEN_CONTENT_ENCODING,
&ce_loc)) ==
       TS_SUCCESS) {
-    if (compression_type == COMPRESSION_TYPE_DEFLATE) {
-      ret = TSMimeHdrFieldValueStringInsert(bufp, hdr_loc, ce_loc, -1, TS_HTTP_VALUE_DEFLATE,
TS_HTTP_LEN_DEFLATE);
-    } else if (compression_type == COMPRESSION_TYPE_GZIP) {
-      ret = TSMimeHdrFieldValueStringInsert(bufp, hdr_loc, ce_loc, -1, TS_HTTP_VALUE_GZIP,
TS_HTTP_LEN_GZIP);
-    }
+    ret = TSMimeHdrFieldValueStringInsert(bufp, hdr_loc, ce_loc, -1, value, value_len);
     if (ret == TS_SUCCESS) {
       ret = TSMimeHdrFieldAppend(bufp, hdr_loc, ce_loc);
     }
@@ -143,7 +190,7 @@ gzip_content_encoding_header(TSMBuffer bufp, TSMLoc hdr_loc, const int
compressi
 }
 
 static TSReturnCode
-gzip_vary_header(TSMBuffer bufp, TSMLoc hdr_loc)
+vary_header(TSMBuffer bufp, TSMLoc hdr_loc)
 {
   TSReturnCode ret;
   TSMLoc ce_loc;
@@ -186,7 +233,7 @@ gzip_vary_header(TSMBuffer bufp, TSMLoc hdr_loc)
 // FIXME: the etag alteration isn't proper. it should modify the value inside quotes
 //       specify a very header..
 static TSReturnCode
-gzip_etag_header(TSMBuffer bufp, TSMLoc hdr_loc)
+etag_header(TSMBuffer bufp, TSMLoc hdr_loc)
 {
   TSReturnCode ret = TS_SUCCESS;
   TSMLoc ce_loc;
@@ -220,7 +267,7 @@ gzip_etag_header(TSMBuffer bufp, TSMLoc hdr_loc)
 
 // FIXME: some things are potentially compressible. those responses
 static void
-gzip_transform_init(TSCont contp, GzipData *data)
+compress_transform_init(TSCont contp, Data *data)
 {
   // update the vary, content-encoding, and etag response headers
   // prepare the downstream for transforming
@@ -236,8 +283,8 @@ gzip_transform_init(TSCont contp, GzipData *data)
     return;
   }
 
-  if (gzip_content_encoding_header(bufp, hdr_loc, data->compression_type) == TS_SUCCESS
&&
-      gzip_vary_header(bufp, hdr_loc) == TS_SUCCESS && gzip_etag_header(bufp, hdr_loc)
== TS_SUCCESS) {
+  if (content_encoding_header(bufp, hdr_loc, data->compression_type, data->compression_algorithms)
== TS_SUCCESS &&
+      vary_header(bufp, hdr_loc) == TS_SUCCESS && etag_header(bufp, hdr_loc) == TS_SUCCESS)
{
     downstream_conn         = TSTransformOutputVConnGet(contp);
     data->downstream_buffer = TSIOBufferCreate();
     data->downstream_reader = TSIOBufferReaderAlloc(data->downstream_buffer);
@@ -248,14 +295,102 @@ gzip_transform_init(TSCont contp, GzipData *data)
 }
 
 static void
-gzip_transform_one(GzipData *data, TSIOBufferReader upstream_reader, int amount)
+gzip_transform_one(Data *data, const char *upstream_buffer, int64_t upstream_length)
 {
   TSIOBufferBlock downstream_blkp;
-  const char *upstream_buffer;
   char *downstream_buffer;
-  int64_t upstream_length, downstream_length;
+  int64_t downstream_length;
+  int err;
+  data->zstrm.next_in  = (unsigned char *)upstream_buffer;
+  data->zstrm.avail_in = upstream_length;
+
+  while (data->zstrm.avail_in > 0) {
+    downstream_blkp   = TSIOBufferStart(data->downstream_buffer);
+    downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length);
+
+    data->zstrm.next_out  = (unsigned char *)downstream_buffer;
+    data->zstrm.avail_out = downstream_length;
+
+    if (!data->hc->flush()) {
+      err = deflate(&data->zstrm, Z_NO_FLUSH);
+    } else {
+      err = deflate(&data->zstrm, Z_SYNC_FLUSH);
+    }
+
+    if (err != Z_OK) {
+      warning("deflate() call failed: %d", err);
+    }
+
+    if (downstream_length > data->zstrm.avail_out) {
+      TSIOBufferProduce(data->downstream_buffer, downstream_length - data->zstrm.avail_out);
+      data->downstream_length += (downstream_length - data->zstrm.avail_out);
+    }
+
+    if (data->zstrm.avail_out > 0) {
+      if (data->zstrm.avail_in != 0) {
+        error("gzip-transform: ERROR: avail_in is (%d): should be 0", data->zstrm.avail_in);
+      }
+    }
+  }
+}
+
+static void
+brotli_transform_one(Data *data, const char *upstream_buffer, int64_t upstream_length)
+{
+#if HAVE_BROTLI_ENCODE_H
+  TSIOBufferBlock downstream_blkp;
+  char *downstream_buffer;
+  int64_t downstream_length;
   int err;
 
+  data->bstrm.avail_in = upstream_length;
+
+  while (data->bstrm.avail_in > 0) {
+    downstream_blkp   = TSIOBufferStart(data->downstream_buffer);
+    downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length);
+
+    data->bstrm.next_out  = (unsigned char *)downstream_buffer;
+    data->bstrm.avail_out = downstream_length;
+    data->bstrm.total_out = 0;
+
+    data->bstrm.next_in = (uint8_t *)upstream_buffer;
+    if (!data->hc->flush()) {
+      err = BrotliEncoderCompressStream(data->bstrm.br, BROTLI_OPERATION_PROCESS, &data->bstrm.avail_in,
+                                        (const uint8_t **)&data->bstrm.next_in, &data->bstrm.avail_out,
&data->bstrm.next_out,
+                                        &data->bstrm.total_out);
+    } else {
+      err = BrotliEncoderCompressStream(data->bstrm.br, BROTLI_OPERATION_FLUSH, &data->bstrm.avail_in,
+                                        (const uint8_t **)&data->bstrm.next_in, &data->bstrm.avail_out,
&data->bstrm.next_out,
+                                        &data->bstrm.total_out);
+    }
+
+    if (err != BROTLI_TRUE) {
+      warning("BrotliEncoderCompressStream() call failed: %d", err);
+    }
+
+    if (downstream_length > (int64_t)data->bstrm.avail_out) {
+      TSIOBufferProduce(data->downstream_buffer, downstream_length - data->bstrm.avail_out);
+      data->downstream_length += (downstream_length - data->bstrm.avail_out);
+    }
+
+    if (data->bstrm.avail_out > 0) {
+      if (data->bstrm.avail_in != 0) {
+        error("brotli-transform: ERROR: brotli avail_in is (%lu): should be 0", data->bstrm.avail_in);
+      }
+    }
+  }
+  data->bstrm.total_in += upstream_length;
+#else
+  error("brotli-transform: ERROR: compile with brotli support");
+#endif
+}
+
+static void
+compress_transform_one(Data *data, TSIOBufferReader upstream_reader, int amount)
+{
+  TSIOBufferBlock downstream_blkp;
+  const char *upstream_buffer;
+  int64_t upstream_length;
   while (amount > 0) {
     downstream_blkp = TSIOBufferReaderStart(upstream_reader);
     if (!downstream_blkp) {
@@ -273,38 +408,13 @@ gzip_transform_one(GzipData *data, TSIOBufferReader upstream_reader,
int amount)
       upstream_length = amount;
     }
 
-    data->zstrm.next_in  = (unsigned char *)upstream_buffer;
-    data->zstrm.avail_in = upstream_length;
-
-    while (data->zstrm.avail_in > 0) {
-      downstream_blkp   = TSIOBufferStart(data->downstream_buffer);
-      downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length);
-
-      data->zstrm.next_out  = (unsigned char *)downstream_buffer;
-      data->zstrm.avail_out = downstream_length;
-
-      if (!data->hc->flush()) {
-        debug("gzip_transform: deflate with Z_NO_FLUSH");
-        err = deflate(&data->zstrm, Z_NO_FLUSH);
-      } else {
-        debug("gzip_transform: deflate with Z_SYNC_FLUSH");
-        err = deflate(&data->zstrm, Z_SYNC_FLUSH);
-      }
-
-      if (err != Z_OK) {
-        warning("deflate() call failed: %d", err);
-      }
-
-      if (downstream_length > data->zstrm.avail_out) {
-        TSIOBufferProduce(data->downstream_buffer, downstream_length - data->zstrm.avail_out);
-        data->downstream_length += (downstream_length - data->zstrm.avail_out);
-      }
-
-      if (data->zstrm.avail_out > 0) {
-        if (data->zstrm.avail_in != 0) {
-          error("gzip-transform: ERROR: avail_in is (%d): should be 0", data->zstrm.avail_in);
-        }
-      }
+    if (data->compression_type & COMPRESSION_TYPE_BROTLI && (data->compression_algorithms
& ALGORITHM_BROTLI)) {
+      brotli_transform_one(data, upstream_buffer, upstream_length);
+    } else if ((data->compression_type & (COMPRESSION_TYPE_GZIP | COMPRESSION_TYPE_DEFLATE))
&&
+               (data->compression_algorithms & (ALGORITHM_GZIP | ALGORITHM_DEFLATE)))
{
+      gzip_transform_one(data, upstream_buffer, upstream_length);
+    } else {
+      warning("No compression supported. Shoudn't come here.");
     }
 
     TSIOBufferReaderConsume(upstream_reader, upstream_length);
@@ -313,7 +423,7 @@ gzip_transform_one(GzipData *data, TSIOBufferReader upstream_reader, int
amount)
 }
 
 static void
-gzip_transform_finish(GzipData *data)
+gzip_transform_finish(Data *data)
 {
   if (data->state == transform_state_output) {
     TSIOBufferBlock downstream_blkp;
@@ -350,30 +460,94 @@ gzip_transform_finish(GzipData *data)
     if (data->downstream_length != (int64_t)(data->zstrm.total_out)) {
       error("gzip-transform: ERROR: output lengths don't match (%d, %ld)", data->downstream_length,
data->zstrm.total_out);
     }
-
+    debug("gzip-transform: Finished gzip");
     gzip_log_ratio(data->zstrm.total_in, data->downstream_length);
   }
 }
 
 static void
-gzip_transform_do(TSCont contp)
+brotli_transform_finish(Data *data)
+{
+#if HAVE_BROTLI_ENCODE_H
+  if (data->state == transform_state_output) {
+    TSIOBufferBlock downstream_blkp;
+    char *downstream_buffer;
+    int64_t downstream_length;
+    int err;
+
+    data->state = transform_state_finished;
+
+    for (;;) {
+      downstream_blkp = TSIOBufferStart(data->downstream_buffer);
+
+      downstream_buffer     = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length);
+      data->bstrm.next_out  = (unsigned char *)downstream_buffer;
+      data->bstrm.avail_out = downstream_length;
+
+      err = BrotliEncoderCompressStream(data->bstrm.br, BROTLI_OPERATION_FINISH, &data->bstrm.avail_in,
+                                        (const uint8_t **)&data->bstrm.next_in, &data->bstrm.avail_out,
&data->bstrm.next_out,
+                                        &data->bstrm.total_out);
+
+      if (downstream_length > (int64_t)data->bstrm.avail_out) {
+        TSIOBufferProduce(data->downstream_buffer, downstream_length - data->bstrm.avail_out);
+        data->downstream_length += (downstream_length - data->bstrm.avail_out);
+      }
+      if (!BrotliEncoderIsFinished(data->bstrm.br)) {
+        continue;
+      }
+
+      if (err != BROTLI_TRUE) { /* some more data to encode */
+        warning("brotli_transform: BrotliEncoderCompressStream should return BROTLI_TRUE");
+      }
+
+      break;
+    }
+
+    if (data->downstream_length != (int64_t)(data->bstrm.total_out)) {
+      error("brotli-transform: ERROR: output lengths don't match (%d, %ld)", data->downstream_length,
data->bstrm.total_out);
+    }
+    debug("brotli-transform: Finished brotli");
+    gzip_log_ratio(data->bstrm.total_in, data->downstream_length);
+  }
+#else
+  error("brotli-transform: compile with brotli support");
+#endif
+}
+
+static void
+compress_transform_finish(Data *data)
+{
+  if (data->compression_type & COMPRESSION_TYPE_BROTLI && data->compression_algorithms
& ALGORITHM_BROTLI) {
+    brotli_transform_finish(data);
+    debug("brotli-transform: Brolti compression finish.");
+  } else if ((data->compression_type & (COMPRESSION_TYPE_GZIP | COMPRESSION_TYPE_DEFLATE))
&&
+             (data->compression_algorithms & (ALGORITHM_GZIP | ALGORITHM_DEFLATE)))
{
+    gzip_transform_finish(data);
+    debug("gzip-transform: Gzip compression finish.");
+  } else {
+    warning("No Compression matched, shouldn't come here.");
+  }
+}
+
+static void
+compress_transform_do(TSCont contp)
 {
   TSVIO upstream_vio;
-  GzipData *data;
+  Data *data;
   int64_t upstream_todo;
   int64_t upstream_avail;
   int64_t downstream_bytes_written;
 
-  data = (GzipData *)TSContDataGet(contp);
+  data = (Data *)TSContDataGet(contp);
   if (data->state == transform_state_initialized) {
-    gzip_transform_init(contp, data);
+    compress_transform_init(contp, data);
   }
 
   upstream_vio             = TSVConnWriteVIOGet(contp);
   downstream_bytes_written = data->downstream_length;
 
   if (!TSVIOBufferGet(upstream_vio)) {
-    gzip_transform_finish(data);
+    compress_transform_finish(data);
 
     TSVIONBytesSet(data->downstream_vio, data->downstream_length);
 
@@ -393,7 +567,7 @@ gzip_transform_do(TSCont contp)
     }
 
     if (upstream_todo > 0) {
-      gzip_transform_one(data, TSVIOReaderGet(upstream_vio), upstream_todo);
+      compress_transform_one(data, TSVIOReaderGet(upstream_vio), upstream_todo);
       TSVIONDoneSet(upstream_vio, TSVIONDoneGet(upstream_vio) + upstream_todo);
     }
   }
@@ -406,7 +580,7 @@ gzip_transform_do(TSCont contp)
       TSContCall(TSVIOContGet(upstream_vio), TS_EVENT_VCONN_WRITE_READY, upstream_vio);
     }
   } else {
-    gzip_transform_finish(data);
+    compress_transform_finish(data);
     TSVIONBytesSet(data->downstream_vio, data->downstream_length);
 
     if (data->downstream_length > downstream_bytes_written) {
@@ -418,10 +592,10 @@ gzip_transform_do(TSCont contp)
 }
 
 static int
-gzip_transform(TSCont contp, TSEvent event, void * /* edata ATS_UNUSED */)
+compress_transform(TSCont contp, TSEvent event, void * /* edata ATS_UNUSED */)
 {
   if (TSVConnClosedGet(contp)) {
-    gzip_data_destroy((GzipData *)TSContDataGet(contp));
+    data_destroy((Data *)TSContDataGet(contp));
     TSContDestroy(contp);
     return 0;
   } else {
@@ -435,14 +609,14 @@ gzip_transform(TSCont contp, TSEvent event, void * /* edata ATS_UNUSED
*/)
       TSVConnShutdown(TSTransformOutputVConnGet(contp), 0, 1);
       break;
     case TS_EVENT_VCONN_WRITE_READY:
-      gzip_transform_do(contp);
+      compress_transform_do(contp);
       break;
     case TS_EVENT_IMMEDIATE:
-      gzip_transform_do(contp);
+      compress_transform_do(contp);
       break;
     default:
       warning("unknown event [%d]", event);
-      gzip_transform_do(contp);
+      compress_transform_do(contp);
       break;
     }
   }
@@ -451,7 +625,7 @@ gzip_transform(TSCont contp, TSEvent event, void * /* edata ATS_UNUSED
*/)
 }
 
 static int
-gzip_transformable(TSHttpTxn txnp, bool server, HostConfiguration *host_configuration, int
*compress_type)
+transformable(TSHttpTxn txnp, bool server, HostConfiguration *host_configuration, int *compress_type,
int *algorithms)
 {
   /* Server response header */
   TSMBuffer bufp;
@@ -468,6 +642,13 @@ gzip_transformable(TSHttpTxn txnp, bool server, HostConfiguration *host_configur
   int i, compression_acceptable, len;
   TSHttpStatus resp_status;
 
+  /*
+    // Before anything, check atleast one compression algorithm is supported
+    if (host_configuration->compression_algorithms() == ALGORITHM_DEFAULT) {
+      info("No compression algorithms configured");
+      return 0;
+    }
+  */
   if (server) {
     if (TS_SUCCESS != TSHttpTxnServerRespGet(txnp, &bufp, &hdr_loc)) {
       return 0;
@@ -503,7 +684,8 @@ gzip_transformable(TSHttpTxn txnp, bool server, HostConfiguration *host_configur
     return 0;
   }
 
-  cfield = TSMimeHdrFieldFind(cbuf, chdr, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING);
+  *algorithms = host_configuration->compression_algorithms();
+  cfield      = TSMimeHdrFieldFind(cbuf, chdr, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING);
   if (cfield != TS_NULL_MLOC) {
     compression_acceptable = 0;
     nvalues                = TSMimeHdrFieldValuesCount(cbuf, chdr, cfield);
@@ -513,14 +695,18 @@ gzip_transformable(TSHttpTxn txnp, bool server, HostConfiguration *host_configur
         continue;
       }
 
-      if (strncasecmp(value, "deflate", sizeof("deflate") - 1) == 0) {
-        compression_acceptable = 1;
-        *compress_type         = COMPRESSION_TYPE_DEFLATE;
-        break;
+      if (strncasecmp(value, "br", sizeof("br") - 1) == 0) {
+        if (*algorithms & ALGORITHM_BROTLI)
+          compression_acceptable = 1;
+        *compress_type |= COMPRESSION_TYPE_BROTLI;
+      } else if (strncasecmp(value, "deflate", sizeof("deflate") - 1) == 0) {
+        if (*algorithms & ALGORITHM_DEFLATE)
+          compression_acceptable = 1;
+        *compress_type |= COMPRESSION_TYPE_DEFLATE;
       } else if (strncasecmp(value, "gzip", sizeof("gzip") - 1) == 0) {
-        compression_acceptable = 1;
-        *compress_type         = COMPRESSION_TYPE_GZIP;
-        break;
+        if (*algorithms & ALGORITHM_GZIP)
+          compression_acceptable = 1;
+        *compress_type |= COMPRESSION_TYPE_GZIP;
       }
     }
 
@@ -528,7 +714,7 @@ gzip_transformable(TSHttpTxn txnp, bool server, HostConfiguration *host_configur
     TSHandleMLocRelease(cbuf, TS_NULL_MLOC, chdr);
 
     if (!compression_acceptable) {
-      info("no acceptable encoding found in request header, not compressible");
+      info("no acceptable encoding match found in request header, not compressible");
       TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
       return 0;
     }
@@ -574,21 +760,24 @@ gzip_transformable(TSHttpTxn txnp, bool server, HostConfiguration *host_configur
 }
 
 static void
-gzip_transform_add(TSHttpTxn txnp, HostConfiguration *hc, int compress_type)
+compress_transform_add(TSHttpTxn txnp, HostConfiguration *hc, int compress_type, int algorithms)
 {
   TSVConn connp;
-  GzipData *data;
+  Data *data;
 
   TSHttpTxnUntransformedRespCache(txnp, 1);
 
   if (!hc->cache()) {
+    debug("TransformedRespCache  not enabled");
     TSHttpTxnTransformedRespCache(txnp, 0);
   } else {
+    debug("TransformedRespCache  enabled");
+    TSHttpTxnUntransformedRespCache(txnp, 0);
     TSHttpTxnTransformedRespCache(txnp, 1);
   }
 
-  connp     = TSTransformCreate(gzip_transform, txnp);
-  data      = gzip_data_alloc(compress_type);
+  connp     = TSTransformCreate(compress_transform, txnp);
+  data      = data_alloc(compress_type, algorithms);
   data->txn = txnp;
   data->hc  = hc;
 
@@ -620,7 +809,8 @@ static int
 transform_plugin(TSCont contp, TSEvent event, void *edata)
 {
   TSHttpTxn txnp        = (TSHttpTxn)edata;
-  int compress_type     = COMPRESSION_TYPE_DEFLATE;
+  int compress_type     = COMPRESSION_TYPE_DEFAULT;
+  int algorithms        = ALGORITHM_DEFAULT;
   HostConfiguration *hc = (HostConfiguration *)TSContDataGet(contp);
 
   switch (event) {
@@ -639,8 +829,8 @@ transform_plugin(TSCont contp, TSEvent event, void *edata)
         }
       }
 
-      if (gzip_transformable(txnp, true, hc, &compress_type)) {
-        gzip_transform_add(txnp, hc, compress_type);
+      if (transformable(txnp, true, hc, &compress_type, &algorithms)) {
+        compress_transform_add(txnp, hc, compress_type, algorithms);
       }
     }
     break;
@@ -667,8 +857,8 @@ transform_plugin(TSCont contp, TSEvent event, void *edata)
     if (TS_ERROR != TSHttpTxnCacheLookupStatusGet(txnp, &obj_status) && (TS_CACHE_LOOKUP_HIT_FRESH
== obj_status)) {
       if (hc != nullptr) {
         info("handling compression of cached object");
-        if (gzip_transformable(txnp, false, hc, &compress_type)) {
-          gzip_transform_add(txnp, hc, compress_type);
+        if (transformable(txnp, false, hc, &compress_type, &algorithms)) {
+          compress_transform_add(txnp, hc, compress_type, algorithms);
         }
       }
     } else {
@@ -703,7 +893,7 @@ transform_plugin(TSCont contp, TSEvent event, void *edata)
  *    further processing
  */
 static void
-handle_gzip_request(TSHttpTxn txnp, Configuration *config)
+handle_request(TSHttpTxn txnp, Configuration *config)
 {
   TSMBuffer req_buf;
   TSMLoc req_loc;
@@ -721,14 +911,12 @@ handle_gzip_request(TSHttpTxn txnp, Configuration *config)
       if (hc->has_disallows()) {
         int url_len;
         char *url = TSHttpTxnEffectiveUrlStringGet(txnp, &url_len);
-
-        allowed = hc->is_url_allowed(url, url_len);
+        allowed   = hc->is_url_allowed(url, url_len);
         TSfree(url);
       } else {
         allowed = true;
       }
     }
-
     if (allowed) {
       TSCont transform_contp = TSContCreate(transform_plugin, nullptr);
 
@@ -753,7 +941,7 @@ transform_global_plugin(TSCont /* contp ATS_UNUSED */, TSEvent event,
void *edat
   switch (event) {
   case TS_EVENT_HTTP_READ_REQUEST_HDR:
     // Handle gzip request and use the global configs
-    handle_gzip_request(txnp, nullptr);
+    handle_request(txnp, nullptr);
     break;
 
   default:
@@ -887,7 +1075,7 @@ TSRemapDoRemap(void *instance, TSHttpTxn txnp, TSRemapRequestInfo *rri)
     info("Remap Rules configured for gzip");
     Configuration *config = (Configuration *)instance;
     // Handle gzip request and use the configs populated from remap instance
-    handle_gzip_request(txnp, config);
+    handle_request(txnp, config);
   }
   return TSREMAP_NO_REMAP;
 }
diff --git a/plugins/gzip/misc.cc b/plugins/gzip/misc.cc
index 161abda..c0d750e 100644
--- a/plugins/gzip/misc.cc
+++ b/plugins/gzip/misc.cc
@@ -47,7 +47,7 @@ normalize_accept_encoding(TSHttpTxn /* txnp ATS_UNUSED */, TSMBuffer reqp,
TSMLo
   TSMLoc field = TSMimeHdrFieldFind(reqp, hdr_loc, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING);
   int deflate  = 0;
   int gzip     = 0;
-
+  int br       = 0;
   // remove the accept encoding field(s),
   // while finding out if gzip or deflate is supported.
   while (field) {
@@ -63,6 +63,9 @@ normalize_accept_encoding(TSHttpTxn /* txnp ATS_UNUSED */, TSMBuffer reqp,
TSMLo
         --value_count;
         val = TSMimeHdrFieldValueStringGet(reqp, hdr_loc, field, value_count, &val_len);
 
+        if (val_len == (int)strlen("br")) {
+          br = !strncmp(val, "br", val_len);
+        }
         if (val_len == (int)strlen("gzip")) {
           gzip = !strncmp(val, "gzip", val_len);
         } else if (val_len == (int)strlen("deflate")) {
@@ -78,10 +81,13 @@ normalize_accept_encoding(TSHttpTxn /* txnp ATS_UNUSED */, TSMBuffer reqp,
TSMLo
   }
 
   // append a new accept-encoding field in the header
-  if (deflate || gzip) {
+  if (deflate || gzip || br) {
     TSMimeHdrFieldCreate(reqp, hdr_loc, &field);
     TSMimeHdrFieldNameSet(reqp, hdr_loc, field, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING);
-
+    if (br) {
+      TSMimeHdrFieldValueStringInsert(reqp, hdr_loc, field, -1, "br", strlen("br"));
+      info("normalized accept encoding to br");
+    }
     if (gzip) {
       TSMimeHdrFieldValueStringInsert(reqp, hdr_loc, field, -1, "gzip", strlen("gzip"));
       info("normalized accept encoding to gzip");
diff --git a/plugins/gzip/misc.h b/plugins/gzip/misc.h
index 60f5aea..6c00cc0 100644
--- a/plugins/gzip/misc.h
+++ b/plugins/gzip/misc.h
@@ -29,6 +29,10 @@
 #include <stdlib.h>
 #include <stdio.h>
 
+#if HAVE_BROTLI_ENCODE_H
+#include <brotli/encode.h>
+#endif
+
 #include "configuration.h"
 
 using namespace Gzip;
@@ -39,8 +43,12 @@ static const int WINDOW_BITS_DEFLATE = -15;
 static const int WINDOW_BITS_GZIP    = 31;
 
 // misc
-static const int COMPRESSION_TYPE_DEFLATE = 1;
-static const int COMPRESSION_TYPE_GZIP    = 2;
+enum CompressionType {
+  COMPRESSION_TYPE_DEFAULT = 0,
+  COMPRESSION_TYPE_DEFLATE = 1,
+  COMPRESSION_TYPE_GZIP    = 2,
+  COMPRESSION_TYPE_BROTLI  = 4
+};
 
 // this one is used to rename the accept encoding header
 // it will be restored later on
@@ -53,6 +61,18 @@ enum transform_state {
   transform_state_finished,
 };
 
+#if HAVE_BROTLI_ENCODE_H
+typedef struct {
+  BrotliEncoderState *br;
+  uint8_t *next_in;
+  size_t avail_in;
+  uint8_t *next_out;
+  size_t avail_out;
+  size_t total_in;
+  size_t total_out;
+} b_stream;
+#endif
+
 typedef struct {
   TSHttpTxn txn;
   HostConfiguration *hc;
@@ -63,7 +83,11 @@ typedef struct {
   z_stream zstrm;
   enum transform_state state;
   int compression_type;
-} GzipData;
+  int compression_algorithms;
+#if HAVE_BROTLI_ENCODE_H
+  b_stream bstrm;
+#endif
+} Data;
 
 voidpf gzip_alloc(voidpf opaque, uInt items, uInt size);
 void gzip_free(voidpf opaque, voidpf address);

-- 
To stop receiving notification emails like this one, please contact
['"commits@trafficserver.apache.org" <commits@trafficserver.apache.org>'].

Mime
View raw message