knox-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kmin...@apache.org
Subject knox git commit: [KNOX-695] - Expose configuration of HttpClient's connection and socket timeout settings
Date Fri, 18 Mar 2016 12:50:01 GMT
Repository: knox
Updated Branches:
  refs/heads/master 15ae0d781 -> 8cb451daf


[KNOX-695] - Expose configuration of HttpClient's connection and socket timeout settings


Project: http://git-wip-us.apache.org/repos/asf/knox/repo
Commit: http://git-wip-us.apache.org/repos/asf/knox/commit/8cb451da
Tree: http://git-wip-us.apache.org/repos/asf/knox/tree/8cb451da
Diff: http://git-wip-us.apache.org/repos/asf/knox/diff/8cb451da

Branch: refs/heads/master
Commit: 8cb451daf2d2db841780eab78afe928de397c966
Parents: 15ae0d7
Author: Kevin Minder <kminder@apache.org>
Authored: Fri Mar 18 08:49:57 2016 -0400
Committer: Kevin Minder <kminder@apache.org>
Committed: Fri Mar 18 08:49:57 2016 -0400

----------------------------------------------------------------------
 CHANGES                                         |  1 +
 .../gateway/config/impl/GatewayConfigImpl.java  | 40 +++++++++++
 gateway-spi/pom.xml                             |  4 ++
 .../hadoop/gateway/config/GatewayConfig.java    |  4 ++
 .../gateway/dispatch/DefaultDispatch.java       |  6 +-
 .../dispatch/DefaultHttpClientFactory.java      | 74 ++++++++++++++++++--
 .../hadoop/gateway/GatewayTestConfig.java       | 10 +++
 .../hadoop/gateway/GatewayTestConfig.java       | 10 +++
 8 files changed, 142 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/knox/blob/8cb451da/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 5963fca..6d21512 100644
--- a/CHANGES
+++ b/CHANGES
@@ -11,6 +11,7 @@ Release Notes - Apache Knox - Version 0.9.0
     * [KNOX-678] - Malformed UTF-8 characters in JSON Response
     * [KNOX-680] - Update Knox's HttpClient dependency to latest version
     * [KNOX-694] - Enhance LDAP user search configurability
+    * [KNOX-695] - Expose configuration of HttpClient's connection and socket timeout settings
 ** Bug
     * [KNOX-681] - A PUT with Content-Type application/xml but no body causes NullPointerException
 

http://git-wip-us.apache.org/repos/asf/knox/blob/8cb451da/gateway-server/src/main/java/org/apache/hadoop/gateway/config/impl/GatewayConfigImpl.java
----------------------------------------------------------------------
diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/config/impl/GatewayConfigImpl.java
b/gateway-server/src/main/java/org/apache/hadoop/gateway/config/impl/GatewayConfigImpl.java
index 711efc8..82a21ee 100644
--- a/gateway-server/src/main/java/org/apache/hadoop/gateway/config/impl/GatewayConfigImpl.java
+++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/config/impl/GatewayConfigImpl.java
@@ -119,6 +119,8 @@ public class GatewayConfigImpl extends Configuration implements GatewayConfig
{
   private static final String XFORWARDED_ENABLED = GATEWAY_CONFIG_FILE_PREFIX + ".xforwarded.enabled";
   private static final String EPHEMERAL_DH_KEY_SIZE = GATEWAY_CONFIG_FILE_PREFIX + ".jdk.tls.ephemeralDHKeySize";
   private static final String HTTP_CLIENT_MAX_CONNECTION = GATEWAY_CONFIG_FILE_PREFIX + ".httpclient.maxConnections";
+  private static final String HTTP_CLIENT_CONNECTION_TIMEOUT = GATEWAY_CONFIG_FILE_PREFIX
+ ".httpclient.connectionTimeout";
+  private static final String HTTP_CLIENT_SOCKET_TIMEOUT = GATEWAY_CONFIG_FILE_PREFIX + ".httpclient.socketTimeout";
   private static final String THREAD_POOL_MAX = GATEWAY_CONFIG_FILE_PREFIX + ".threadpool.max";
   public static final String HTTP_SERVER_REQUEST_BUFFER = GATEWAY_CONFIG_FILE_PREFIX + ".httpserver.requestBuffer";
   public static final String HTTP_SERVER_REQUEST_HEADER_BUFFER = GATEWAY_CONFIG_FILE_PREFIX
+ ".httpserver.requestHeaderBuffer";
@@ -500,6 +502,34 @@ public class GatewayConfigImpl extends Configuration implements GatewayConfig
{
     return getInt( HTTP_CLIENT_MAX_CONNECTION, 32 );
   }
 
+  @Override
+  public int getHttpClientConnectionTimeout() {
+    int t = -1;
+    String s = get( HTTP_CLIENT_CONNECTION_TIMEOUT, null );
+    if ( s != null ) {
+      try {
+        t = (int)parseNetworkTimeout( s );
+      } catch ( Exception e ) {
+        // Ignore it and use the default.
+      }
+    }
+    return t;
+  }
+
+  @Override
+  public int getHttpClientSocketTimeout() {
+    int t = -1;
+    String s = get( HTTP_CLIENT_SOCKET_TIMEOUT, null );
+    if ( s != null ) {
+      try {
+        t = (int)parseNetworkTimeout( s );
+      } catch ( Exception e ) {
+        // Ignore it and use the default.
+      }
+    }
+    return t;
+  }
+
   /* (non-Javadoc)
    * @see org.apache.hadoop.gateway.config.GatewayConfig#getThreadPoolMax()
    */
@@ -572,4 +602,14 @@ public class GatewayConfigImpl extends Configuration implements GatewayConfig
{
   public String getSigningKeyAlias() {
     return get(SIGNING_KEY_ALIAS);
   }
+
+  private static long parseNetworkTimeout( String s ) {
+    PeriodFormatter f = new PeriodFormatterBuilder()
+        .appendMinutes().appendSuffix("m"," min")
+        .appendSeconds().appendSuffix("s"," sec")
+        .appendMillis().toFormatter();
+    Period p = Period.parse( s, f );
+    return p.toStandardDuration().getMillis();
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/knox/blob/8cb451da/gateway-spi/pom.xml
----------------------------------------------------------------------
diff --git a/gateway-spi/pom.xml b/gateway-spi/pom.xml
index 196515e..1da99d9 100644
--- a/gateway-spi/pom.xml
+++ b/gateway-spi/pom.xml
@@ -110,6 +110,10 @@
             <groupId>commons-collections</groupId>
             <artifactId>commons-collections</artifactId>
         </dependency>
+        <dependency>
+            <groupId>joda-time</groupId>
+            <artifactId>joda-time</artifactId>
+        </dependency>
 
         <dependency>
             <groupId>junit</groupId>

http://git-wip-us.apache.org/repos/asf/knox/blob/8cb451da/gateway-spi/src/main/java/org/apache/hadoop/gateway/config/GatewayConfig.java
----------------------------------------------------------------------
diff --git a/gateway-spi/src/main/java/org/apache/hadoop/gateway/config/GatewayConfig.java
b/gateway-spi/src/main/java/org/apache/hadoop/gateway/config/GatewayConfig.java
index 5558578..d830887 100644
--- a/gateway-spi/src/main/java/org/apache/hadoop/gateway/config/GatewayConfig.java
+++ b/gateway-spi/src/main/java/org/apache/hadoop/gateway/config/GatewayConfig.java
@@ -121,6 +121,10 @@ public interface GatewayConfig {
 
   int getHttpClientMaxConnections();
 
+  int getHttpClientConnectionTimeout();
+
+  int getHttpClientSocketTimeout();
+
   int getThreadPoolMax();
 
   int getHttpServerRequestBuffer();

http://git-wip-us.apache.org/repos/asf/knox/blob/8cb451da/gateway-spi/src/main/java/org/apache/hadoop/gateway/dispatch/DefaultDispatch.java
----------------------------------------------------------------------
diff --git a/gateway-spi/src/main/java/org/apache/hadoop/gateway/dispatch/DefaultDispatch.java
b/gateway-spi/src/main/java/org/apache/hadoop/gateway/dispatch/DefaultDispatch.java
index d1cef55..6c72fde 100644
--- a/gateway-spi/src/main/java/org/apache/hadoop/gateway/dispatch/DefaultDispatch.java
+++ b/gateway-spi/src/main/java/org/apache/hadoop/gateway/dispatch/DefaultDispatch.java
@@ -254,9 +254,11 @@ public class DefaultDispatch extends AbstractGatewayDispatch {
    @Override
    public void doGet(URI url, HttpServletRequest request, HttpServletResponse response)
          throws IOException, URISyntaxException {
-     HttpGet method = new HttpGet(url);
+      HttpGet method = new HttpGet(url);
       // https://issues.apache.org/jira/browse/KNOX-107 - Service URLs not rewritten for
WebHDFS GET redirects
-      method.getParams().setBooleanParameter("http.protocol.handle-redirects", false);
+      // This is now taken care of in DefaultHttpClientFactory.createHttpClient
+      // and setting params here causes configuration setup there to be ignored there.
+      // method.getParams().setBooleanParameter("http.protocol.handle-redirects", false);
       copyRequestHeaderFields(method, request);
       executeRequest(method, request, response);
    }

http://git-wip-us.apache.org/repos/asf/knox/blob/8cb451da/gateway-spi/src/main/java/org/apache/hadoop/gateway/dispatch/DefaultHttpClientFactory.java
----------------------------------------------------------------------
diff --git a/gateway-spi/src/main/java/org/apache/hadoop/gateway/dispatch/DefaultHttpClientFactory.java
b/gateway-spi/src/main/java/org/apache/hadoop/gateway/dispatch/DefaultHttpClientFactory.java
index 57de2e4..92652e2 100644
--- a/gateway-spi/src/main/java/org/apache/hadoop/gateway/dispatch/DefaultHttpClientFactory.java
+++ b/gateway-spi/src/main/java/org/apache/hadoop/gateway/dispatch/DefaultHttpClientFactory.java
@@ -30,17 +30,20 @@ import org.apache.http.client.HttpClient;
 import org.apache.http.client.HttpRequestRetryHandler;
 import org.apache.http.client.RedirectStrategy;
 import org.apache.http.client.config.AuthSchemes;
+import org.apache.http.client.config.RequestConfig;
 import org.apache.http.client.methods.HttpUriRequest;
 import org.apache.http.config.Registry;
 import org.apache.http.config.RegistryBuilder;
 import org.apache.http.cookie.Cookie;
 import org.apache.http.impl.DefaultConnectionReuseStrategy;
-import org.apache.http.impl.auth.SPNegoSchemeFactory;
 import org.apache.http.impl.client.BasicCredentialsProvider;
 import org.apache.http.impl.client.DefaultConnectionKeepAliveStrategy;
 import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.impl.client.HttpClients;
 import org.apache.http.protocol.HttpContext;
+import org.joda.time.Period;
+import org.joda.time.format.PeriodFormatter;
+import org.joda.time.format.PeriodFormatterBuilder;
 
 import javax.servlet.FilterConfig;
 import java.io.IOException;
@@ -72,15 +75,31 @@ public class DefaultHttpClientFactory implements HttpClientFactory {
 
     builder.setKeepAliveStrategy( DefaultConnectionKeepAliveStrategy.INSTANCE );
     builder.setConnectionReuseStrategy( DefaultConnectionReuseStrategy.INSTANCE );
+    builder.setRedirectStrategy( new NeverRedirectStrategy() );
+    builder.setRetryHandler( new NeverRetryHandler() );
 
     int maxConnections = getMaxConnections( filterConfig );
     builder.setMaxConnTotal( maxConnections );
     builder.setMaxConnPerRoute( maxConnections );
 
-    return builder
-        .setRedirectStrategy(new NeverRedirectStrategy())
-        .setRetryHandler(new NeverRetryHandler())
-        .build();
+    builder.setDefaultRequestConfig( getRequestConfig( filterConfig ) );
+
+    HttpClient client = builder.build();
+    return client;
+  }
+
+  private static RequestConfig getRequestConfig( FilterConfig config ) {
+    RequestConfig.Builder builder = RequestConfig.custom();
+    int connectionTimeout = getConnectionTimeout( config );
+    if ( connectionTimeout != -1 ) {
+      builder.setConnectTimeout( connectionTimeout );
+      builder.setConnectionRequestTimeout( connectionTimeout );
+    }
+    int socketTimeout = getSocketTimeout( config );
+    if( socketTimeout != -1 ) {
+      builder.setSocketTimeout( socketTimeout );
+    }
+    return builder.build();
   }
 
   private class NoCookieStore implements CookieStore {
@@ -156,4 +175,49 @@ public class DefaultHttpClientFactory implements HttpClientFactory {
     return maxConnections;
   }
 
+  private static int getConnectionTimeout( FilterConfig filterConfig ) {
+    int timeout = -1;
+    GatewayConfig globalConfig =
+        (GatewayConfig)filterConfig.getServletContext().getAttribute( GatewayConfig.GATEWAY_CONFIG_ATTRIBUTE
);
+    if( globalConfig != null ) {
+      timeout = globalConfig.getHttpClientConnectionTimeout();
+    }
+    String str = filterConfig.getInitParameter( "httpclient.connectionTimeout" );
+    if( str != null ) {
+      try {
+        timeout = (int)parseTimeout( str );
+      } catch ( Exception e ) {
+        // Ignore it and use the default.
+      }
+    }
+    return timeout;
+  }
+
+  private static int getSocketTimeout( FilterConfig filterConfig ) {
+    int timeout = -1;
+    GatewayConfig globalConfig =
+        (GatewayConfig)filterConfig.getServletContext().getAttribute( GatewayConfig.GATEWAY_CONFIG_ATTRIBUTE
);
+    if( globalConfig != null ) {
+      timeout = globalConfig.getHttpClientSocketTimeout();
+    }
+    String str = filterConfig.getInitParameter( "httpclient.socketTimeout" );
+    if( str != null ) {
+      try {
+        timeout = (int)parseTimeout( str );
+      } catch ( Exception e ) {
+        // Ignore it and use the default.
+      }
+    }
+    return timeout;
+  }
+
+  private static long parseTimeout( String s ) {
+    PeriodFormatter f = new PeriodFormatterBuilder()
+        .appendMinutes().appendSuffix("m"," min")
+        .appendSeconds().appendSuffix("s"," sec")
+        .appendMillis().toFormatter();
+    Period p = Period.parse( s, f );
+    return p.toStandardDuration().getMillis();
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/knox/blob/8cb451da/gateway-test-release/webhdfs-kerb-test/src/test/java/org/apache/hadoop/gateway/GatewayTestConfig.java
----------------------------------------------------------------------
diff --git a/gateway-test-release/webhdfs-kerb-test/src/test/java/org/apache/hadoop/gateway/GatewayTestConfig.java
b/gateway-test-release/webhdfs-kerb-test/src/test/java/org/apache/hadoop/gateway/GatewayTestConfig.java
index e5be4be..3c53597 100644
--- a/gateway-test-release/webhdfs-kerb-test/src/test/java/org/apache/hadoop/gateway/GatewayTestConfig.java
+++ b/gateway-test-release/webhdfs-kerb-test/src/test/java/org/apache/hadoop/gateway/GatewayTestConfig.java
@@ -285,6 +285,16 @@ public class GatewayTestConfig extends Configuration implements GatewayConfig
{
   }
 
   @Override
+  public int getHttpClientConnectionTimeout() {
+    return -1;
+  }
+
+  @Override
+  public int getHttpClientSocketTimeout() {
+    return -1;
+  }
+
+  @Override
   public int getThreadPoolMax() {
     return 16;
   }

http://git-wip-us.apache.org/repos/asf/knox/blob/8cb451da/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayTestConfig.java
----------------------------------------------------------------------
diff --git a/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayTestConfig.java b/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayTestConfig.java
index cd71e7f..4c75cb9 100644
--- a/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayTestConfig.java
+++ b/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayTestConfig.java
@@ -326,6 +326,16 @@ public class GatewayTestConfig extends Configuration implements GatewayConfig
{
   }
 
   @Override
+  public int getHttpClientConnectionTimeout() {
+    return -1;
+  }
+
+  @Override
+  public int getHttpClientSocketTimeout() {
+    return -1;
+  }
+
+  @Override
   public int getThreadPoolMax() {
     return 16;
   }


Mime
View raw message