sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a..@apache.org
Subject sqoop git commit: SQOOP-1527: Sqoop2: Kerberos support (SPNEGO) in communication between server and client
Date Mon, 17 Nov 2014 05:11:18 GMT
Repository: sqoop
Updated Branches:
  refs/heads/sqoop2 9f9ac6fe0 -> 423de5d51


SQOOP-1527: Sqoop2: Kerberos support (SPNEGO) in communication between server and client

(Richard Zhou via Abraham Elmahrek)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/423de5d5
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/423de5d5
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/423de5d5

Branch: refs/heads/sqoop2
Commit: 423de5d515545df2ca535d5ac5e84a4e9db30ba0
Parents: 9f9ac6f
Author: Abraham Elmahrek <abraham@elmahrek.com>
Authored: Sun Nov 16 20:54:13 2014 -0800
Committer: Abraham Elmahrek <abraham@elmahrek.com>
Committed: Sun Nov 16 20:54:13 2014 -0800

----------------------------------------------------------------------
 client/pom.xml                                  |  11 +-
 .../sqoop/client/request/ResourceRequest.java   | 173 ++++++++++++-------
 .../sqoop/security/AuthenticationConstants.java |  21 +++
 .../sqoop/security/AuthenticationError.java     |   8 +-
 dist/src/main/server/conf/sqoop.properties      |   4 +-
 server/pom.xml                                  |  49 ++++++
 .../sqoop/filter/SqoopAuthenticationFilter.java |  80 +++++++++
 server/src/main/webapp/WEB-INF/web.xml          |  11 ++
 shell/pom.xml                                   |   9 +-
 9 files changed, 292 insertions(+), 74 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/423de5d5/client/pom.xml
----------------------------------------------------------------------
diff --git a/client/pom.xml b/client/pom.xml
index b2e221e..b53c9d3 100644
--- a/client/pom.xml
+++ b/client/pom.xml
@@ -51,9 +51,14 @@ limitations under the License.
       <artifactId>sqoop-common</artifactId>
     </dependency>
     <dependency>
-      <groupId>com.sun.jersey</groupId>
-      <artifactId>jersey-client</artifactId>
-      <version>1.11</version>
+      <groupId>javax.ws.rs</groupId>
+      <artifactId>javax.ws.rs-api</artifactId>
+      <version>2.0.1</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-auth</artifactId>
+      <scope>provided</scope>
     </dependency>
   </dependencies>
 

http://git-wip-us.apache.org/repos/asf/sqoop/blob/423de5d5/client/src/main/java/org/apache/sqoop/client/request/ResourceRequest.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/sqoop/client/request/ResourceRequest.java b/client/src/main/java/org/apache/sqoop/client/request/ResourceRequest.java
index c84a83e..028444c 100644
--- a/client/src/main/java/org/apache/sqoop/client/request/ResourceRequest.java
+++ b/client/src/main/java/org/apache/sqoop/client/request/ResourceRequest.java
@@ -17,15 +17,9 @@
  */
 package org.apache.sqoop.client.request;
 
-import javax.ws.rs.core.MediaType;
-
-import com.sun.jersey.api.client.Client;
-import com.sun.jersey.api.client.ClientRequest;
-import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.WebResource;
-import com.sun.jersey.api.client.WebResource.Builder;
-import com.sun.jersey.api.client.filter.ClientFilter;
-
+import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.log4j.Logger;
 import org.apache.sqoop.client.ClientError;
 import org.apache.sqoop.common.SqoopException;
 import org.apache.sqoop.common.SqoopProtocolConstants;
@@ -33,81 +27,132 @@ import org.apache.sqoop.json.ThrowableBean;
 import org.json.simple.JSONObject;
 import org.json.simple.JSONValue;
 
+import javax.ws.rs.HttpMethod;
+import javax.ws.rs.core.MediaType;
+import java.io.BufferedReader;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
+import java.net.URL;
 import java.util.Locale;
+
 /**
  * Represents the sqoop REST resource requests
- *
  */
-public class ResourceRequest
-{
-  private static ServerExceptionFilter serverExceptionFilter;
+public class ResourceRequest {
+  private static final Logger LOG = Logger.getLogger(ResourceRequest.class);
 
-  static {
-    serverExceptionFilter = new ServerExceptionFilter();
+  protected String doHttpRequest(String strURL, String method) {
+    return doHttpRequest(strURL, method, "");
   }
 
-  protected Builder getBuilder(String url) {
-    Client client = Client.create();
-    WebResource resource = client.resource(url);
+  protected String doHttpRequest(String strURL, String method, String data) {
+    DataOutputStream wr = null;
+    BufferedReader reader = null;
+    try {
+      AuthenticatedURL.Token token = new AuthenticatedURL.Token();
+      URL url = new URL(strURL);
+      HttpURLConnection conn = new AuthenticatedURL().openConnection(url, token);
+
+      conn.setRequestMethod(method);
+//      Provide name of user executing request
+      conn.setRequestProperty(SqoopProtocolConstants.HEADER_SQOOP_USERNAME, System.getProperty("user.name"));
+//      Sqoop is using JSON for data transfers
+      conn.setRequestProperty("Accept", MediaType.APPLICATION_JSON);
+//      Transfer client locale to return client specific data
+      conn.setRequestProperty("Accept-Language", Locale.getDefault().toString());
+      if (method.equalsIgnoreCase(HttpMethod.PUT) || method.equalsIgnoreCase(HttpMethod.POST))
{
+        conn.setDoOutput(true);
+        data = data == null ? "" : data;
+        conn.setRequestProperty("Content-Length", Integer.toString(data.getBytes().length));
+//        Send request
+        wr = new DataOutputStream(conn.getOutputStream());
+        wr.writeBytes(data);
+        wr.flush();
+        wr.close();
+      }
+
+      LOG.debug("Status code: " + conn.getResponseCode() + " " + conn.getResponseMessage());
+      StringBuilder result = new StringBuilder();
+      int responseCode = conn.getResponseCode();
 
-    // Provide filter that will rebuild exception that is sent from server
-    resource.addFilter(serverExceptionFilter);
+      if (responseCode == HttpURLConnection.HTTP_OK) {
+        reader = new BufferedReader(new InputStreamReader(conn.getInputStream()));
+        String line = reader.readLine();
+        while (line != null) {
+          result.append(line);
+          line = reader.readLine();
+        }
+        reader.close();
+      } else if (responseCode == HttpURLConnection.HTTP_INTERNAL_ERROR) {
+        /**
+         * Client filter to intercepting exceptions sent by sqoop server and
+         * recreating them on client side. Current implementation will create new
+         * instance of SqoopException and will attach original error code and message.
+         *
+         * Special handling for 500 internal server error in case that server
+         * has sent us it's exception correctly. We're using default route
+         * for all other 500 occurrences.
+         */
+        if (conn.getHeaderFields().keySet().contains(
+                SqoopProtocolConstants.HEADER_SQOOP_INTERNAL_ERROR_CODE)) {
 
-    return resource
-    // Provide name of user executing request.
-        .header(SqoopProtocolConstants.HEADER_SQOOP_USERNAME, System.getProperty("user.name"))
-        // Sqoop is using JSON for data transfers
-        .accept(MediaType.APPLICATION_JSON_TYPE)
-        // Transfer client locale to return client specific data
-        .acceptLanguage(Locale.getDefault());
+          ThrowableBean ex = new ThrowableBean();
+
+          result = new StringBuilder();
+          reader = new BufferedReader(new InputStreamReader(conn.getErrorStream()));
+          String line = reader.readLine();
+          while (line != null) {
+            result.append(line);
+            line = reader.readLine();
+          }
+          reader.close();
+
+          JSONObject json = (JSONObject) JSONValue.parse(result.toString());
+          ex.restore(json);
+
+          throw new SqoopException(ClientError.CLIENT_0001, ex.getThrowable());
+        }
+      }
+      return result.toString();
+    } catch (IOException ex) {
+      LOG.trace("ERROR: ", ex);
+      return "";
+    } catch (AuthenticationException ex) {
+      LOG.trace("ERROR: ", ex);
+      return "";
+    } finally {
+      try {
+        if (wr != null) {
+          wr.close();
+        }
+      } catch (IOException e) {
+        LOG.trace("Cannot close DataOutputStream.", e);
+      }
+      try {
+        if (reader != null) {
+          reader.close();
+        }
+      } catch (IOException e) {
+        LOG.trace("Cannot close BufferReader.", e);
+      }
+    }
   }
 
   public String get(String url) {
-    return getBuilder(url).get(String.class);
+    return doHttpRequest(url, HttpMethod.GET);
   }
 
   public String post(String url, String data) {
-    return getBuilder(url).post(String.class, data);
+    return doHttpRequest(url, HttpMethod.POST, data);
   }
 
   public String put(String url, String data) {
-    return getBuilder(url).put(String.class, data);
+    return doHttpRequest(url, HttpMethod.PUT, data);
   }
 
   public String delete(String url) {
-    return getBuilder(url).delete(String.class);
-  }
-
-  /**
-   * Client filter to intercepting exceptions sent by sqoop server and
-   * recreating them on client side. Current implementation will create new
-   * instance of SqoopException and will attach original error code and message.
-   */
-  private static class ServerExceptionFilter extends ClientFilter {
-    @Override
-    public ClientResponse handle(ClientRequest cr) {
-      ClientResponse resp = getNext().handle(cr);
-
-      // Special handling for 500 internal server error in case that server
-      // has sent us it's exception correctly. We're using default route
-      // for all other 500 occurrences.
-      if(resp.getClientResponseStatus()
-        == ClientResponse.Status.INTERNAL_SERVER_ERROR) {
-
-        if(resp.getHeaders().containsKey(
-          SqoopProtocolConstants.HEADER_SQOOP_INTERNAL_ERROR_CODE)) {
-
-          ThrowableBean ex = new ThrowableBean();
-
-          String responseText = resp.getEntity(String.class);
-          JSONObject json = (JSONObject) JSONValue.parse(responseText);
-          ex.restore(json);
-
-          throw new SqoopException(ClientError.CLIENT_0001, ex.getThrowable());
-        }
-      }
-
-      return resp;
-    }
+    return doHttpRequest(url, HttpMethod.DELETE);
   }
 }

http://git-wip-us.apache.org/repos/asf/sqoop/blob/423de5d5/core/src/main/java/org/apache/sqoop/security/AuthenticationConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/security/AuthenticationConstants.java b/core/src/main/java/org/apache/sqoop/security/AuthenticationConstants.java
index 645555f..26f83a9 100644
--- a/core/src/main/java/org/apache/sqoop/security/AuthenticationConstants.java
+++ b/core/src/main/java/org/apache/sqoop/security/AuthenticationConstants.java
@@ -68,6 +68,27 @@ public final class AuthenticationConstants {
   public static final String AUTHENTICATION_KERBEROS_KEYTAB =
     PREFIX_AUTHENTICATION_KERBEROS_CONFIG + "keytab";
 
+  /**
+   * All kerberos authentication for http related configuration is prefixed with this:
+   * <tt>org.apache.sqoop.authentication.kerberos.http.</tt>
+   */
+  public static final String PREFIX_AUTHENTICATION_KERBEROS_HTTP_CONFIG =
+          PREFIX_AUTHENTICATION_KERBEROS_CONFIG + "http.";
+
+  /**
+   * The config specifies the kerberos principal for http.
+   * <tt>org.apache.sqoop.authentication.kerberos.http.principal</tt>.
+   */
+  public static final String AUTHENTICATION_KERBEROS_HTTP_PRINCIPAL =
+          PREFIX_AUTHENTICATION_KERBEROS_HTTP_CONFIG + "principal";
+
+  /**
+   * The config specifies the kerberos keytab for http.
+   * <tt>org.apache.sqoop.authentication.kerberos.http.principal</tt>.
+   */
+  public static final String AUTHENTICATION_KERBEROS_HTTP_KEYTAB =
+          PREFIX_AUTHENTICATION_KERBEROS_HTTP_CONFIG + "keytab";
+
   public static enum TYPE {SIMPLE, KERBEROS}
 
   private AuthenticationConstants() {

http://git-wip-us.apache.org/repos/asf/sqoop/blob/423de5d5/core/src/main/java/org/apache/sqoop/security/AuthenticationError.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/security/AuthenticationError.java b/core/src/main/java/org/apache/sqoop/security/AuthenticationError.java
index 73cd8cf..abb5c90 100644
--- a/core/src/main/java/org/apache/sqoop/security/AuthenticationError.java
+++ b/core/src/main/java/org/apache/sqoop/security/AuthenticationError.java
@@ -34,7 +34,13 @@ public enum AuthenticationError implements ErrorCode {
   AUTH_0003("Unable to login using Kerberos keytab and principal"),
 
   /** Invalid authentication type {simple, Kerberos}. */
-  AUTH_0004("Invalid authentication type");
+  AUTH_0004("Invalid authentication type"),
+
+  /** The system was not able to find Kerberos keytab for http in sqoop configuration. */
+  AUTH_0005("Unable to find Kerberos keytab for http"),
+
+  /** The system was not able to find Kerberos principal for http in sqoop configuration.
*/
+  AUTH_0006("Unable to find Kerberos principal for http");
 
   private final String message;
 

http://git-wip-us.apache.org/repos/asf/sqoop/blob/423de5d5/dist/src/main/server/conf/sqoop.properties
----------------------------------------------------------------------
diff --git a/dist/src/main/server/conf/sqoop.properties b/dist/src/main/server/conf/sqoop.properties
index c5faeca..1666283 100755
--- a/dist/src/main/server/conf/sqoop.properties
+++ b/dist/src/main/server/conf/sqoop.properties
@@ -149,4 +149,6 @@ org.apache.sqoop.authentication.handler=org.apache.sqoop.security.SimpleAuthenti
 #org.apache.sqoop.authentication.type=KERBEROS
 #org.apache.sqoop.authentication.handler=org.apache.sqoop.security.KerberosAuthenticationHandler
 #org.apache.sqoop.authentication.kerberos.principal=sqoop/_HOST@NOVALOCAL
-#org.apache.sqoop.authentication.kerberos.keytab=/home/kerberos/sqoop.keytab
\ No newline at end of file
+#org.apache.sqoop.authentication.kerberos.keytab=/home/kerberos/sqoop.keytab
+#org.apache.sqoop.authentication.kerberos.http.principal=HTTP/_HOST@NOVALOCAL
+#org.apache.sqoop.authentication.kerberos.http.keytab=/home/kerberos/sqoop.keytab
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/sqoop/blob/423de5d5/server/pom.xml
----------------------------------------------------------------------
diff --git a/server/pom.xml b/server/pom.xml
index 4a5eb5e..559e389 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -99,6 +99,55 @@ limitations under the License.
     </dependency>
   </dependencies>
 
+  <!-- Profiles for various supported Hadoop distributions -->
+  <profiles>
+
+    <!-- Hadoop 1.x -->
+    <profile>
+      <id>hadoop100</id>
+
+      <activation>
+        <property>
+          <name>hadoop.profile</name>
+          <value>100</value>
+        </property>
+      </activation>
+
+      <dependencies>
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-core</artifactId>
+          <scope>provided</scope>
+        </dependency>
+      </dependencies>
+    </profile>
+
+    <!-- Hadoop 2.x (active by default) -->
+    <profile>
+      <id>hadoop200</id>
+
+      <activation>
+        <activeByDefault>true</activeByDefault>
+        <property>
+          <name>hadoop.profile</name>
+          <value>200</value>
+        </property>
+      </activation>
+
+      <properties>
+        <hadoop.profile>200</hadoop.profile>
+      </properties>
+
+      <dependencies>
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-common</artifactId>
+          <scope>provided</scope>
+        </dependency>
+      </dependencies>
+    </profile>
+  </profiles>
+
   <build>
     <finalName>sqoop</finalName>
   </build>

http://git-wip-us.apache.org/repos/asf/sqoop/blob/423de5d5/server/src/main/java/org/apache/sqoop/filter/SqoopAuthenticationFilter.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/sqoop/filter/SqoopAuthenticationFilter.java b/server/src/main/java/org/apache/sqoop/filter/SqoopAuthenticationFilter.java
new file mode 100644
index 0000000..a6a79d3
--- /dev/null
+++ b/server/src/main/java/org/apache/sqoop/filter/SqoopAuthenticationFilter.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.filter;
+
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
+import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
+import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler;
+import org.apache.sqoop.common.MapContext;
+import org.apache.sqoop.common.SqoopException;
+import org.apache.sqoop.core.SqoopConfiguration;
+import org.apache.sqoop.security.AuthenticationConstants;
+import org.apache.sqoop.security.AuthenticationError;
+
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletException;
+import java.io.IOException;
+import java.util.Properties;
+
+public class SqoopAuthenticationFilter extends AuthenticationFilter {
+
+  @Override
+  protected Properties getConfiguration(String configPrefix,
+                                        FilterConfig filterConfig) throws ServletException
{
+    Properties properties = super.getConfiguration(configPrefix, filterConfig);
+    MapContext mapContext = SqoopConfiguration.getInstance().getContext();
+    String type = mapContext.getString(
+            AuthenticationConstants.AUTHENTICATION_TYPE).trim();
+
+    if (type.equalsIgnoreCase(AuthenticationConstants.TYPE.KERBEROS.name())) {
+      properties.setProperty(AUTH_TYPE, AuthenticationConstants.TYPE.KERBEROS.name().toLowerCase());
+
+      String keytab = mapContext.getString(
+              AuthenticationConstants.AUTHENTICATION_KERBEROS_HTTP_KEYTAB).trim();
+      if (keytab.length() == 0) {
+        throw new SqoopException(AuthenticationError.AUTH_0005,
+                AuthenticationConstants.AUTHENTICATION_KERBEROS_HTTP_KEYTAB);
+      }
+
+      String principal = mapContext.getString(
+              AuthenticationConstants.AUTHENTICATION_KERBEROS_HTTP_PRINCIPAL).trim();
+      if (principal.length() == 0) {
+        throw new SqoopException(AuthenticationError.AUTH_0006,
+                AuthenticationConstants.AUTHENTICATION_KERBEROS_HTTP_PRINCIPAL);
+      }
+
+      String hostPrincipal = "";
+      try {
+        hostPrincipal = SecurityUtil.getServerPrincipal(principal, "0.0.0.0");
+      } catch (IOException e) {
+        throw new SqoopException(AuthenticationError.AUTH_0006,
+                AuthenticationConstants.AUTHENTICATION_KERBEROS_HTTP_PRINCIPAL);
+      }
+
+      properties.setProperty(KerberosAuthenticationHandler.PRINCIPAL, hostPrincipal);
+      properties.setProperty(KerberosAuthenticationHandler.KEYTAB, keytab);
+    } else if (type.equalsIgnoreCase(AuthenticationConstants.TYPE.SIMPLE.name())) {
+      properties.setProperty(AUTH_TYPE, PseudoAuthenticationHandler.class.getName());
+    } else {
+      throw new SqoopException(AuthenticationError.AUTH_0004, type);
+    }
+
+    return properties;
+  }
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/423de5d5/server/src/main/webapp/WEB-INF/web.xml
----------------------------------------------------------------------
diff --git a/server/src/main/webapp/WEB-INF/web.xml b/server/src/main/webapp/WEB-INF/web.xml
index d405c88..85ae26b 100644
--- a/server/src/main/webapp/WEB-INF/web.xml
+++ b/server/src/main/webapp/WEB-INF/web.xml
@@ -27,6 +27,17 @@ limitations under the License.
     <listener-class>org.apache.sqoop.server.ServerInitializer</listener-class>
   </listener>
 
+  <!-- Filter -->
+  <filter>
+    <filter-name>authFilter</filter-name>
+    <filter-class>org.apache.sqoop.filter.SqoopAuthenticationFilter</filter-class>
+  </filter>
+
+  <filter-mapping>
+    <filter-name>authFilter</filter-name>
+    <url-pattern>/*</url-pattern>
+  </filter-mapping>
+
   <!-- Version servlet -->
   <servlet>
     <servlet-name>VersionServlet</servlet-name>

http://git-wip-us.apache.org/repos/asf/sqoop/blob/423de5d5/shell/pom.xml
----------------------------------------------------------------------
diff --git a/shell/pom.xml b/shell/pom.xml
index 947eab0..2092690 100644
--- a/shell/pom.xml
+++ b/shell/pom.xml
@@ -56,11 +56,6 @@ limitations under the License.
       <version>2.6</version>
     </dependency>
     <dependency>
-      <groupId>com.sun.jersey</groupId>
-      <artifactId>jersey-client</artifactId>
-      <version>1.11</version>
-    </dependency>
-    <dependency>
       <groupId>jline</groupId>
       <artifactId>jline</artifactId>
       <version>0.9.94</version>
@@ -75,6 +70,10 @@ limitations under the License.
       <artifactId>groovy-all</artifactId>
       <version>1.8.5</version>
     </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-auth</artifactId>
+    </dependency>
   </dependencies>
 
   <profiles>


Mime
View raw message