SQOOP-1527: Sqoop2: Kerberos support (SPNEGO) in communication between server and client
(Richard Zhou via Abraham Elmahrek)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/1558d940
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/1558d940
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/1558d940
Branch: refs/heads/cdh5-1.99.4
Commit: 1558d940b880d95145943fd67d38bc316a781810
Parents: dc226c4
Author: Abraham Elmahrek <abraham@elmahrek.com>
Authored: Sun Nov 16 20:54:13 2014 -0800
Committer: Abraham Elmahrek <abraham@elmahrek.com>
Committed: Wed Nov 19 13:51:46 2014 -0800
----------------------------------------------------------------------
client/pom.xml | 11 +-
.../sqoop/client/request/ResourceRequest.java | 173 ++++++++++++-------
.../sqoop/security/AuthenticationConstants.java | 21 +++
.../sqoop/security/AuthenticationError.java | 8 +-
dist/src/main/server/conf/sqoop.properties | 4 +-
server/pom.xml | 49 ++++++
.../sqoop/filter/SqoopAuthenticationFilter.java | 80 +++++++++
server/src/main/webapp/WEB-INF/web.xml | 11 ++
shell/pom.xml | 9 +-
9 files changed, 292 insertions(+), 74 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/1558d940/client/pom.xml
----------------------------------------------------------------------
diff --git a/client/pom.xml b/client/pom.xml
index bbc436e..4f7aabd 100644
--- a/client/pom.xml
+++ b/client/pom.xml
@@ -51,9 +51,14 @@ limitations under the License.
<artifactId>sqoop-common</artifactId>
</dependency>
<dependency>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-client</artifactId>
- <version>1.11</version>
+ <groupId>javax.ws.rs</groupId>
+ <artifactId>javax.ws.rs-api</artifactId>
+ <version>2.0.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
+ <scope>provided</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/sqoop/blob/1558d940/client/src/main/java/org/apache/sqoop/client/request/ResourceRequest.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/sqoop/client/request/ResourceRequest.java b/client/src/main/java/org/apache/sqoop/client/request/ResourceRequest.java
index c84a83e..028444c 100644
--- a/client/src/main/java/org/apache/sqoop/client/request/ResourceRequest.java
+++ b/client/src/main/java/org/apache/sqoop/client/request/ResourceRequest.java
@@ -17,15 +17,9 @@
*/
package org.apache.sqoop.client.request;
-import javax.ws.rs.core.MediaType;
-
-import com.sun.jersey.api.client.Client;
-import com.sun.jersey.api.client.ClientRequest;
-import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.WebResource;
-import com.sun.jersey.api.client.WebResource.Builder;
-import com.sun.jersey.api.client.filter.ClientFilter;
-
+import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.log4j.Logger;
import org.apache.sqoop.client.ClientError;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.common.SqoopProtocolConstants;
@@ -33,81 +27,132 @@ import org.apache.sqoop.json.ThrowableBean;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
+import javax.ws.rs.HttpMethod;
+import javax.ws.rs.core.MediaType;
+import java.io.BufferedReader;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
+import java.net.URL;
import java.util.Locale;
+
/**
* Represents the sqoop REST resource requests
- *
*/
-public class ResourceRequest
-{
- private static ServerExceptionFilter serverExceptionFilter;
+public class ResourceRequest {
+ private static final Logger LOG = Logger.getLogger(ResourceRequest.class);
- static {
- serverExceptionFilter = new ServerExceptionFilter();
+ protected String doHttpRequest(String strURL, String method) {
+ return doHttpRequest(strURL, method, "");
}
- protected Builder getBuilder(String url) {
- Client client = Client.create();
- WebResource resource = client.resource(url);
+ protected String doHttpRequest(String strURL, String method, String data) {
+ DataOutputStream wr = null;
+ BufferedReader reader = null;
+ try {
+ AuthenticatedURL.Token token = new AuthenticatedURL.Token();
+ URL url = new URL(strURL);
+ HttpURLConnection conn = new AuthenticatedURL().openConnection(url, token);
+
+ conn.setRequestMethod(method);
+// Provide name of user executing request
+ conn.setRequestProperty(SqoopProtocolConstants.HEADER_SQOOP_USERNAME, System.getProperty("user.name"));
+// Sqoop is using JSON for data transfers
+ conn.setRequestProperty("Accept", MediaType.APPLICATION_JSON);
+// Transfer client locale to return client specific data
+ conn.setRequestProperty("Accept-Language", Locale.getDefault().toString());
+ if (method.equalsIgnoreCase(HttpMethod.PUT) || method.equalsIgnoreCase(HttpMethod.POST))
{
+ conn.setDoOutput(true);
+ data = data == null ? "" : data;
+ conn.setRequestProperty("Content-Length", Integer.toString(data.getBytes().length));
+// Send request
+ wr = new DataOutputStream(conn.getOutputStream());
+ wr.writeBytes(data);
+ wr.flush();
+ wr.close();
+ }
+
+ LOG.debug("Status code: " + conn.getResponseCode() + " " + conn.getResponseMessage());
+ StringBuilder result = new StringBuilder();
+ int responseCode = conn.getResponseCode();
- // Provide filter that will rebuild exception that is sent from server
- resource.addFilter(serverExceptionFilter);
+ if (responseCode == HttpURLConnection.HTTP_OK) {
+ reader = new BufferedReader(new InputStreamReader(conn.getInputStream()));
+ String line = reader.readLine();
+ while (line != null) {
+ result.append(line);
+ line = reader.readLine();
+ }
+ reader.close();
+ } else if (responseCode == HttpURLConnection.HTTP_INTERNAL_ERROR) {
+ /**
+ * Client filter to intercepting exceptions sent by sqoop server and
+ * recreating them on client side. Current implementation will create new
+ * instance of SqoopException and will attach original error code and message.
+ *
+ * Special handling for 500 internal server error in case that server
+ * has sent us it's exception correctly. We're using default route
+ * for all other 500 occurrences.
+ */
+ if (conn.getHeaderFields().keySet().contains(
+ SqoopProtocolConstants.HEADER_SQOOP_INTERNAL_ERROR_CODE)) {
- return resource
- // Provide name of user executing request.
- .header(SqoopProtocolConstants.HEADER_SQOOP_USERNAME, System.getProperty("user.name"))
- // Sqoop is using JSON for data transfers
- .accept(MediaType.APPLICATION_JSON_TYPE)
- // Transfer client locale to return client specific data
- .acceptLanguage(Locale.getDefault());
+ ThrowableBean ex = new ThrowableBean();
+
+ result = new StringBuilder();
+ reader = new BufferedReader(new InputStreamReader(conn.getErrorStream()));
+ String line = reader.readLine();
+ while (line != null) {
+ result.append(line);
+ line = reader.readLine();
+ }
+ reader.close();
+
+ JSONObject json = (JSONObject) JSONValue.parse(result.toString());
+ ex.restore(json);
+
+ throw new SqoopException(ClientError.CLIENT_0001, ex.getThrowable());
+ }
+ }
+ return result.toString();
+ } catch (IOException ex) {
+ LOG.trace("ERROR: ", ex);
+ return "";
+ } catch (AuthenticationException ex) {
+ LOG.trace("ERROR: ", ex);
+ return "";
+ } finally {
+ try {
+ if (wr != null) {
+ wr.close();
+ }
+ } catch (IOException e) {
+ LOG.trace("Cannot close DataOutputStream.", e);
+ }
+ try {
+ if (reader != null) {
+ reader.close();
+ }
+ } catch (IOException e) {
+ LOG.trace("Cannot close BufferReader.", e);
+ }
+ }
}
public String get(String url) {
- return getBuilder(url).get(String.class);
+ return doHttpRequest(url, HttpMethod.GET);
}
public String post(String url, String data) {
- return getBuilder(url).post(String.class, data);
+ return doHttpRequest(url, HttpMethod.POST, data);
}
public String put(String url, String data) {
- return getBuilder(url).put(String.class, data);
+ return doHttpRequest(url, HttpMethod.PUT, data);
}
public String delete(String url) {
- return getBuilder(url).delete(String.class);
- }
-
- /**
- * Client filter to intercepting exceptions sent by sqoop server and
- * recreating them on client side. Current implementation will create new
- * instance of SqoopException and will attach original error code and message.
- */
- private static class ServerExceptionFilter extends ClientFilter {
- @Override
- public ClientResponse handle(ClientRequest cr) {
- ClientResponse resp = getNext().handle(cr);
-
- // Special handling for 500 internal server error in case that server
- // has sent us it's exception correctly. We're using default route
- // for all other 500 occurrences.
- if(resp.getClientResponseStatus()
- == ClientResponse.Status.INTERNAL_SERVER_ERROR) {
-
- if(resp.getHeaders().containsKey(
- SqoopProtocolConstants.HEADER_SQOOP_INTERNAL_ERROR_CODE)) {
-
- ThrowableBean ex = new ThrowableBean();
-
- String responseText = resp.getEntity(String.class);
- JSONObject json = (JSONObject) JSONValue.parse(responseText);
- ex.restore(json);
-
- throw new SqoopException(ClientError.CLIENT_0001, ex.getThrowable());
- }
- }
-
- return resp;
- }
+ return doHttpRequest(url, HttpMethod.DELETE);
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/1558d940/core/src/main/java/org/apache/sqoop/security/AuthenticationConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/security/AuthenticationConstants.java b/core/src/main/java/org/apache/sqoop/security/AuthenticationConstants.java
index 645555f..26f83a9 100644
--- a/core/src/main/java/org/apache/sqoop/security/AuthenticationConstants.java
+++ b/core/src/main/java/org/apache/sqoop/security/AuthenticationConstants.java
@@ -68,6 +68,27 @@ public final class AuthenticationConstants {
public static final String AUTHENTICATION_KERBEROS_KEYTAB =
PREFIX_AUTHENTICATION_KERBEROS_CONFIG + "keytab";
+ /**
+ * All kerberos authentication for http related configuration is prefixed with this:
+ * <tt>org.apache.sqoop.authentication.kerberos.http.</tt>
+ */
+ public static final String PREFIX_AUTHENTICATION_KERBEROS_HTTP_CONFIG =
+ PREFIX_AUTHENTICATION_KERBEROS_CONFIG + "http.";
+
+ /**
+ * The config specifies the kerberos principal for http.
+ * <tt>org.apache.sqoop.authentication.kerberos.http.principal</tt>.
+ */
+ public static final String AUTHENTICATION_KERBEROS_HTTP_PRINCIPAL =
+ PREFIX_AUTHENTICATION_KERBEROS_HTTP_CONFIG + "principal";
+
+ /**
+ * The config specifies the kerberos keytab for http.
+ * <tt>org.apache.sqoop.authentication.kerberos.http.principal</tt>.
+ */
+ public static final String AUTHENTICATION_KERBEROS_HTTP_KEYTAB =
+ PREFIX_AUTHENTICATION_KERBEROS_HTTP_CONFIG + "keytab";
+
public static enum TYPE {SIMPLE, KERBEROS}
private AuthenticationConstants() {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/1558d940/core/src/main/java/org/apache/sqoop/security/AuthenticationError.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/security/AuthenticationError.java b/core/src/main/java/org/apache/sqoop/security/AuthenticationError.java
index 73cd8cf..abb5c90 100644
--- a/core/src/main/java/org/apache/sqoop/security/AuthenticationError.java
+++ b/core/src/main/java/org/apache/sqoop/security/AuthenticationError.java
@@ -34,7 +34,13 @@ public enum AuthenticationError implements ErrorCode {
AUTH_0003("Unable to login using Kerberos keytab and principal"),
/** Invalid authentication type {simple, Kerberos}. */
- AUTH_0004("Invalid authentication type");
+ AUTH_0004("Invalid authentication type"),
+
+ /** The system was not able to find Kerberos keytab for http in sqoop configuration. */
+ AUTH_0005("Unable to find Kerberos keytab for http"),
+
+ /** The system was not able to find Kerberos principal for http in sqoop configuration.
*/
+ AUTH_0006("Unable to find Kerberos principal for http");
private final String message;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/1558d940/dist/src/main/server/conf/sqoop.properties
----------------------------------------------------------------------
diff --git a/dist/src/main/server/conf/sqoop.properties b/dist/src/main/server/conf/sqoop.properties
index c5faeca..1666283 100755
--- a/dist/src/main/server/conf/sqoop.properties
+++ b/dist/src/main/server/conf/sqoop.properties
@@ -149,4 +149,6 @@ org.apache.sqoop.authentication.handler=org.apache.sqoop.security.SimpleAuthenti
#org.apache.sqoop.authentication.type=KERBEROS
#org.apache.sqoop.authentication.handler=org.apache.sqoop.security.KerberosAuthenticationHandler
#org.apache.sqoop.authentication.kerberos.principal=sqoop/_HOST@NOVALOCAL
-#org.apache.sqoop.authentication.kerberos.keytab=/home/kerberos/sqoop.keytab
\ No newline at end of file
+#org.apache.sqoop.authentication.kerberos.keytab=/home/kerberos/sqoop.keytab
+#org.apache.sqoop.authentication.kerberos.http.principal=HTTP/_HOST@NOVALOCAL
+#org.apache.sqoop.authentication.kerberos.http.keytab=/home/kerberos/sqoop.keytab
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/sqoop/blob/1558d940/server/pom.xml
----------------------------------------------------------------------
diff --git a/server/pom.xml b/server/pom.xml
index 2a3961c..7b093ac 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -94,6 +94,55 @@ limitations under the License.
</dependency>
</dependencies>
+ <!-- Profiles for various supported Hadoop distributions -->
+ <profiles>
+
+ <!-- Hadoop 1.x -->
+ <profile>
+ <id>hadoop100</id>
+
+ <activation>
+ <property>
+ <name>hadoop.profile</name>
+ <value>100</value>
+ </property>
+ </activation>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+ </profile>
+
+ <!-- Hadoop 2.x (active by default) -->
+ <profile>
+ <id>hadoop200</id>
+
+ <activation>
+ <activeByDefault>true</activeByDefault>
+ <property>
+ <name>hadoop.profile</name>
+ <value>200</value>
+ </property>
+ </activation>
+
+ <properties>
+ <hadoop.profile>200</hadoop.profile>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+ </profile>
+ </profiles>
+
<build>
<finalName>sqoop</finalName>
</build>
http://git-wip-us.apache.org/repos/asf/sqoop/blob/1558d940/server/src/main/java/org/apache/sqoop/filter/SqoopAuthenticationFilter.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/sqoop/filter/SqoopAuthenticationFilter.java b/server/src/main/java/org/apache/sqoop/filter/SqoopAuthenticationFilter.java
new file mode 100644
index 0000000..a6a79d3
--- /dev/null
+++ b/server/src/main/java/org/apache/sqoop/filter/SqoopAuthenticationFilter.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.filter;
+
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
+import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
+import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler;
+import org.apache.sqoop.common.MapContext;
+import org.apache.sqoop.common.SqoopException;
+import org.apache.sqoop.core.SqoopConfiguration;
+import org.apache.sqoop.security.AuthenticationConstants;
+import org.apache.sqoop.security.AuthenticationError;
+
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletException;
+import java.io.IOException;
+import java.util.Properties;
+
+public class SqoopAuthenticationFilter extends AuthenticationFilter {
+
+ @Override
+ protected Properties getConfiguration(String configPrefix,
+ FilterConfig filterConfig) throws ServletException
{
+ Properties properties = super.getConfiguration(configPrefix, filterConfig);
+ MapContext mapContext = SqoopConfiguration.getInstance().getContext();
+ String type = mapContext.getString(
+ AuthenticationConstants.AUTHENTICATION_TYPE).trim();
+
+ if (type.equalsIgnoreCase(AuthenticationConstants.TYPE.KERBEROS.name())) {
+ properties.setProperty(AUTH_TYPE, AuthenticationConstants.TYPE.KERBEROS.name().toLowerCase());
+
+ String keytab = mapContext.getString(
+ AuthenticationConstants.AUTHENTICATION_KERBEROS_HTTP_KEYTAB).trim();
+ if (keytab.length() == 0) {
+ throw new SqoopException(AuthenticationError.AUTH_0005,
+ AuthenticationConstants.AUTHENTICATION_KERBEROS_HTTP_KEYTAB);
+ }
+
+ String principal = mapContext.getString(
+ AuthenticationConstants.AUTHENTICATION_KERBEROS_HTTP_PRINCIPAL).trim();
+ if (principal.length() == 0) {
+ throw new SqoopException(AuthenticationError.AUTH_0006,
+ AuthenticationConstants.AUTHENTICATION_KERBEROS_HTTP_PRINCIPAL);
+ }
+
+ String hostPrincipal = "";
+ try {
+ hostPrincipal = SecurityUtil.getServerPrincipal(principal, "0.0.0.0");
+ } catch (IOException e) {
+ throw new SqoopException(AuthenticationError.AUTH_0006,
+ AuthenticationConstants.AUTHENTICATION_KERBEROS_HTTP_PRINCIPAL);
+ }
+
+ properties.setProperty(KerberosAuthenticationHandler.PRINCIPAL, hostPrincipal);
+ properties.setProperty(KerberosAuthenticationHandler.KEYTAB, keytab);
+ } else if (type.equalsIgnoreCase(AuthenticationConstants.TYPE.SIMPLE.name())) {
+ properties.setProperty(AUTH_TYPE, PseudoAuthenticationHandler.class.getName());
+ } else {
+ throw new SqoopException(AuthenticationError.AUTH_0004, type);
+ }
+
+ return properties;
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/1558d940/server/src/main/webapp/WEB-INF/web.xml
----------------------------------------------------------------------
diff --git a/server/src/main/webapp/WEB-INF/web.xml b/server/src/main/webapp/WEB-INF/web.xml
index d405c88..85ae26b 100644
--- a/server/src/main/webapp/WEB-INF/web.xml
+++ b/server/src/main/webapp/WEB-INF/web.xml
@@ -27,6 +27,17 @@ limitations under the License.
<listener-class>org.apache.sqoop.server.ServerInitializer</listener-class>
</listener>
+ <!-- Filter -->
+ <filter>
+ <filter-name>authFilter</filter-name>
+ <filter-class>org.apache.sqoop.filter.SqoopAuthenticationFilter</filter-class>
+ </filter>
+
+ <filter-mapping>
+ <filter-name>authFilter</filter-name>
+ <url-pattern>/*</url-pattern>
+ </filter-mapping>
+
<!-- Version servlet -->
<servlet>
<servlet-name>VersionServlet</servlet-name>
http://git-wip-us.apache.org/repos/asf/sqoop/blob/1558d940/shell/pom.xml
----------------------------------------------------------------------
diff --git a/shell/pom.xml b/shell/pom.xml
index ba268ce..0249e6e 100644
--- a/shell/pom.xml
+++ b/shell/pom.xml
@@ -56,11 +56,6 @@ limitations under the License.
<version>2.6</version>
</dependency>
<dependency>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-client</artifactId>
- <version>1.11</version>
- </dependency>
- <dependency>
<groupId>jline</groupId>
<artifactId>jline</artifactId>
<version>0.9.94</version>
@@ -75,6 +70,10 @@ limitations under the License.
<artifactId>groovy-all</artifactId>
<version>1.8.5</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
+ </dependency>
</dependencies>
<profiles>
|