sentry-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sp...@apache.org
Subject sentry git commit: SENTRY-2329: Integrate sentry with Hadoop 3.1.1 (kalyan kumar kalvagadda, reviewed by Sergio Pena)
Date Mon, 05 Nov 2018 20:00:38 GMT
Repository: sentry
Updated Branches:
  refs/heads/master 5d4465284 -> d4c622845


SENTRY-2329: Integrate sentry with Hadoop 3.1.1 (kalyan kumar kalvagadda, reviewed by Sergio
Pena)

This integration is compatible with old Hadoop 2.x versions. You can compile with Hadoop 2.x
by
changing the pom.xml or you can add the Sentry/HDFS binding jars built with Hadoop3 in the
Hadoop 2 classpath. Both ways are verified and working.


Project: http://git-wip-us.apache.org/repos/asf/sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/d4c62284
Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/d4c62284
Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/d4c62284

Branch: refs/heads/master
Commit: d4c622845b9d32f214d1704e81fe443ce8c63290
Parents: 5d44652
Author: Sergio Pena <sergio.pena@cloudera.com>
Authored: Mon Nov 5 13:58:48 2018 -0600
Committer: Sergio Pena <sergio.pena@cloudera.com>
Committed: Mon Nov 5 13:58:48 2018 -0600

----------------------------------------------------------------------
 pom.xml                                         | 22 +++++++++++++++++++-
 sentry-binding/sentry-binding-solr/pom.xml      | 16 ++++++++++++++
 sentry-hdfs/sentry-hdfs-common/pom.xml          |  6 ++++++
 .../org/apache/sentry/hdfs/PathsUpdate.java     |  8 +++----
 .../hdfs/SentryINodeAttributesProvider.java     |  5 +++++
 .../thrift/TestSentryWebServerWithKerberos.java | 21 ++++++++++++++++---
 6 files changed, 69 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sentry/blob/d4c62284/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index acbdcc2..46ca38e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -70,7 +70,6 @@ limitations under the License.
     <easymock.version>3.0</easymock.version>
     <fest.reflect.version>1.4.1</fest.reflect.version>
     <guava.version>14.0.1</guava.version>
-    <hadoop.version>2.7.5</hadoop.version>
     <hamcrest.version>1.3</hamcrest.version>
     <hive.version>2.3.3</hive.version>
     <jackson.version>1.9.13</jackson.version>
@@ -101,6 +100,10 @@ limitations under the License.
     <test.sentry.hadoop.classpath>${maven.test.classpath}</test.sentry.hadoop.classpath>
     <zookeeper.version>3.4.5</zookeeper.version>
     <maven.jar.plugin.version>3.0.2</maven.jar.plugin.version>
+    <httpcomponents.version>4.5.3</httpcomponents.version>
+
+    <!-- Package versions for Sentry binding components -->
+    <hadoop.version>3.1.1</hadoop.version>
 
     <!-- Datanucleus package versions -->
     <datanucleus.maven.plugin.version>4.0.5</datanucleus.maven.plugin.version>
@@ -197,6 +200,7 @@ limitations under the License.
         <groupId>org.apache.hadoop</groupId>
         <artifactId>hadoop-minicluster</artifactId>
         <version>${hadoop.version}</version>
+        <scope>test</scope>
         <exclusions>
           <exclusion>
             <artifactId>curator-client</artifactId>
@@ -410,6 +414,10 @@ limitations under the License.
             <artifactId>jackson-mapper-asl</artifactId>
             <groupId>org.codehaus.jackson</groupId>
           </exclusion>
+          <exclusion>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-archives</artifactId>
+          </exclusion>
         </exclusions>
       </dependency>
       <dependency>
@@ -521,6 +529,10 @@ limitations under the License.
             <artifactId>jackson-xc</artifactId>
             <groupId>org.codehaus.jackson</groupId>
           </exclusion>
+          <exclusion>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
+          </exclusion>
         </exclusions>
       </dependency>
       <dependency>
@@ -608,6 +620,10 @@ limitations under the License.
             <artifactId>jackson-xc</artifactId>
             <groupId>org.codehaus.jackson</groupId>
           </exclusion>
+          <exclusion>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-yarn-registry</artifactId>
+          </exclusion>
         </exclusions>
       </dependency>
       <dependency>
@@ -945,6 +961,10 @@ limitations under the License.
             <artifactId>jackson-xc</artifactId>
             <groupId>org.codehaus.jackson</groupId>
           </exclusion>
+          <exclusion>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-archives</artifactId>
+          </exclusion>
         </exclusions>
       </dependency>
     </dependencies>

http://git-wip-us.apache.org/repos/asf/sentry/blob/d4c62284/sentry-binding/sentry-binding-solr/pom.xml
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-solr/pom.xml b/sentry-binding/sentry-binding-solr/pom.xml
index f2a5fca..97b1879 100644
--- a/sentry-binding/sentry-binding-solr/pom.xml
+++ b/sentry-binding/sentry-binding-solr/pom.xml
@@ -29,6 +29,22 @@ limitations under the License.
   <name>Sentry Binding for Solr</name>
 
   <dependencies>
+    <!-- This jetty dependency must be added before the solr-core dependency to avoid
conflicting
+         other jetty dependencies -->
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-server</artifactId>
+      <version>${jetty.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <!-- This jetty dependency must be added before the solr-core dependency to avoid
conflicting
+         other jetty dependencies -->
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-webapp</artifactId>
+      <version>${jetty.version}</version>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>org.apache.solr</groupId>
       <artifactId>solr-core</artifactId>

http://git-wip-us.apache.org/repos/asf/sentry/blob/d4c62284/sentry-hdfs/sentry-hdfs-common/pom.xml
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-common/pom.xml b/sentry-hdfs/sentry-hdfs-common/pom.xml
index df6f04c..4d70d13 100644
--- a/sentry-hdfs/sentry-hdfs-common/pom.xml
+++ b/sentry-hdfs/sentry-hdfs-common/pom.xml
@@ -80,6 +80,12 @@ limitations under the License.
       <artifactId>mockito-all</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.httpcomponents</groupId>
+      <artifactId>httpclient</artifactId>
+      <version>${httpcomponents.version}</version>
+      <scope>provided</scope>
+    </dependency>
 
   </dependencies>
   <build>

http://git-wip-us.apache.org/repos/asf/sentry/blob/d4c62284/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java
b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java
index c9ecc40..edc3afc 100644
--- a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java
+++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java
@@ -26,11 +26,10 @@ import java.util.List;
 import com.google.common.annotations.VisibleForTesting;
 import org.apache.sentry.hdfs.service.thrift.TPathChanges;
 import org.apache.sentry.hdfs.service.thrift.TPathsUpdate;
-import org.apache.commons.httpclient.util.URIUtil;
-import org.apache.commons.httpclient.URIException;
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.http.client.utils.URIBuilder;
 
 import org.apache.thrift.TException;
 
@@ -137,11 +136,10 @@ public class PathsUpdate implements Updateable.Update {
 
     URI uri;
     try {
-      uri = new URI(URIUtil.encodePath(path));
+      // Below converts unescaped path to escaped string and the constructs a URI from it.
+      uri = new URI(StringUtils.stripStart(new URIBuilder().setPath(path).toString(), "/"));
     } catch (URISyntaxException e) {
       throw new SentryMalformedPathException("Incomprehensible path [" + path + "]", e);
-    } catch (URIException e) {
-      throw new SentryMalformedPathException("Unable to create URI from path[" + path + "]",
e);
     }
 
     String scheme = uri.getScheme();

http://git-wip-us.apache.org/repos/asf/sentry/blob/d4c62284/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryINodeAttributesProvider.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryINodeAttributesProvider.java
b/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryINodeAttributesProvider.java
index 18b6265..4d2ef00 100644
--- a/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryINodeAttributesProvider.java
+++ b/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryINodeAttributesProvider.java
@@ -331,6 +331,11 @@ public class SentryINodeAttributesProvider extends INodeAttributeProvider
   public INodeAttributes getAttributes(String[] pathElements,
                                        INodeAttributes inode) {
     Preconditions.checkNotNull(pathElements);
+    
+    if (pathElements.length == 0) {
+      return inode;
+    }
+
     pathElements = "".equals(pathElements[0]) && pathElements.length > 1 ?
             Arrays.copyOfRange(pathElements, 1, pathElements.length) :
             pathElements;

http://git-wip-us.apache.org/repos/asf/sentry/blob/d4c62284/sentry-service/sentry-service-server/src/test/java/org/apache/sentry/api/service/thrift/TestSentryWebServerWithKerberos.java
----------------------------------------------------------------------
diff --git a/sentry-service/sentry-service-server/src/test/java/org/apache/sentry/api/service/thrift/TestSentryWebServerWithKerberos.java
b/sentry-service/sentry-service-server/src/test/java/org/apache/sentry/api/service/thrift/TestSentryWebServerWithKerberos.java
index 5d94d4b..add2c28 100644
--- a/sentry-service/sentry-service-server/src/test/java/org/apache/sentry/api/service/thrift/TestSentryWebServerWithKerberos.java
+++ b/sentry-service/sentry-service-server/src/test/java/org/apache/sentry/api/service/thrift/TestSentryWebServerWithKerberos.java
@@ -88,7 +88,7 @@ public class TestSentryWebServerWithKerberos extends SentryServiceIntegrationBas
       new AuthenticatedURL(new KerberosAuthenticator()).openConnection(url, new AuthenticatedURL.Token());
       fail("Here should fail.");
     } catch (Exception e) {
-      boolean isExpectError = e.getMessage().contains("No valid credentials provided");
+      boolean isExpectError = exceptionContainsMessage(e,"No valid credentials provided");
       Assert.assertTrue("Here should fail by 'No valid credentials provided'," +
           " but the exception is:" + e, isExpectError);
     }
@@ -124,7 +124,7 @@ public class TestSentryWebServerWithKerberos extends SentryServiceIntegrationBas
           fail("Here should fail.");
         } catch (AuthenticationException e) {
           String expectedError = "status code: 403";
-          if (!e.getMessage().contains(expectedError)) {
+          if (!exceptionContainsMessage(e, expectedError)) {
             LOG.error("UnexpectedError: " + e.getMessage(), e);
             fail("UnexpectedError: " + e.getMessage());
           }
@@ -155,7 +155,7 @@ public class TestSentryWebServerWithKerberos extends SentryServiceIntegrationBas
           fail("Login with user1 should fail");
         } catch (AuthenticationException e) {
           String expectedError = "status code: 403";
-          if (!e.getMessage().contains(expectedError)) {
+          if (!exceptionContainsMessage(e, expectedError)) {
             LOG.error("UnexpectedError: " + e.getMessage(), e);
             fail("UnexpectedError: " + e.getMessage());
           }
@@ -172,4 +172,19 @@ public class TestSentryWebServerWithKerberos extends SentryServiceIntegrationBas
     conn.setRequestMethod("TRACE");
     Assert.assertEquals(HttpURLConnection.HTTP_FORBIDDEN, conn.getResponseCode());
   }
+
+  /*
+   * Check if the exception contains the specified message in any of the exception message
+   * or cause message.
+   *
+   * <p/>i.e. Hadoop 2.x has a 'no valid privileges' message in the e.getMessage()
whereas
+   * Hadoop 3.x has the 'no valid privileges' message in the e.getCause().getMessage().
+   */
+  private boolean exceptionContainsMessage(Exception e, String message) {
+    if (e.getMessage().contains(message)) {
+      return true;
+    }
+
+    return e.getCause().getMessage().contains(message);
+  }
 }


Mime
View raw message