sentry-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ann...@apache.org
Subject sentry git commit: SENTRY-1447: When s3 is configured as HDFS defaultFS and Hive Warehouse Dir, need to fix some e2e test failures. For example, TestDbHdfsMaxGroups.java. (Anne Yu, reviewed by Hao Hao). [Forced Update!]
Date Thu, 25 Aug 2016 05:17:29 GMT
Repository: sentry
Updated Branches:
  refs/heads/master aa87868ff -> 53dc2d6f4 (forced update)


SENTRY-1447: When s3 is configured as HDFS defaultFS and Hive Warehouse Dir, need to fix some
e2e test failures. For example, TestDbHdfsMaxGroups.java. (Anne Yu, reviewed by Hao Hao).


Project: http://git-wip-us.apache.org/repos/asf/sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/53dc2d6f
Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/53dc2d6f
Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/53dc2d6f

Branch: refs/heads/master
Commit: 53dc2d6f41633b55711f0aff8d5e4c9be301255d
Parents: f45727a
Author: Anne Yu <anneyu@cloudera.com>
Authored: Tue Aug 23 12:16:41 2016 -0700
Committer: Anne Yu <anneyu@cloudera.com>
Committed: Wed Aug 24 22:55:36 2016 -0700

----------------------------------------------------------------------
 sentry-tests/sentry-tests-hive/pom.xml          |   1 +
 .../e2e/dbprovider/TestColumnEndToEnd.java      |  62 ---------
 .../e2e/dbprovider/TestDbCrossOperations.java   | 130 +++++++++++++++++++
 .../sentry/tests/e2e/hdfs/TestDbHdfsBase.java   |   6 +
 .../e2e/hive/TestExportImportPrivileges.java    |   4 +
 .../tests/e2e/hive/TestOperationsPart2.java     |   5 +
 6 files changed, 146 insertions(+), 62 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sentry/blob/53dc2d6f/sentry-tests/sentry-tests-hive/pom.xml
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/pom.xml b/sentry-tests/sentry-tests-hive/pom.xml
index 2979b45..a2512ee 100644
--- a/sentry-tests/sentry-tests-hive/pom.xml
+++ b/sentry-tests/sentry-tests-hive/pom.xml
@@ -491,6 +491,7 @@ limitations under the License.
           <include>**/TestDbHdfsExtMaxGroups.java</include>
           <include>**/TestHiveWarehouseOnExtFs.java</include>
           <include>**/TestTableOnExtFS.java</include>
+          <include>**/TestDbCrossOperations.java</include>
         </includes>
         <argLine>-Dsentry.e2etest.hiveServer2Type=UnmanagedHiveServer2 -Dsentry.e2etest.DFSType=ClusterDFS
-Dsentry.e2etest.external.sentry=true</argLine>
        </configuration>

http://git-wip-us.apache.org/repos/asf/sentry/blob/53dc2d6f/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestColumnEndToEnd.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestColumnEndToEnd.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestColumnEndToEnd.java
index 8c15257..22d4cf8 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestColumnEndToEnd.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestColumnEndToEnd.java
@@ -414,66 +414,4 @@ public class TestColumnEndToEnd extends AbstractTestWithStaticConfiguration
{
     statement.close();
     connection.close();
   }
-
-  @Test
-  public void testCrossDbTableOperations() throws Exception {
-    //The privilege of user_role1 is used to test create table as select.
-    //The privilege of user_role2 is used to test create view as select.
-    Connection connection = context.createConnection(ADMIN1);
-    Statement statement = context.createStatement(connection);
-    statement.execute("CREATE database " + DB1);
-    statement.execute("CREATE database " + DB2);
-    statement.execute("use " + DB1);
-    statement.execute("CREATE ROLE user_role1");
-    statement.execute("CREATE ROLE user_role2");
-    statement.execute("CREATE TABLE tb1 (id int , name String)");
-    statement.execute("GRANT CREATE ON DATABASE db_1 TO ROLE user_role1");
-    statement.execute("GRANT CREATE ON DATABASE db_1 TO ROLE user_role2");
-    statement.execute("GRANT SELECT (id) ON TABLE tb1 TO ROLE user_role1");
-    statement.execute("GRANT SELECT  ON TABLE tb1 TO ROLE user_role2");
-    statement.execute("GRANT ROLE user_role1 TO GROUP " + USERGROUP1);
-    statement.execute("GRANT ROLE user_role2 TO GROUP " + USERGROUP2);
-    statement.execute("load data local inpath '" + dataFile.getPath() + "' into table db_1.tb1"
);
-    statement.execute("use " + DB2);
-    statement.execute("CREATE TABLE tb2 (id int, num String)");
-    statement.execute("CREATE TABLE tb3 (id int, val String)");
-    statement.execute("GRANT SELECT (num) ON TABLE tb2 TO ROLE user_role1");
-    statement.execute("GRANT SELECT (val) ON TABLE tb3 TO ROLE user_role1");
-    statement.execute("GRANT SELECT  ON TABLE tb2 TO ROLE user_role2");
-    statement.execute("GRANT SELECT  ON TABLE tb3 TO ROLE user_role2");
-    statement.execute("GRANT ROLE user_role1 TO GROUP " + USERGROUP1);
-    statement.execute("GRANT ROLE user_role2 TO GROUP " + USERGROUP2);
-    statement.execute("load data local inpath '" + dataFile.getPath() + "' into table db_2.tb2"
);
-    statement.execute("load data local inpath '" + dataFile.getPath() + "' into table db_2.tb3"
);
-    statement.close();
-    connection.close();
-
-    connection =context.createConnection(USER1_1);
-    statement = context.createStatement(connection);
-    statement.execute("use " + DB1);
-    statement.execute("CREATE table db_1.t1 as select tb1.id, tb3.val, tb2.num from db_1.tb1,db_2.tb3,db_2.tb2");
-    statement.close();
-    connection.close();
-
-    connection =context.createConnection(USER2_1);
-    statement = context.createStatement(connection);
-    //The db_1.tb1 and db_2.tb3 is same with db_2.tb2.
-    ResultSet res = statement.executeQuery("select * from db_2.tb2 limit 2");
-    List<String> expectedResult = new ArrayList<String>();
-    List<String> returnedResult = new ArrayList<String>();
-    expectedResult.add("238");
-    expectedResult.add("86");
-    while(res.next()){
-      returnedResult.add(res.getString(1).trim());
-    }
-    validateReturnedResult(expectedResult, returnedResult);
-    expectedResult.clear();
-    returnedResult.clear();
-
-    statement.execute("use " + DB1);
-    statement.execute("CREATE VIEW db_1.v1 as select tb1.id, tb3.val, tb2.num from db_1.tb1,db_2.tb3,db_2.tb2");
-    statement.close();
-    connection.close();
-
-  }
 }

http://git-wip-us.apache.org/repos/asf/sentry/blob/53dc2d6f/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbCrossOperations.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbCrossOperations.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbCrossOperations.java
new file mode 100644
index 0000000..62c0f58
--- /dev/null
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbCrossOperations.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.tests.e2e.dbprovider;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.sentry.tests.e2e.hive.AbstractTestWithStaticConfiguration;
+
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.io.Resources;
+
+public class TestDbCrossOperations extends AbstractTestWithStaticConfiguration {
+  private static final Logger LOGGER = LoggerFactory.
+      getLogger(TestDbCrossOperations.class);
+
+  private final String SINGLE_TYPE_DATA_FILE_NAME = "kv1.dat";
+  private File dataFile;
+
+  @BeforeClass
+  public static void setupTestStaticConfiguration() throws Exception{
+    LOGGER.info("TestColumnView setupTestStaticConfiguration");
+    useSentryService = true;
+    AbstractTestWithStaticConfiguration.setupTestStaticConfiguration();
+  }
+
+  @Override
+  @Before
+  public void setup() throws Exception {
+    super.setupAdmin();
+    super.setup();
+    super.setupPolicy();
+    dataFile = new File(dataDir, SINGLE_TYPE_DATA_FILE_NAME);
+    FileOutputStream to = new FileOutputStream(dataFile);
+    Resources.copy(Resources.getResource(SINGLE_TYPE_DATA_FILE_NAME), to);
+    to.close();
+  }
+
+  private void createTestData() throws Exception {
+    try (Connection connection = context.createConnection(ADMIN1)) {
+      try (Statement statement = context.createStatement(connection)) {
+        exec(statement, "CREATE database " + DB1);
+        exec(statement, "CREATE database " + DB2);
+        exec(statement, "use " + DB1);
+        exec(statement, "CREATE ROLE user_role1");
+        exec(statement, "CREATE ROLE user_role2");
+        exec(statement, "CREATE TABLE tb1 (id int , name String)");
+        exec(statement, "GRANT ALL ON DATABASE db_1 TO ROLE user_role1");
+        exec(statement, "GRANT ALL ON DATABASE db_1 TO ROLE user_role2");
+        exec(statement, "GRANT SELECT (id) ON TABLE tb1 TO ROLE user_role1");
+        exec(statement, "GRANT SELECT  ON TABLE tb1 TO ROLE user_role2");
+        exec(statement, "GRANT ROLE user_role1 TO GROUP " + USERGROUP1);
+        exec(statement, "GRANT ROLE user_role2 TO GROUP " + USERGROUP2);
+        exec(statement, "load data local inpath '" + dataFile.getPath() + "' into table db_1.tb1"
);
+        exec(statement, "use " + DB2);
+        exec(statement, "CREATE TABLE tb2 (id int, num String)");
+        exec(statement, "CREATE TABLE tb3 (id int, val String)");
+        exec(statement, "GRANT SELECT (num) ON TABLE tb2 TO ROLE user_role1");
+        exec(statement, "GRANT SELECT (val) ON TABLE tb3 TO ROLE user_role1");
+        exec(statement, "GRANT SELECT  ON TABLE tb2 TO ROLE user_role2");
+        exec(statement, "GRANT SELECT  ON TABLE tb3 TO ROLE user_role2");
+        exec(statement, "GRANT ROLE user_role1 TO GROUP " + USERGROUP1);
+        exec(statement, "GRANT ROLE user_role2 TO GROUP " + USERGROUP2);
+        exec(statement, "load data local inpath '" + dataFile.getPath() + "' into table db_2.tb2"
);
+        exec(statement, "load data local inpath '" + dataFile.getPath() + "' into table db_2.tb3"
);
+      }
+    }
+  }
+
+  @Test
+  public void testCrossDbTableOperations() throws Exception {
+    //The privilege of user_role1 is used to test create table as select.
+    //The privilege of user_role2 is used to test create view as select.
+    createTestData();
+
+    //Test create table as select from cross db table
+    try (Connection connection =context.createConnection(USER1_1)) {
+      try (Statement statement = context.createStatement(connection)) {
+        statement.execute("use " + DB1);
+        statement.execute("CREATE table db_1.t1 as select tb1.id, tb3.val, tb2.num from db_1.tb1,db_2.tb3,db_2.tb2");
+      }
+    }
+
+    //Test create view as select from cross db tables
+    try (Connection connection =context.createConnection(USER2_1)) {
+      try (Statement statement = context.createStatement(connection)) {
+        //The db_1.tb1 and db_2.tb3 is same with db_2.tb2.
+        ResultSet res = statement.executeQuery("select * from db_2.tb2 limit 2");
+        List<String> expectedResult = new ArrayList<String>();
+        List<String> returnedResult = new ArrayList<String>();
+        expectedResult.add("238");
+        expectedResult.add("86");
+        while(res.next()){
+          returnedResult.add(res.getString(1).trim());
+        }
+        validateReturnedResult(expectedResult, returnedResult);
+        expectedResult.clear();
+        returnedResult.clear();
+        exec(statement, "use " + DB1);
+        exec(statement, "CREATE VIEW db_1.v1 as select tb1.id, tb3.val, tb2.num from db_1.tb1,db_2.tb3,db_2.tb2");
+      }
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/sentry/blob/53dc2d6f/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestDbHdfsBase.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestDbHdfsBase.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestDbHdfsBase.java
index e545c37..17bc612 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestDbHdfsBase.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestDbHdfsBase.java
@@ -40,6 +40,7 @@ import static org.apache.sentry.tests.e2e.hive.hiveserver.HiveServerFactory.Hive
 
 import org.apache.sentry.tests.e2e.hive.fs.TestFSContants;
 import org.junit.After;
+import org.junit.Assume;
 import org.junit.BeforeClass;
 
 import static org.hamcrest.Matchers.equalTo;
@@ -96,9 +97,14 @@ public abstract class TestDbHdfsBase extends AbstractTestWithStaticConfiguration
       System.getProperty(TestFSContants.SENTRY_E2E_TEST_DFS_TYPE, DFSType.MiniDFS.name());
 
   protected final static String dfsAdmin = System.getProperty(TestFSContants.SENTRY_E2E_TEST_DFS_ADMIN,
"hdfs");
+  protected final static String storageUriStr = System.getProperty(TestFSContants.SENTRY_E2E_TEST_STORAGE_URI);
 
   @BeforeClass
   public static void setupTestStaticConfiguration() throws Exception {
+    if (!Strings.isNullOrEmpty(storageUriStr)) {
+      LOGGER.warn("Skip HDFS tests if HDFS fileSystem is not configured on hdfs");
+      Assume.assumeTrue(storageUriStr.toLowerCase().startsWith("hdfs"));
+    }
     useSentryService = true;
     enableHDFSAcls = true;
     AbstractTestWithStaticConfiguration.setupTestStaticConfiguration();

http://git-wip-us.apache.org/repos/asf/sentry/blob/53dc2d6f/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestExportImportPrivileges.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestExportImportPrivileges.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestExportImportPrivileges.java
index b38eaaf..a02a156 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestExportImportPrivileges.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestExportImportPrivileges.java
@@ -131,6 +131,8 @@ public class TestExportImportPrivileges extends AbstractTestWithStaticConfigurat
     connection = context.createConnection(USER1_1);
     statement = context.createStatement(connection);
     statement.execute("use " + DB1);
+    // Import/Export works with s3 storage system only when this is turned on.
+    exec(statement, "set hive.exim.uri.scheme.whitelist=hdfs,pfile,s3a;");
     statement.execute("EXPORT TABLE " + TBL1 + " TO '" + exportDir + "'");
     statement.close();
     connection.close();
@@ -139,6 +141,7 @@ public class TestExportImportPrivileges extends AbstractTestWithStaticConfigurat
     connection = context.createConnection(USER2_1);
     statement = context.createStatement(connection);
     statement.execute("use " + DB1);
+    exec(statement, "set hive.exim.uri.scheme.whitelist=hdfs,pfile,s3a;");
     context.assertAuthzException(statement, "IMPORT TABLE " + TBL2 + " FROM '" + exportDir
+ "'");
     statement.close();
     connection.close();
@@ -147,6 +150,7 @@ public class TestExportImportPrivileges extends AbstractTestWithStaticConfigurat
     connection = context.createConnection(USER1_1);
     statement = context.createStatement(connection);
     statement.execute("use " + DB1);
+    exec(statement, "set hive.exim.uri.scheme.whitelist=hdfs,pfile,s3a;");
     statement.execute("IMPORT TABLE " + TBL2 + " FROM '" + exportDir + "'");
     statement.close();
     connection.close();

http://git-wip-us.apache.org/repos/asf/sentry/blob/53dc2d6f/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestOperationsPart2.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestOperationsPart2.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestOperationsPart2.java
index 7186f1a..e11a817 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestOperationsPart2.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestOperationsPart2.java
@@ -442,6 +442,8 @@ public class TestOperationsPart2 extends AbstractTestWithStaticConfiguration
{
     connection = context.createConnection(USER3_1);
     statement = context.createStatement(connection);
     statement.execute("Use " + DB1);
+    // Import/Export works with s3 storage system only when this is turned on.
+    exec(statement, "set hive.exim.uri.scheme.whitelist=hdfs,pfile,s3a;");
     context.assertSentrySemanticException(statement, "export table tb1 to '" + location +
"'",
         semanticException);
     statement.close();
@@ -451,6 +453,7 @@ public class TestOperationsPart2 extends AbstractTestWithStaticConfiguration
{
     connection = context.createConnection(USER1_1);
     statement = context.createStatement(connection);
     statement.execute("Use " + DB1);
+    exec(statement, "set hive.exim.uri.scheme.whitelist=hdfs,pfile,s3a;");
     statement.execute("export table tb1 to '" + location + "'" );
     statement.close();
     connection.close();
@@ -459,6 +462,7 @@ public class TestOperationsPart2 extends AbstractTestWithStaticConfiguration
{
     connection = context.createConnection(USER3_1);
     statement = context.createStatement(connection);
     statement.execute("Use " + DB1);
+    exec(statement, "set hive.exim.uri.scheme.whitelist=hdfs,pfile,s3a;");
     context.assertSentrySemanticException(statement, "import table tb2 from '" + location
+ "'",
         semanticException);
     statement.close();
@@ -468,6 +472,7 @@ public class TestOperationsPart2 extends AbstractTestWithStaticConfiguration
{
     connection = context.createConnection(USER2_1);
     statement = context.createStatement(connection);
     statement.execute("Use " + DB1);
+    exec(statement, "set hive.exim.uri.scheme.whitelist=hdfs,pfile,s3a;");
     statement.execute("import table tb2 from '" + location + "'");
     statement.close();
     connection.close();


Mime
View raw message