sentry-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sp...@apache.org
Subject [1/2] sentry git commit: SENTRY-1958: Bump to Hive version 2.0 (Sergio Pena, reviewed by Alexander Kolbasov)
Date Thu, 21 Sep 2017 21:58:26 GMT
Repository: sentry
Updated Branches:
  refs/heads/master 596841949 -> da1863f34


http://git-wip-us.apache.org/repos/asf/sentry/blob/da1863f3/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/HiveServerFactory.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/HiveServerFactory.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/HiveServerFactory.java
index f3f58f6..35cb2bb 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/HiveServerFactory.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/HiveServerFactory.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.sentry.binding.hive.authz.SentryHiveAuthorizerFactory;
 import org.apache.sentry.binding.hive.conf.HiveAuthzConf;
 import org.apache.sentry.binding.hive.conf.HiveAuthzConf.AuthzConfVars;
 import org.apache.sentry.provider.file.LocalGroupResourceAuthorizationProvider;
@@ -129,8 +130,9 @@ public class HiveServerFactory {
     }
     if(!properties.containsKey(METASTORE_CONNECTION_URL)) {
       properties.put(METASTORE_CONNECTION_URL,
-          String.format("jdbc:derby:;databaseName=%s;create=true",
+          String.format("jdbc:derby:;databaseName=%s;create=true;createDatabaseIfNotExist=true",
               new File(baseDir, "metastore").getPath()));
+      properties.put("datanucleus.schema.autoCreateTables", "true");
     }
     if(!properties.containsKey(ACCESS_TESTING_MODE)) {
       properties.put(ACCESS_TESTING_MODE, "true");
@@ -180,11 +182,20 @@ public class HiveServerFactory {
       properties.put(ConfVars.METASTORESERVERMINTHREADS.varname, "5");
     }
 
-    // set the SentryMetaStoreFilterHook for HiveServer2 only, not for metastore
-    if (!HiveServer2Type.InternalMetastore.equals(type)) {
-      properties.put(ConfVars.METASTORE_FILTER_HOOK.varname,
-          org.apache.sentry.binding.metastore.SentryMetaStoreFilterHook.class.getName());
-    }
+    properties.put(ConfVars.HIVE_AUTHORIZATION_ENABLED.varname, "false");
+    properties.put(ConfVars.HIVE_AUTHORIZATION_MANAGER.varname, SentryHiveAuthorizerFactory.class.getName());
+
+    // CBO has a bug on Hive 2.0.0 with VIEWS because ReadIdentity objects are sent without
+    // parent information for partitioned columns
+    properties.put(ConfVars.HIVE_CBO_ENABLED.varname, "false");
+
+    // Hive 2.x set the following configuration to TRUE by default and it causes test issues
on
+    // Sentry because we're trying to change columns with different column types
+    properties.put(ConfVars.METASTORE_DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES.varname, "false");
+
+    // This configuration will avoid starting the HS2 WebUI that was causing test failures
when
+    // HS2 is configured for concurrency
+    properties.put(ConfVars.HIVE_IN_TEST.varname, "true");
 
     if (!properties.containsKey(METASTORE_BYPASS)) {
       properties.put(METASTORE_BYPASS, "hive,impala," + System.getProperty("user.name", ""));

http://git-wip-us.apache.org/repos/asf/sentry/blob/da1863f3/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestDBNotificationListenerInBuiltDeserializer.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestDBNotificationListenerInBuiltDeserializer.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestDBNotificationListenerInBuiltDeserializer.java
index e9b3a43..ea44e33 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestDBNotificationListenerInBuiltDeserializer.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestDBNotificationListenerInBuiltDeserializer.java
@@ -348,7 +348,7 @@ public class TestDBNotificationListenerInBuiltDeserializer extends AbstractMetas
     assertEquals(HCatEventMessage.EventType.ALTER_PARTITION, alterPartitionMessage.getEventType());
     assertThat(alterPartitionMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB));//
dbName
     assertThat(alterPartitionMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable));//
tableName
-    assertEquals(partVals1, alterPartitionMessage.getValues());
+    assertEquals(partVals1, alterPartitionMessage.getKeyValues());
     if (alterPartitionMessage instanceof SentryJSONAlterPartitionMessage) {
       SentryJSONAlterPartitionMessage sjAlterPartitionMessage = (SentryJSONAlterPartitionMessage)
alterPartitionMessage;
       assertEquals(partVals1, sjAlterPartitionMessage.getNewValues());
@@ -364,7 +364,7 @@ public class TestDBNotificationListenerInBuiltDeserializer extends AbstractMetas
     assertEquals(HCatEventMessage.EventType.ALTER_PARTITION, alterPartitionMessage.getEventType());
     assertThat(alterPartitionMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB));//
dbName
     assertThat(alterPartitionMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable));//
tableName
-    assertEquals(partVals1, alterPartitionMessage.getValues());
+    assertEquals(partVals1, alterPartitionMessage.getKeyValues());
     if (alterPartitionMessage instanceof SentryJSONAlterPartitionMessage) {
       SentryJSONAlterPartitionMessage sjAlterPartitionMessage = (SentryJSONAlterPartitionMessage)
alterPartitionMessage;
       assertEquals(partVals2, sjAlterPartitionMessage.getNewValues());

http://git-wip-us.apache.org/repos/asf/sentry/blob/da1863f3/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerSentryDeserializer.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerSentryDeserializer.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerSentryDeserializer.java
index f43b316..f33b2f3 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerSentryDeserializer.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerSentryDeserializer.java
@@ -29,6 +29,7 @@ import org.hamcrest.text.IsEqualIgnoringCase;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
 
 import org.junit.*;
 
@@ -374,7 +375,8 @@ public class TestSentryListenerSentryDeserializer extends AbstractMetastoreTestW
     if(!useDbNotificationListener) {
       assertEquals(oldLocation.toLowerCase(), alterPartitionMessage.getOldLocation());
       assertEquals(newLocation.toLowerCase(), alterPartitionMessage.getNewLocation());
-      assertEquals(partVals1, alterPartitionMessage.getValues());
+      assertTrue(partVals1.containsAll(alterPartitionMessage.getKeyValues().values()) &&
+          alterPartitionMessage.getKeyValues().values().containsAll(partVals1));
       assertEquals(partVals1, alterPartitionMessage.getNewValues());
     }
 
@@ -389,9 +391,12 @@ public class TestSentryListenerSentryDeserializer extends AbstractMetastoreTestW
     assertThat(alterPartitionMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB));//
dbName
     assertThat(alterPartitionMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable));//
tableName
     if(!useDbNotificationListener) {
-      assertEquals(partVals1, alterPartitionMessage.getValues());
+      assertTrue(partVals1.containsAll(alterPartitionMessage.getKeyValues().values()) &&
+          alterPartitionMessage.getKeyValues().values().containsAll(partVals1));
       assertEquals(partVals2, alterPartitionMessage.getNewValues());
     }
+
+    dropMetastoreDBIfExists(client, testDB);
   }
 }
 

http://git-wip-us.apache.org/repos/asf/sentry/blob/da1863f3/sentry-tests/sentry-tests-hive/src/test/resources/log4j2.properties
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/resources/log4j2.properties b/sentry-tests/sentry-tests-hive/src/test/resources/log4j2.properties
new file mode 100644
index 0000000..a6ddc29
--- /dev/null
+++ b/sentry-tests/sentry-tests-hive/src/test/resources/log4j2.properties
@@ -0,0 +1,53 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Define some default values that can be overridden by system properties.
+#
+# For testing, it may also be convenient to specify
+
+# list of properties
+property.sentry.root.logger = INFO
+
+# list of all appenders
+appenders = console
+
+# console appender
+appender.console.name = console
+appender.console.type = Console
+appender.console.target = SYSTEM_ERR
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %d (%t) [%p - %l] %m%n
+
+# list of all loggers
+loggers = Sentry, HadoopConf, Datanucleus
+
+logger.Sentry.name = org.apache.sentry
+logger.Sentry.level = DEBUG
+
+logger.HadoopConf.name = org.apache.hadoop.conf.Configuration
+logger.HadoopConf.level = ERROR
+
+logger.Datanucleus.name = org.datanucleus
+logger.Datanucleus.level = ERROR
+
+# root logger
+rootLogger.level = ${sys:sentry.root.logger}
+rootLogger.appenderRefs = console
+rootLogger.appenderRef.console.ref = console
+rootLogger.appenderRef.console.level = ${sys:sentry.root.logger}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/sentry/blob/da1863f3/sentry-tests/sentry-tests-kafka/pom.xml
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-kafka/pom.xml b/sentry-tests/sentry-tests-kafka/pom.xml
index 56a3ef1..c65323d 100644
--- a/sentry-tests/sentry-tests-kafka/pom.xml
+++ b/sentry-tests/sentry-tests-kafka/pom.xml
@@ -65,69 +65,25 @@ limitations under the License.
       <artifactId>curator-framework</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.datanucleus</groupId>
+      <artifactId>datanucleus-core</artifactId>
+      <version>${datanucleus-core.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.datanucleus</groupId>
+      <artifactId>datanucleus-api-jdo</artifactId>
+      <version>${datanucleus-api-jdo.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.datanucleus</groupId>
+      <artifactId>datanucleus-rdbms</artifactId>
+      <version>${datanucleus-rdbms.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.datanucleus</groupId>
+      <artifactId>javax.jdo</artifactId>
+      <version>${datanucleus-jdo.version}</version>
+    </dependency>
   </dependencies>
-  <profiles>
-    <profile>
-      <id>datanucleus3</id>
-      <activation>
-        <activeByDefault>true</activeByDefault>
-      </activation>
-      <properties>
-        <datanucleus-api-jdo.version>3.2.6</datanucleus-api-jdo.version>
-        <datanucleus-core.version>3.2.12</datanucleus-core.version>
-        <datanucleus-rdbms.version>3.2.12</datanucleus-rdbms.version>
-      </properties>
-      <dependencies>
-        <dependency>
-          <groupId>org.datanucleus</groupId>
-          <artifactId>datanucleus-core</artifactId>
-          <version>${datanucleus-core.version}</version>
-        </dependency>
-        <dependency>
-          <groupId>org.datanucleus</groupId>
-          <artifactId>datanucleus-api-jdo</artifactId>
-          <version>${datanucleus-api-jdo.version}</version>
-        </dependency>
-        <dependency>
-          <groupId>org.datanucleus</groupId>
-          <artifactId>datanucleus-rdbms</artifactId>
-          <version>${datanucleus-rdbms.version}</version>
-        </dependency>
-      </dependencies>
-    </profile>
-    <profile>
-      <id>datanucleus4</id>
-      <activation>
-        <activeByDefault>false</activeByDefault>
-      </activation>
-      <properties>
-        <datanucleus-api-jdo.version>4.2.1</datanucleus-api-jdo.version>
-        <datanucleus-core.version>4.1.6</datanucleus-core.version>
-        <datanucleus-rdbms.version>4.1.7</datanucleus-rdbms.version>
-        <datanucleus-jdo.version>3.2.0-m3</datanucleus-jdo.version>
-      </properties>
-      <dependencies>
-        <dependency>
-          <groupId>org.datanucleus</groupId>
-          <artifactId>datanucleus-core</artifactId>
-          <version>${datanucleus-core.version}</version>
-        </dependency>
-        <dependency>
-          <groupId>org.datanucleus</groupId>
-          <artifactId>datanucleus-api-jdo</artifactId>
-          <version>${datanucleus-api-jdo.version}</version>
-        </dependency>
-        <dependency>
-          <groupId>org.datanucleus</groupId>
-          <artifactId>datanucleus-rdbms</artifactId>
-          <version>${datanucleus-rdbms.version}</version>
-        </dependency>
-        <dependency>
-          <groupId>org.datanucleus</groupId>
-          <artifactId>javax.jdo</artifactId>
-          <version>${datanucleus-jdo.version}</version>
-        </dependency>
-      </dependencies>
-    </profile>
- </profiles>
 </project>

http://git-wip-us.apache.org/repos/asf/sentry/blob/da1863f3/sentry-tests/sentry-tests-solr/pom.xml
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-solr/pom.xml b/sentry-tests/sentry-tests-solr/pom.xml
index c704768..efb708c 100644
--- a/sentry-tests/sentry-tests-solr/pom.xml
+++ b/sentry-tests/sentry-tests-solr/pom.xml
@@ -82,6 +82,26 @@ limitations under the License.
         </exclusion>
       </exclusions>
     </dependency>
+    <dependency>
+      <groupId>org.datanucleus</groupId>
+      <artifactId>datanucleus-core</artifactId>
+      <version>${datanucleus-core.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.datanucleus</groupId>
+      <artifactId>datanucleus-api-jdo</artifactId>
+      <version>${datanucleus-api-jdo.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.datanucleus</groupId>
+      <artifactId>datanucleus-rdbms</artifactId>
+      <version>${datanucleus-rdbms.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.datanucleus</groupId>
+      <artifactId>javax.jdo</artifactId>
+      <version>${datanucleus-jdo.version}</version>
+    </dependency>
   </dependencies>
   <build>
     <plugins>
@@ -94,68 +114,4 @@ limitations under the License.
       </plugin>
     </plugins>
   </build>
- <profiles>
-    <profile>
-      <id>datanucleus3</id>
-      <activation>
-        <activeByDefault>true</activeByDefault>
-      </activation>
-      <properties>
-        <datanucleus-api-jdo.version>3.2.6</datanucleus-api-jdo.version>
-        <datanucleus-core.version>3.2.12</datanucleus-core.version>
-        <datanucleus-rdbms.version>3.2.12</datanucleus-rdbms.version>
-      </properties>
-      <dependencies>
-        <dependency>
-          <groupId>org.datanucleus</groupId>
-          <artifactId>datanucleus-core</artifactId>
-          <version>${datanucleus-core.version}</version>
-        </dependency>
-        <dependency>
-          <groupId>org.datanucleus</groupId>
-          <artifactId>datanucleus-api-jdo</artifactId>
-          <version>${datanucleus-api-jdo.version}</version>
-        </dependency>
-        <dependency>
-          <groupId>org.datanucleus</groupId>
-          <artifactId>datanucleus-rdbms</artifactId>
-          <version>${datanucleus-rdbms.version}</version>
-        </dependency>
-      </dependencies>
-    </profile>
-    <profile>
-      <id>datanucleus4</id>
-      <activation>
-        <activeByDefault>false</activeByDefault>
-      </activation>
-      <properties>
-        <datanucleus-api-jdo.version>4.2.1</datanucleus-api-jdo.version>
-        <datanucleus-core.version>4.1.6</datanucleus-core.version>
-        <datanucleus-rdbms.version>4.1.7</datanucleus-rdbms.version>
-        <datanucleus-jdo.version>3.2.0-m3</datanucleus-jdo.version>
-      </properties>
-      <dependencies>
-        <dependency>
-          <groupId>org.datanucleus</groupId>
-          <artifactId>datanucleus-core</artifactId>
-          <version>${datanucleus-core.version}</version>
-        </dependency>
-        <dependency>
-          <groupId>org.datanucleus</groupId>
-          <artifactId>datanucleus-api-jdo</artifactId>
-          <version>${datanucleus-api-jdo.version}</version>
-        </dependency>
-        <dependency>
-          <groupId>org.datanucleus</groupId>
-          <artifactId>datanucleus-rdbms</artifactId>
-          <version>${datanucleus-rdbms.version}</version>
-        </dependency>
-        <dependency>
-          <groupId>org.datanucleus</groupId>
-          <artifactId>javax.jdo</artifactId>
-          <version>${datanucleus-jdo.version}</version>
-        </dependency>
-      </dependencies>
-    </profile>
- </profiles>
 </project>

http://git-wip-us.apache.org/repos/asf/sentry/blob/da1863f3/sentry-tests/sentry-tests-sqoop/pom.xml
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-sqoop/pom.xml b/sentry-tests/sentry-tests-sqoop/pom.xml
index 9e7cee9..05b1653 100644
--- a/sentry-tests/sentry-tests-sqoop/pom.xml
+++ b/sentry-tests/sentry-tests-sqoop/pom.xml
@@ -95,71 +95,29 @@ limitations under the License.
       <groupId>com.google.guava</groupId>
       <artifactId>guava</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.datanucleus</groupId>
+      <artifactId>datanucleus-core</artifactId>
+      <version>${datanucleus-core.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.datanucleus</groupId>
+      <artifactId>datanucleus-api-jdo</artifactId>
+      <version>${datanucleus-api-jdo.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.datanucleus</groupId>
+      <artifactId>datanucleus-rdbms</artifactId>
+      <version>${datanucleus-rdbms.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.datanucleus</groupId>
+      <artifactId>javax.jdo</artifactId>
+      <version>${datanucleus-jdo.version}</version>
+    </dependency>
   </dependencies>
   <profiles>
     <profile>
-      <id>datanucleus3</id>
-      <activation>
-        <activeByDefault>true</activeByDefault>
-      </activation>
-      <properties>
-        <datanucleus-api-jdo.version>3.2.6</datanucleus-api-jdo.version>
-        <datanucleus-core.version>3.2.12</datanucleus-core.version>
-        <datanucleus-rdbms.version>3.2.12</datanucleus-rdbms.version>
-      </properties>
-      <dependencies>
-        <dependency>
-          <groupId>org.datanucleus</groupId>
-          <artifactId>datanucleus-core</artifactId>
-          <version>${datanucleus-core.version}</version>
-        </dependency>
-        <dependency>
-          <groupId>org.datanucleus</groupId>
-          <artifactId>datanucleus-api-jdo</artifactId>
-          <version>${datanucleus-api-jdo.version}</version>
-        </dependency>
-        <dependency>
-          <groupId>org.datanucleus</groupId>
-          <artifactId>datanucleus-rdbms</artifactId>
-          <version>${datanucleus-rdbms.version}</version>
-        </dependency>
-      </dependencies>
-    </profile>
-    <profile>
-      <id>datanucleus4</id>
-      <activation>
-        <activeByDefault>false</activeByDefault>
-      </activation>
-      <properties>
-        <datanucleus-api-jdo.version>4.2.1</datanucleus-api-jdo.version>
-        <datanucleus-core.version>4.1.6</datanucleus-core.version>
-        <datanucleus-rdbms.version>4.1.7</datanucleus-rdbms.version>
-        <datanucleus-jdo.version>3.2.0-m3</datanucleus-jdo.version>
-      </properties>
-      <dependencies>
-        <dependency>
-          <groupId>org.datanucleus</groupId>
-          <artifactId>datanucleus-core</artifactId>
-          <version>${datanucleus-core.version}</version>
-        </dependency>
-        <dependency>
-          <groupId>org.datanucleus</groupId>
-          <artifactId>datanucleus-api-jdo</artifactId>
-          <version>${datanucleus-api-jdo.version}</version>
-        </dependency>
-        <dependency>
-          <groupId>org.datanucleus</groupId>
-          <artifactId>datanucleus-rdbms</artifactId>
-          <version>${datanucleus-rdbms.version}</version>
-        </dependency>
-        <dependency>
-          <groupId>org.datanucleus</groupId>
-          <artifactId>javax.jdo</artifactId>
-          <version>${datanucleus-jdo.version}</version>
-        </dependency>
-      </dependencies>
-    </profile>
-    <profile>
       <id>download-sqoop2</id>
       <activation>
         <activeByDefault>true</activeByDefault>


Mime
View raw message