sentry-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ak...@apache.org
Subject sentry git commit: SENTRY-1387: Add HDFS sync tests for drop partition for external/implicit locations (Nachiket Vaidya, Review by kalyan kumar kalvagadda, Alex Kolbasov and Hao Hao)
Date Wed, 08 Feb 2017 20:10:53 GMT
Repository: sentry
Updated Branches:
  refs/heads/sentry-ha-redesign 183af2c9b -> e840bdb15


SENTRY-1387: Add HDFS sync tests for drop partition for external/implicit locations (Nachiket
Vaidya, Review by kalyan kumar kalvagadda, Alex Kolbasov and Hao Hao)

Root cause: makeMetastoreBasePartitionObject() assigns the table location in partition.
There are two versions of makeMetastorePartitionObject() that call makeMetastoreBasePartitionObject().
One sets given location and other keeps the location unchanged (which is table location).
To fix this: set the partition location to null in makeMetastoreBasePartitionObject().
It will auto-generated by Hive in the format : <part_column>=<part_value>.

Testing done:
  * ran complete test cases of sentry
  * modified unit test case.


Project: http://git-wip-us.apache.org/repos/asf/sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/e840bdb1
Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/e840bdb1
Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/e840bdb1

Branch: refs/heads/sentry-ha-redesign
Commit: e840bdb1572d597205d9bbd85b80ff0f3a9f7c8c
Parents: 183af2c
Author: Alexander Kolbasov <akolb@cloudera.com>
Authored: Wed Feb 8 12:08:23 2017 -0800
Committer: Alexander Kolbasov <akolb@cloudera.com>
Committed: Wed Feb 8 12:08:23 2017 -0800

----------------------------------------------------------------------
 ...bstractMetastoreTestWithStaticConfiguration.java |  1 +
 .../TestSentryListenerSentryDeserializer.java       | 16 ++++++++++------
 2 files changed, 11 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sentry/blob/e840bdb1/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/AbstractMetastoreTestWithStaticConfiguration.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/AbstractMetastoreTestWithStaticConfiguration.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/AbstractMetastoreTestWithStaticConfiguration.java
index 567b4c8..18720eb 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/AbstractMetastoreTestWithStaticConfiguration.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/AbstractMetastoreTestWithStaticConfiguration.java
@@ -188,6 +188,7 @@ public abstract class AbstractMetastoreTestWithStaticConfiguration extends
     part4.setValues(ptnVals);
     part4.setParameters(new HashMap<String, String>());
     part4.setSd(tbl.getSd().deepCopy());
+    part4.getSd().setLocation(null);
     part4.getSd().setSerdeInfo(tbl.getSd().getSerdeInfo().deepCopy());
     part4.setParameters(new HashMap<String, String>());
     return part4;

http://git-wip-us.apache.org/repos/asf/sentry/blob/e840bdb1/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerSentryDeserializer.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerSentryDeserializer.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerSentryDeserializer.java
index 4d8889a..5a51d93 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerSentryDeserializer.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerSentryDeserializer.java
@@ -221,15 +221,17 @@ public class TestSentryListenerSentryDeserializer extends AbstractMetastoreTestW
   public void testAddDropPartition() throws Exception {
     testDB = "N_db" + random.nextInt(Integer.SIZE - 1);
     String testTable = "N_table" + random.nextInt(Integer.SIZE - 1);
+    String partColName = "part_col1";
+    String partColValue = "part1";
 
     NotificationEventResponse response;
     CurrentNotificationEventId latestID, previousID;
     // Create database and table
     createMetastoreDB(client, testDB);
     Table tbl1 = createMetastoreTableWithPartition(client, testDB, testTable, Lists.newArrayList(new
FieldSchema("col1", "int", "")),
-        Lists.newArrayList(new FieldSchema("part_col1", "string", "")));
+        Lists.newArrayList(new FieldSchema(partColName, "string", "")));
 
-    ArrayList<String> partVals1 = Lists.newArrayList("part1");
+    ArrayList<String> partVals1 = Lists.newArrayList(partColValue);
 
     //Add partition
     // We need:
@@ -243,7 +245,7 @@ public class TestSentryListenerSentryDeserializer extends AbstractMetastoreTestW
     assertEquals(HCatEventMessage.EventType.ADD_PARTITION, addPartitionMessage.getEventType());
     assertThat(addPartitionMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB));//
dbName (returns lowered version)
     assertThat(addPartitionMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable));//
tableName (returns lowered version)
-    String expectedLocation = warehouseDir + "/" + testDB + ".db/" + testTable ; //TODO:
SENTRY-1387: Tablelocation is stored instead of partition location
+    String expectedLocation = warehouseDir + "/" + testDB + ".db/" + testTable + "/" + partColName
+ "=" + partColValue;
     if(!useDbNotificationListener) {
       Assert.assertEquals(expectedLocation.toLowerCase(), addPartitionMessage.getLocations().get(0));
     }
@@ -340,6 +342,8 @@ public class TestSentryListenerSentryDeserializer extends AbstractMetastoreTestW
   public void testAlterPartition() throws Exception {
     testDB = "N_db" + random.nextInt(Integer.SIZE - 1);
     String testTable = "N_table" + random.nextInt(Integer.SIZE - 1);
+    String partColName = "part_col1";
+    String partColValue = "part1";
 
     NotificationEventResponse response;
     CurrentNotificationEventId latestID;
@@ -349,8 +353,8 @@ public class TestSentryListenerSentryDeserializer extends AbstractMetastoreTestW
     // Create table with partition
     Table tbl1 = createMetastoreTableWithPartition(client, testDB,
         testTable, Lists.newArrayList(new FieldSchema("col1", "int", "")),
-        Lists.newArrayList(new FieldSchema("part_col1", "string", "")));
-    ArrayList<String> partVals1 = Lists.newArrayList("part1");
+        Lists.newArrayList(new FieldSchema(partColName, "string", "")));
+    ArrayList<String> partVals1 = Lists.newArrayList(partColValue);
     Partition partition = addPartition(client, testDB, testTable, partVals1, tbl1);
 
     //Alter partition with location
@@ -358,7 +362,7 @@ public class TestSentryListenerSentryDeserializer extends AbstractMetastoreTestW
     // - dbName
     // - tableName
     // - partition location
-    String oldLocation = tbl1.getSd().getLocation(); //TODO: SENTRY-1387: Tablelocation is
stored instead of partition location
+    String oldLocation = tbl1.getSd().getLocation()  + "/" + partColName + "=" + partColValue;
     String newLocation = warehouseDir + File.separator + "newpart";
     alterPartitionWithLocation(client, partition, newLocation);
     latestID = client.getCurrentNotificationEventId();


Mime
View raw message