sentry-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sra...@apache.org
Subject [1/3] incubator-sentry git commit: SENTRY-953: External Partitions which are referenced by more than one table can cause some unexpected behavior with Sentry HDFS sync ( Hao Hao, Reviewed by: Sravya Tirukkovalur)
Date Thu, 17 Dec 2015 06:44:16 GMT
Repository: incubator-sentry
Updated Branches:
  refs/heads/master 9fe720232 -> 4b33ad929


http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/4b33ad92/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationInfo.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationInfo.java
b/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationInfo.java
index c9accc1..c8d56be 100644
--- a/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationInfo.java
+++ b/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationInfo.java
@@ -17,9 +17,7 @@
  */
 package org.apache.sentry.hdfs;
 
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
+import java.util.*;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ThreadFactory;
@@ -126,6 +124,7 @@ public class SentryAuthorizationInfo implements Runnable {
   }
 
   private boolean update() {
+    //Looks like getting same updates multiple times
     SentryAuthzUpdate updates = updater.getUpdates();
     // Updates can be null if Sentry Service is un-reachable
     if (updates != null) {
@@ -274,13 +273,24 @@ public class SentryAuthorizationInfo implements Runnable {
   public List<AclEntry> getAclEntries(String[] pathElements) {
     lock.readLock().lock();
     try {
-      String authzObj = authzPaths.findAuthzObject(pathElements);
+      Set<String> authzObjs = authzPaths.findAuthzObject(pathElements);
       // Apparently setFAcl throws error if 'group::---' is not present
       AclEntry noGroup = AclEntry.parseAclEntry("group::---", true);
-      ArrayList<AclEntry> retList = Lists.newArrayList(noGroup);
-      retList.addAll((authzObj != null) ? authzPermissions.getAcls(authzObj)
-          : Collections.EMPTY_LIST);
-      return retList;
+
+      Set<AclEntry> retSet = new HashSet<AclEntry>();
+      retSet.add(noGroup);
+
+      if (authzObjs == null) {
+        retSet.addAll(Collections.EMPTY_LIST);
+        return new ArrayList<AclEntry>(retSet);
+      }
+
+      // No duplicate acls should be added.
+      for (String authzObj: authzObjs) {
+        retSet.addAll(authzPermissions.getAcls(authzObj));
+      }
+
+      return new ArrayList<AclEntry>(retSet);
     } finally {
       lock.readLock().unlock();
     }

http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/4b33ad92/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryPermissions.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryPermissions.java
b/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryPermissions.java
index 2c50ea9..daa87cf 100644
--- a/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryPermissions.java
+++ b/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryPermissions.java
@@ -17,13 +17,7 @@
  */
 package org.apache.sentry.hdfs;
 
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
 
 import org.apache.hadoop.fs.permission.AclEntry;
 import org.apache.hadoop.fs.permission.AclEntryScope;
@@ -79,9 +73,12 @@ public class SentryPermissions implements AuthzPermissions {
     }
   }
 
-  private final Map<String, PrivilegeInfo> privileges = new HashMap<String, PrivilegeInfo>();
+  // Comparison of authorizable object should be case insensitive.
+  private final Map<String, PrivilegeInfo> privileges = new TreeMap<String, PrivilegeInfo>(String.CASE_INSENSITIVE_ORDER);
+  private Map<String, Set<String>> authzObjChildren = new TreeMap<String,
Set<String>>(String.CASE_INSENSITIVE_ORDER);
+
+  // Should the comparison of role be case insensitive?
   private final Map<String, RoleInfo> roles = new HashMap<String, RoleInfo>();
-  private Map<String, Set<String>> authzObjChildren = new HashMap<String,
Set<String>>();
 
   String getParentAuthzObject(String authzObject) {
     int dot = authzObject.indexOf('.');

http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/4b33ad92/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestMetastoreCacheInitializer.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestMetastoreCacheInitializer.java
b/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestMetastoreCacheInitializer.java
index 437ba94..2c9e19d 100644
--- a/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestMetastoreCacheInitializer.java
+++ b/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestMetastoreCacheInitializer.java
@@ -29,6 +29,8 @@ import org.junit.Test;
 import org.mockito.Mockito;
 
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
 
 public class TestMetastoreCacheInitializer {
 
@@ -113,19 +115,19 @@ public class TestMetastoreCacheInitializer {
             MetastoreCacheInitializer(hmsHandler, conf);
     UpdateableAuthzPaths update = cacheInitializer.createInitialUpdate();
 
-    Assert.assertEquals("db1", update.findAuthzObjectExactMatch(new
+    Assert.assertEquals(new HashSet<String>(Arrays.asList("db1")), update.findAuthzObjectExactMatches(new
             String[]{"db1"}));
-    Assert.assertEquals("db2", update.findAuthzObjectExactMatch(new
+    Assert.assertEquals(new HashSet<String>(Arrays.asList("db2")), update.findAuthzObjectExactMatches(new
             String[]{"db2"}));
-    Assert.assertEquals("db2.tab21", update.findAuthzObjectExactMatch(new
+    Assert.assertEquals(new HashSet<String>(Arrays.asList("db2.tab21")), update.findAuthzObjectExactMatches(new
             String[]{"db2", "tab21"}));
-    Assert.assertEquals("db3", update.findAuthzObjectExactMatch(new
+    Assert.assertEquals(new HashSet<String>(Arrays.asList("db3")), update.findAuthzObjectExactMatches(new
             String[]{"db3"}));
-    Assert.assertEquals("db3.tab31", update.findAuthzObjectExactMatch(new
+    Assert.assertEquals(new HashSet<String>(Arrays.asList("db3.tab31")), update.findAuthzObjectExactMatches(new
             String[]{"db3", "tab31"}));
-    Assert.assertEquals("db3.tab31", update.findAuthzObjectExactMatch(new
+    Assert.assertEquals(new HashSet<String>(Arrays.asList("db3.tab31")), update.findAuthzObjectExactMatches(new
             String[]{"db3", "tab31", "part311"}));
-    Assert.assertEquals("db3.tab31", update.findAuthzObjectExactMatch(new
+    Assert.assertEquals(new HashSet<String>(Arrays.asList("db3.tab31")), update.findAuthzObjectExactMatches(new
             String[]{"db3", "tab31", "part312"}));
     cacheInitializer.close();
 

http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/4b33ad92/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegration.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegration.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegration.java
index 208c93b..5a93ba0 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegration.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegration.java
@@ -76,8 +76,8 @@ import org.apache.hadoop.mapred.TextOutputFormat;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.sentry.binding.hive.SentryHiveAuthorizationTaskFactoryImpl;
 import org.apache.sentry.binding.hive.conf.HiveAuthzConf;
-import org.apache.sentry.hdfs.SentryAuthorizationConstants;
 import org.apache.sentry.hdfs.SentryAuthorizationProvider;
+import org.apache.sentry.provider.db.SentryAlreadyExistsException;
 import org.apache.sentry.provider.db.SimpleDBProviderBackend;
 import org.apache.sentry.provider.file.LocalGroupResourceAuthorizationProvider;
 import org.apache.sentry.provider.file.PolicyFile;
@@ -1124,6 +1124,203 @@ public class TestHDFSIntegration {
     conn.close();
   }
 
+  /* SENTRY-953 */
+  @Test
+  public void testAuthzObjOnPartitionMultipleTables() throws Throwable {
+    String dbName = "db1";
+
+    tmpHDFSDir = new Path("/tmp/external");
+    miniDFS.getFileSystem().mkdirs(tmpHDFSDir);
+    Path partitionDir = new Path("/tmp/external/p1");
+    miniDFS.getFileSystem().mkdirs(partitionDir);
+
+    dbNames = new String[]{dbName};
+    roles = new String[]{"admin_role", "tab1_role", "tab2_role", "tab3_role"};
+    admin = StaticUserGroup.ADMIN1;
+
+    Connection conn;
+    Statement stmt;
+
+    conn = hiveServer2.createConnection("hive", "hive");
+    stmt = conn.createStatement();
+
+    stmt.execute("create role admin_role");
+    stmt.execute("grant all on server server1 to role admin_role");
+    stmt.execute("grant role admin_role to group " + StaticUserGroup.ADMINGROUP);
+
+    // Create external table tab1 on location '/tmp/external/p1'.
+    // Create tab1_role, and grant it with insert permission on table tab1 to user_group1.
+    conn = hiveServer2.createConnection(StaticUserGroup.ADMIN1, StaticUserGroup.ADMIN1);
+    stmt = conn.createStatement();
+    stmt.execute("create database " + dbName);
+    stmt.execute("use " + dbName);
+    stmt.execute("create external table tab1 (s string) partitioned by (month int) location
'/tmp/external/p1'");
+    stmt.execute("create role tab1_role");
+    stmt.execute("grant insert on table tab1 to role tab1_role");
+    stmt.execute("grant role tab1_role to group " + StaticUserGroup.USERGROUP1);
+    Thread.sleep(CACHE_REFRESH);//Wait till sentry cache is updated in Namenode
+
+    // Verify that user_group1 has insert(write_execute) permission on '/tmp/external/p1'.
+    verifyOnAllSubDirs("/tmp/external/p1", FsAction.WRITE_EXECUTE, StaticUserGroup.USERGROUP1,
true);
+
+    // Create external table tab2 and partition on location '/tmp/external'.
+    // Create tab2_role, and grant it with select permission on table tab2 to user_group2.
+    stmt.execute("create external table tab2 (s string) partitioned by (month int)");
+    stmt.execute("alter table tab2 add partition (month = 1) location '/tmp/external'");
+    stmt.execute("create role tab2_role");
+    stmt.execute("grant select on table tab2 to role tab2_role");
+    stmt.execute("grant role tab2_role to group " + StaticUserGroup.USERGROUP2);
+    Thread.sleep(CACHE_REFRESH);//Wait till sentry cache is updated in Namenode
+
+    // Verify that user_group2 have select(read_execute) permission on both paths.
+    verifyOnAllSubDirs("/user/hive/warehouse/" + dbName + ".db/tab2", FsAction.READ_EXECUTE,
StaticUserGroup.USERGROUP2, true);
+    verifyOnPath("/tmp/external", FsAction.READ_EXECUTE, StaticUserGroup.USERGROUP2, true);
+
+    // Create table tab3 and partition on the same location '/tmp/external' as tab2.
+    // Create tab3_role, and grant it with insert permission on table tab3 to user_group3.
+    stmt.execute("create table tab3 (s string) partitioned by (month int)");
+    stmt.execute("alter table tab3 add partition (month = 1) location '/tmp/external'");
+    stmt.execute("create role tab3_role");
+    stmt.execute("grant insert on table tab3 to role tab3_role");
+    stmt.execute("grant role tab3_role to group " + StaticUserGroup.USERGROUP3);
+    Thread.sleep(CACHE_REFRESH);//Wait till sentry cache is updated in Namenode
+
+    // When two partitions of different tables pointing to the same location with different
grants,
+    // ACLs should have union (no duplicates) of both rules.
+    verifyOnAllSubDirs("/user/hive/warehouse/" + dbName + ".db/tab3", FsAction.WRITE_EXECUTE,
StaticUserGroup.USERGROUP3, true);
+    verifyOnPath("/tmp/external", FsAction.READ_EXECUTE, StaticUserGroup.USERGROUP2, true);
+    verifyOnPath("/tmp/external", FsAction.WRITE_EXECUTE, StaticUserGroup.USERGROUP3, true);
+
+    // When alter the table name (tab2 to be tabx), ACLs should remain the same.
+    stmt.execute("alter table tab2 rename to tabx");
+    Thread.sleep(CACHE_REFRESH);//Wait till sentry cache is updated in Namenode
+    verifyOnPath("/tmp/external", FsAction.READ_EXECUTE, StaticUserGroup.USERGROUP2, true);
+    verifyOnPath("/tmp/external", FsAction.WRITE_EXECUTE, StaticUserGroup.USERGROUP3, true);
+
+    // When drop a partition that shares the same location with other partition belonging
to
+    // other table, should still have the other table permissions.
+    stmt.execute("ALTER TABLE tabx DROP PARTITION (month = 1)");
+    Thread.sleep(CACHE_REFRESH);//Wait till sentry cache is updated in Namenode
+    verifyOnAllSubDirs("/user/hive/warehouse/" + dbName + ".db/tab3", FsAction.WRITE_EXECUTE,
StaticUserGroup.USERGROUP3, true);
+    verifyOnPath("/tmp/external", FsAction.WRITE_EXECUTE, StaticUserGroup.USERGROUP3, true);
+
+    // When drop a table that has a partition shares the same location with other partition
+    // belonging to other table, should still have the other table permissions.
+    stmt.execute("DROP TABLE IF EXISTS tabx");
+    Thread.sleep(CACHE_REFRESH);//Wait till sentry cache is updated in Namenode
+    verifyOnAllSubDirs("/user/hive/warehouse/" + dbName + ".db/tab3", FsAction.WRITE_EXECUTE,
StaticUserGroup.USERGROUP3, true);
+    verifyOnPath("/tmp/external", FsAction.WRITE_EXECUTE, StaticUserGroup.USERGROUP3, true);
+
+    stmt.close();
+    conn.close();
+
+    miniDFS.getFileSystem().delete(partitionDir, true);
+  }
+
+  /* SENTRY-953 */
+  @Test
+  public void testAuthzObjOnPartitionSameTable() throws Throwable {
+    String dbName = "db1";
+
+    tmpHDFSDir = new Path("/tmp/external/p1");
+    miniDFS.getFileSystem().mkdirs(tmpHDFSDir);
+
+    dbNames = new String[]{dbName};
+    roles = new String[]{"admin_role", "tab1_role"};
+    admin = StaticUserGroup.ADMIN1;
+
+    Connection conn;
+    Statement stmt;
+
+    conn = hiveServer2.createConnection("hive", "hive");
+    stmt = conn.createStatement();
+
+    stmt.execute("create role admin_role");
+    stmt.execute("grant all on server server1 to role admin_role");
+    stmt.execute("grant role admin_role to group " + StaticUserGroup.ADMINGROUP);
+
+    // Create table tab1 and partition on the same location '/tmp/external/p1'.
+    // Create tab1_role, and grant it with insert permission on table tab1 to user_group1.
+    conn = hiveServer2.createConnection(StaticUserGroup.ADMIN1, StaticUserGroup.ADMIN1);
+    stmt = conn.createStatement();
+    stmt.execute("create database " + dbName);
+    stmt.execute("use " + dbName);
+    stmt.execute("create table tab1 (s string) partitioned by (month int)");
+    stmt.execute("alter table tab1 add partition (month = 1) location '/tmp/external/p1'");
+    stmt.execute("create role tab1_role");
+    stmt.execute("grant insert on table tab1 to role tab1_role");
+    stmt.execute("grant role tab1_role to group " + StaticUserGroup.USERGROUP1);
+    Thread.sleep(CACHE_REFRESH);//Wait till sentry cache is updated in Namenode
+
+    // Verify that user_group1 has insert(write_execute) permission on '/tmp/external/p1'.
+    verifyOnAllSubDirs("/tmp/external/p1", FsAction.WRITE_EXECUTE, StaticUserGroup.USERGROUP1,
true);
+
+    // When two partitions of the same table pointing to the same location,
+    // ACLS should not be repeated. Exception will be thrown if there are duplicates.
+    stmt.execute("alter table tab1 add partition (month = 2) location '/tmp/external/p1'");
+    verifyOnPath("/tmp/external/p1", FsAction.WRITE_EXECUTE, StaticUserGroup.USERGROUP1,
true);
+
+    stmt.close();
+    conn.close();
+  }
+
+  /* SENTRY-953 */
+  @Test
+  public void testAuthzObjOnMultipleTables() throws Throwable {
+    String dbName = "db1";
+
+    tmpHDFSDir = new Path("/tmp/external/p1");
+    miniDFS.getFileSystem().mkdirs(tmpHDFSDir);
+
+    dbNames = new String[]{dbName};
+    roles = new String[]{"admin_role", "tab1_role", "tab2_role"};
+    admin = StaticUserGroup.ADMIN1;
+
+    Connection conn;
+    Statement stmt;
+
+    conn = hiveServer2.createConnection("hive", "hive");
+    stmt = conn.createStatement();
+
+    stmt.execute("create role admin_role");
+    stmt.execute("grant all on server server1 to role admin_role");
+    stmt.execute("grant role admin_role to group " + StaticUserGroup.ADMINGROUP);
+
+    // Create external table tab1 on location '/tmp/external/p1'.
+    // Create tab1_role, and grant it with insert permission on table tab1 to user_group1.
+    conn = hiveServer2.createConnection(StaticUserGroup.ADMIN1, StaticUserGroup.ADMIN1);
+    stmt = conn.createStatement();
+    stmt.execute("create database " + dbName);
+    stmt.execute("use " + dbName);
+    stmt.execute("create external table tab1 (s string) partitioned by (month int) location
'/tmp/external/p1'");
+    stmt.execute("create role tab1_role");
+    stmt.execute("grant insert on table tab1 to role tab1_role");
+    stmt.execute("grant role tab1_role to group " + StaticUserGroup.USERGROUP1);
+    Thread.sleep(CACHE_REFRESH);//Wait till sentry cache is updated in Namenode
+
+    // Verify that user_group1 has insert(write_execute) permission on '/tmp/external/p1'.
+    verifyOnAllSubDirs("/tmp/external/p1", FsAction.WRITE_EXECUTE, StaticUserGroup.USERGROUP1,
true);
+
+    // Create table tab2 on the same location '/tmp/external/p1' as table tab1.
+    // Create tab2_role, and grant it with select permission on table tab2 to user_group1.
+    stmt.execute("create table tab2 (s string) partitioned by (month int) location '/tmp/external/p1'");
+    stmt.execute("create role tab2_role");
+    stmt.execute("grant select on table tab2 to role tab2_role");
+    stmt.execute("grant role tab2_role to group " + StaticUserGroup.USERGROUP1);
+
+    // When two tables pointing to the same location, ACLS should have union (no duplicates)
+    // of both rules.
+    verifyOnPath("/tmp/external/p1", FsAction.ALL, StaticUserGroup.USERGROUP1, true);
+
+    // When drop table tab1, ACLs of tab2 still remain.
+    stmt.execute("DROP TABLE IF EXISTS tab1");
+    Thread.sleep(CACHE_REFRESH);//Wait till sentry cache is updated in Namenode
+    verifyOnPath("/tmp/external/p1", FsAction.READ_EXECUTE, StaticUserGroup.USERGROUP1, true);
+
+    stmt.close();
+    conn.close();
+  }
+
   private void verifyAccessToPath(String user, String group, String path, boolean hasPermission)
throws Exception{
     Path p = new Path(path);
     UserGroupInformation hadoopUser =
@@ -1305,7 +1502,13 @@ public class TestHDFSIntegration {
     Map<String, FsAction> acls = new HashMap<String, FsAction>();
     for (AclEntry ent : aclStatus.getEntries()) {
       if (ent.getType().equals(AclEntryType.GROUP)) {
-        acls.put(ent.getName(), ent.getPermission());
+
+        // In case of duplicate acl exist, exception should be thrown.
+        if (acls.containsKey(ent.getName())) {
+          throw new SentryAlreadyExistsException("The acl " + ent.getName() + " already exists.\n");
+        } else {
+          acls.put(ent.getName(), ent.getPermission());
+        }
       }
     }
     return acls;


Mime
View raw message