sentry-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ha...@apache.org
Subject [7/7] sentry git commit: SENTRY-1587
Date Fri, 27 Jan 2017 23:20:54 GMT
SENTRY-1587

Change-Id: Ic21fdff7d7fd54d785683f0ec3dd3598cec33f74


Project: http://git-wip-us.apache.org/repos/asf/sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/96e1d9a1
Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/96e1d9a1
Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/96e1d9a1

Branch: refs/heads/sentry-ha-redesign-1
Commit: 96e1d9a1cc272c9c044f68034648300fcf9b8477
Parents: 18be1d5
Author: hahao <hao.hao@cloudera.com>
Authored: Fri Jan 27 15:00:36 2017 -0800
Committer: hahao <hao.hao@cloudera.com>
Committed: Fri Jan 27 15:00:36 2017 -0800

----------------------------------------------------------------------
 .../db/service/persistent/SentryStore.java      | 62 +++++++++++++++++---
 .../sentry/service/thrift/HMSFollower.java      | 56 ++++++++++++++++--
 .../db/service/persistent/TestSentryStore.java  | 11 ++++
 3 files changed, 116 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sentry/blob/96e1d9a1/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java
b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java
index d1edcb1..9019350 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java
@@ -2269,13 +2269,12 @@ public class SentryStore {
     return (PathsImage) tm.executeTransaction(
     new TransactionBlock() {
       public Object execute(PersistenceManager pm) throws Exception {
-        long curChangeID = getLastProcessedPermChangeIDCore(pm);
+
+        // curChangeID could be 0, since first full snapshot is fetching
+        // from HMS, and does not have corresponding delta update.
+        long curChangeID = getLastProcessedPathChangeIDCore(pm);
         Map<String, Set<String>> pathImage = retrieveFullPathsImageCore(pm);
 
-        if (curChangeID == EMPTY_CHANGE_ID && !pathImage.isEmpty()) {
-          throw new Exception("Non-empty full paths image should not have" +
-              "an empty change ID.");
-        }
         return new PathsImage(pathImage, curChangeID);
       }
     });
@@ -2331,7 +2330,7 @@ public class SentryStore {
   private void createAuthzPathsMappingCore(PersistenceManager pm, String authzObj,
       Set<String> paths) throws SentryAlreadyExistsException {
 
-    MAuthzPathsMapping mAuthzPathsMapping = getMAuthzPathsMapping(pm, authzObj);
+    MAuthzPathsMapping mAuthzPathsMapping = getMAuthzPathsMappingCore(pm, authzObj);
 
     if (mAuthzPathsMapping == null) {
       mAuthzPathsMapping =
@@ -2342,10 +2341,59 @@ public class SentryStore {
     }
   }
 
+
+  public void updateAuthzPathsMapping(final String hiveObj, final Set<String> paths)
+        throws Exception {
+    tm.executeTransactionWithRetry (
+      new TransactionBlock() {
+        public Object execute(PersistenceManager pm) throws Exception {
+          updateAuthzPathsMappingCore(pm, hiveObj, paths);
+          return null;
+        }
+      });
+  }
+
+  public void updateAuthzPathsMapping(final String hiveObj, final Set<String> paths,
+  final DeltaTransactionBlock deltaTransactionBlock) throws Exception {
+    execute(deltaTransactionBlock, new TransactionBlock() {
+      public Object execute(PersistenceManager pm) throws Exception {
+        updateAuthzPathsMappingCore(pm, hiveObj, paths);
+        return null;
+      }
+    });
+  }
+
+  private void updateAuthzPathsMappingCore(PersistenceManager pm, String authzObj,
+    Set<String> paths) throws SentryAlreadyExistsException {
+
+    MAuthzPathsMapping mAuthzPathsMapping = getMAuthzPathsMappingCore(pm, authzObj);
+
+    if (mAuthzPathsMapping == null) {
+      mAuthzPathsMapping =
+        new MAuthzPathsMapping(authzObj, paths, System.currentTimeMillis());
+    } else {
+      mAuthzPathsMapping.getPaths().addAll(paths);
+    }
+    pm.makePersistent(mAuthzPathsMapping);
+  }
+
+  /**
+   * Get the MAuthzPathsMapping object from authzObj
+   */
+  public MAuthzPathsMapping getMAuthzPathsMapping(final String authzObj)
+      throws Exception{
+    return tm.executeTransactionWithRetry(
+    new TransactionBlock<MAuthzPathsMapping>() {
+      public MAuthzPathsMapping execute(PersistenceManager pm) throws Exception {
+        return getMAuthzPathsMappingCore(pm, authzObj);
+      }
+    });
+  }
+
   /**
    * Get the MAuthzPathsMapping object from authzObj
    */
-  public MAuthzPathsMapping getMAuthzPathsMapping(PersistenceManager pm, String authzObj)
{
+  private MAuthzPathsMapping getMAuthzPathsMappingCore(PersistenceManager pm, String authzObj)
{
     Query query = pm.newQuery(MAuthzPathsMapping.class);
     query.setFilter("this.authzObjName == t");
     query.declareParameters("java.lang.String t");

http://git-wip-us.apache.org/repos/asf/sentry/blob/96e1d9a1/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
index ad6bdda..7dce745 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
@@ -19,6 +19,7 @@ package org.apache.sentry.service.thrift;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
+import com.google.common.collect.Sets;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
@@ -32,8 +33,8 @@ import org.apache.sentry.binding.hive.conf.HiveAuthzConf;
 import org.apache.sentry.core.common.exception.*;
 import org.apache.sentry.hdfs.PathsUpdate;
 import org.apache.sentry.hdfs.PermissionsUpdate;
-import org.apache.sentry.hdfs.UpdateableAuthzPaths;
 import org.apache.sentry.hdfs.FullUpdateInitializer;
+import org.apache.sentry.hdfs.SentryMalformedPathException;
 import org.apache.sentry.hdfs.service.thrift.TPrivilegeChanges;
 import org.apache.sentry.provider.db.SentryPolicyStorePlugin;
 import org.apache.sentry.provider.db.service.persistent.DeltaTransactionBlock;
@@ -321,12 +322,12 @@ public class HMSFollower implements Runnable {
   /**
    * Throws SentryInvalidHMSEventException if Notification event contains insufficient information
    */
-  void processNotificationEvents(List<NotificationEvent> events) throws
-      SentryInvalidHMSEventException, SentryInvalidInputException {
+  void processNotificationEvents(List<NotificationEvent> events) throws Exception {
     SentryJSONMessageDeserializer deserializer = new SentryJSONMessageDeserializer();
 
     for (NotificationEvent event : events) {
-      String dbName, tableName, oldLocation, newLocation, location;
+      String dbName, tableName, oldLocation, newLocation, location, authzObj;
+      List<String> pathTree;
       switch (HCatEventMessage.EventType.valueOf(event.getEventType())) {
         case CREATE_DATABASE:
           SentryJSONCreateDatabaseMessage message = deserializer.getCreateDatabaseMessage(event.getMessage());
@@ -346,7 +347,15 @@ public class HMSFollower implements Runnable {
               throw new SentryInvalidInputException("Could not process Create database event.
Event: " + event.toString(), e);
             }
           }
-          //TODO: HDFSPlugin.addPath(dbName, location)
+          // addPath into Sentry DB. Skip update if encounter malformed path.
+          authzObj = dbName;
+          pathTree = getPath(location);
+          if (pathTree != null) {
+            DeltaTransactionBlock deltaTransactionBlock = addPaths(authzObj, pathTree);
+            Set<String> paths = Sets.newHashSet();
+            paths.add(PathsUpdate.getPath(pathTree));
+            sentryStore.updateAuthzPathsMapping(authzObj, paths, deltaTransactionBlock);
+          }
           break;
         case DROP_DATABASE:
           SentryJSONDropDatabaseMessage dropDatabaseMessage = deserializer.getDropDatabaseMessage(event.getMessage());
@@ -384,7 +393,15 @@ public class HMSFollower implements Runnable {
               throw new SentryInvalidInputException("Could not process Create table event.
Event: " + event.toString(), e);
             }
           }
-          //TODO: HDFSPlugin.deletePath(dbName, location)
+          // addPath into Sentry DB. Skip update if encounter malformed path.
+          authzObj = dbName + "." + tableName;
+          pathTree = getPath(location);
+          if (pathTree != null) {
+            DeltaTransactionBlock deltaTransactionBlock = addPaths(authzObj, pathTree);
+            Set<String> paths = Sets.newHashSet();
+            paths.add(PathsUpdate.getPath(pathTree));
+            sentryStore.updateAuthzPathsMapping(authzObj, paths, deltaTransactionBlock);
+          }
           break;
         case DROP_TABLE:
           SentryJSONDropTableMessage dropTableMessage = deserializer.getDropTableMessage(event.getMessage());
@@ -522,4 +539,31 @@ public class HMSFollower implements Runnable {
     }
     return authzObj == null ? null : authzObj.toLowerCase();
   }
+
+  private DeltaTransactionBlock addPaths(String authzObj, List<String> pathTree) {
+    LOGGER.debug("#### HMS Path Update ["
+        + "OP : addPath, "
+        + "authzObj : " + authzObj.toLowerCase() + ", "
+        + "path : " + pathTree.toString() + "]");
+    PathsUpdate update = new PathsUpdate();
+    update.newPathChange(authzObj.toLowerCase()).addToAddPaths(pathTree);
+    return new DeltaTransactionBlock(update);
+  }
+
+  /**
+   * Get path tree from a given path. It throws SentryMalformedPathException
+   * if encountered malformed path.
+   *
+   * @param path
+   * @return the path tree.
+   */
+  private List<String> getPath(String path) {
+    try {
+      return PathsUpdate.parsePath(path);
+    } catch (SentryMalformedPathException e) {
+      LOGGER.error("Unexpected path = " + path);
+      e.printStackTrace();
+      return null;
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/sentry/blob/96e1d9a1/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
index a26f996..a2d48a8 100644
--- a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
+++ b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
@@ -39,6 +39,7 @@ import org.apache.sentry.hdfs.PermissionsUpdate;
 import org.apache.sentry.hdfs.Updateable;
 import org.apache.sentry.hdfs.service.thrift.TPrivilegeChanges;
 import org.apache.sentry.hdfs.service.thrift.TRoleChanges;
+import org.apache.sentry.provider.db.service.model.MAuthzPathsMapping;
 import org.apache.sentry.provider.db.service.model.MSentryPermChange;
 import org.apache.sentry.provider.db.service.model.MSentryPrivilege;
 import org.apache.sentry.provider.db.service.model.MSentryRole;
@@ -2200,6 +2201,16 @@ public class TestSentryStore extends org.junit.Assert {
     assertEquals(Sets.newHashSet("/user/hive/warehouse/db1.db/table1"), pathImage.get("db1.table1"));
   }
 
+  @Test
+  public void testUpdateAuthzPathsMapping() throws Exception {
+    sentryStore.updateAuthzPathsMapping("db1.table", Sets.newHashSet("/user/hive/warehouse/db1.db/table1"));
+    sentryStore.updateAuthzPathsMapping("db1.table", Sets.newHashSet("/user/hive/warehouse/db1.db/table2"));
+
+    MAuthzPathsMapping mAuthzPathsMapping = sentryStore.getMAuthzPathsMapping("db1.table");
+    assertEquals(Sets.newHashSet("/user/hive/warehouse/db1.db/table1", "/user/hive/warehouse/db1.db/table2"),
+        mAuthzPathsMapping.getPaths());
+  }
+
   public void testQueryParamBuilder() {
     SentryStore.QueryParamBuilder paramBuilder;
     paramBuilder = new SentryStore.QueryParamBuilder();


Mime
View raw message