sentry-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ha...@apache.org
Subject sentry git commit: SENTRY-1325: Store HMSPaths in Sentry DB to allow fast failover (Hao Hao, Reviewed by Colin Patrick McCabe and Anne Yu)
Date Fri, 24 Jun 2016 22:19:50 GMT
Repository: sentry
Updated Branches:
  refs/heads/sentry-ha-redesign 5630fc5ce -> e7ec257a7


SENTRY-1325: Store HMSPaths in Sentry DB to allow fast failover (Hao Hao, Reviewed by Colin
Patrick McCabe and Anne Yu)

Change-Id: Idc4bb72436b6b4be098496ef074a05a9390ae135


Project: http://git-wip-us.apache.org/repos/asf/sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/e7ec257a
Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/e7ec257a
Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/e7ec257a

Branch: refs/heads/sentry-ha-redesign
Commit: e7ec257a76c7175a49e95b17ca4676dbd74a8d67
Parents: 5630fc5
Author: hahao <hao.hao@cloudera.com>
Authored: Fri Jun 24 15:18:24 2016 -0700
Committer: hahao <hao.hao@cloudera.com>
Committed: Fri Jun 24 15:18:24 2016 -0700

----------------------------------------------------------------------
 .../java/org/apache/sentry/hdfs/HMSPaths.java   |  93 +++++++++++++----
 .../org/apache/sentry/hdfs/HMSPathsDumper.java  |   2 +-
 .../db/service/model/MAuthzPathsMapping.java    | 102 +++++++++++++++++++
 .../provider/db/service/model/package.jdo       |  25 +++++
 .../db/service/persistent/SentryStore.java      |  71 +++++++++++++
 .../db/service/persistent/TestSentryStore.java  |  12 +++
 6 files changed, 287 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sentry/blob/e7ec257a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/HMSPaths.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/HMSPaths.java
b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/HMSPaths.java
index 6d2ab23..e4850f4 100644
--- a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/HMSPaths.java
+++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/HMSPaths.java
@@ -116,7 +116,7 @@ public class HMSPaths implements AuthzPaths {
 
     // Path of child element to the path entry mapping.
     // e.g. 'b' -> '/a/b'
-    private final Map<String, Entry> children;
+    private final Map<String, Entry> children = new HashMap<String, Entry>();
 
     Entry(Entry parent, String pathElement, EntryType type,
         String authzObj) {
@@ -125,7 +125,6 @@ public class HMSPaths implements AuthzPaths {
       this.pathElement = pathElement;
       this.authzObjs = new TreeSet<String>(String.CASE_INSENSITIVE_ORDER);
       addAuthzObj(authzObj);
-      children = new HashMap<String, Entry>();
     }
 
     Entry(Entry parent, String pathElement, EntryType type,
@@ -135,7 +134,6 @@ public class HMSPaths implements AuthzPaths {
       this.pathElement = pathElement;
       this.authzObjs = new TreeSet<String>(String.CASE_INSENSITIVE_ORDER);
       addAuthzObjs(authzObjs);
-      children = new HashMap<String, Entry>();
     }
 
     // Get all the mapping of the children element to
@@ -179,6 +177,66 @@ public class HMSPaths implements AuthzPaths {
           getFullPath(), type, Joiner.on(",").join(authzObjs));
     }
 
+    @Override
+    public boolean equals(Object obj) {
+      if (this == obj) {
+        return true;
+      }
+      if (obj == null) {
+        return false;
+      }
+      if (getClass() != obj.getClass()) {
+        return false;
+      }
+
+      Entry other = (Entry) obj;
+      if (parent == null) {
+        if (other.parent != null) {
+          return false;
+        }
+      } else if (!parent.equals(other.parent)) {
+        return false;
+      }
+
+      if (type == null) {
+        if (other.type != null) {
+          return false;
+        }
+      } else if (!type.equals(other.type)) {
+        return false;
+      }
+
+      if (pathElement == null) {
+        if (other.pathElement != null) {
+          return false;
+        }
+      } else if (!pathElement.equals(other.pathElement)) {
+        return false;
+      }
+
+      if (authzObjs == null) {
+        if (other.authzObjs != null) {
+          return false;
+        }
+      } else if (!authzObjs.equals(other.authzObjs)) {
+        return false;
+      }
+
+      return true;
+    }
+
+    @Override
+    public int hashCode() {
+      final int prime = 31;
+      int result = 1;
+      result = prime * result + ((parent == null) ? 0 : parent.hashCode());
+      result = prime * result + ((type == null) ? 0 : type.hashCode());
+      result = prime * result + ((pathElement == null) ? 0 : pathElement.hashCode());
+      result = prime * result + ((authzObjs == null) ? 0 : authzObjs.hashCode());
+
+      return result;
+    }
+
     /**
      * Create a child entry based on the path, type and authzObj that
      * associates with it.
@@ -332,12 +390,10 @@ public class HMSPaths implements AuthzPaths {
       return authzObjs;
     }
 
-
-
     public Entry findPrefixEntry(List<String> pathElements) {
       Preconditions.checkArgument(pathElements != null,
           "pathElements cannot be NULL");
-      return (getType() == EntryType.PREFIX) 
+      return (getType() == EntryType.PREFIX)
              ? this : findPrefixEntry(pathElements, 0);
     }
 
@@ -413,7 +469,10 @@ public class HMSPaths implements AuthzPaths {
 
   // The hive authorized objects to path entries mapping.
   // One authorized object can map to a set of path entries.
-  private Map<String, Set<Entry>> authzObjToPath;
+  private Map<String, Set<Entry>> authzObjToEntries;
+
+  public HMSPaths() {
+  }
 
   public HMSPaths(String[] pathPrefixes) {
     boolean rootPrefix = false;
@@ -433,7 +492,7 @@ public class HMSPaths implements AuthzPaths {
       }
     }
 
-    authzObjToPath = new TreeMap<String, Set<Entry>>(String.CASE_INSENSITIVE_ORDER);
+    authzObjToEntries = new TreeMap<String, Set<Entry>>(String.CASE_INSENSITIVE_ORDER);
   }
 
   void _addAuthzObject(String authzObj, List<String> authzObjPaths) {
@@ -441,7 +500,7 @@ public class HMSPaths implements AuthzPaths {
   }
 
   void addAuthzObject(String authzObj, List<List<String>> authzObjPathElements)
{
-    Set<Entry> previousEntries = authzObjToPath.get(authzObj);
+    Set<Entry> previousEntries = authzObjToEntries.get(authzObj);
     Set<Entry> newEntries = new HashSet<Entry>(authzObjPathElements.size());
     for (List<String> pathElements : authzObjPathElements) {
       Entry e = root.createAuthzObjPath(pathElements, authzObj);
@@ -451,7 +510,7 @@ public class HMSPaths implements AuthzPaths {
         LOG.warn("Ignoring path, no prefix");
       }
     }
-    authzObjToPath.put(authzObj, newEntries);
+    authzObjToEntries.put(authzObj, newEntries);
     if (previousEntries != null) {
       previousEntries.removeAll(newEntries);
       if (!previousEntries.isEmpty()) {
@@ -464,7 +523,7 @@ public class HMSPaths implements AuthzPaths {
 
   void addPathsToAuthzObject(String authzObj,
       List<List<String>> authzObjPathElements, boolean createNew) {
-    Set<Entry> entries = authzObjToPath.get(authzObj);
+    Set<Entry> entries = authzObjToEntries.get(authzObj);
     if (entries != null) {
       Set<Entry> newEntries = new HashSet<Entry>(authzObjPathElements.size());
       for (List<String> pathElements : authzObjPathElements) {
@@ -501,7 +560,7 @@ public class HMSPaths implements AuthzPaths {
    */
   void deletePathsFromAuthzObject(String authzObj,
       List<List<String>> authzObjPathElements) {
-    Set<Entry> entries = authzObjToPath.get(authzObj);
+    Set<Entry> entries = authzObjToEntries.get(authzObj);
     if (entries != null) {
       Set<Entry> toDelEntries = new HashSet<Entry>(authzObjPathElements.size());
       for (List<String> pathElements : authzObjPathElements) {
@@ -522,7 +581,7 @@ public class HMSPaths implements AuthzPaths {
   }
 
   void deleteAuthzObject(String authzObj) {
-    Set<Entry> entries = authzObjToPath.remove(authzObj);
+    Set<Entry> entries = authzObjToEntries.remove(authzObj);
     if (entries != null) {
       for (Entry entry : entries) {
         entry.deleteAuthzObject(authzObj);
@@ -593,12 +652,12 @@ public class HMSPaths implements AuthzPaths {
       }
     }
     if(samePaths) {
-      Set<Entry> eSet = authzObjToPath.get(oldName);
+      Set<Entry> eSet = authzObjToEntries.get(oldName);
       if (eSet == null) {
         LOG.warn("Unexpected state in renameAuthzObject, cannot find oldName in authzObjToPath:
oldName=" + oldName + " newName=" + newName +
                 " oldPath=" + oldPathElems + " newPath=" + newPathElems);
       } else {
-        authzObjToPath.put(newName, eSet);
+        authzObjToEntries.put(newName, eSet);
         for (Entry e : eSet) {
           if (e.getAuthzObjs().contains(oldName)) {
             e.addAuthzObj(newName);
@@ -632,8 +691,8 @@ public class HMSPaths implements AuthzPaths {
     this.root = root;
   }
 
-  void setAuthzObjToPathMapping(Map<String, Set<Entry>> mapping) {
-    authzObjToPath = mapping;
+  void setAuthzObjToEntryMapping(Map<String, Set<Entry>> mapping) {
+    authzObjToEntries = mapping;
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/sentry/blob/e7ec257a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/HMSPathsDumper.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/HMSPathsDumper.java
b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/HMSPathsDumper.java
index e759ff1..7e56952 100644
--- a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/HMSPathsDumper.java
+++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/HMSPathsDumper.java
@@ -86,7 +86,7 @@ public class HMSPathsDumper implements AuthzPathsDumper<HMSPaths>
{
     cloneToEntry(tRootEntry, rootEntry, pathDump.getNodeMap(), authzObjToPath,
         rootEntry.getType() == EntryType.PREFIX);
     newHmsPaths.setRootEntry(rootEntry);
-    newHmsPaths.setAuthzObjToPathMapping(authzObjToPath);
+    newHmsPaths.setAuthzObjToEntryMapping(authzObjToPath);
 
     return newHmsPaths;
   }

http://git-wip-us.apache.org/repos/asf/sentry/blob/e7ec257a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MAuthzPathsMapping.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MAuthzPathsMapping.java
b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MAuthzPathsMapping.java
new file mode 100644
index 0000000..56e456e
--- /dev/null
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MAuthzPathsMapping.java
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.provider.db.service.model;
+
+import javax.jdo.annotations.PersistenceCapable;
+import java.util.Set;
+
+/**
+ * Transactional database backend for storing HMS Paths Updates. Any changes to this object
+ * require re-running the maven build so DN can re-enhance.
+ */
+@PersistenceCapable
+public class MAuthzPathsMapping {
+
+  private String authzObjName;
+  private Set<String> paths;
+  private long createTimeMs;
+
+  public MAuthzPathsMapping(String authzObjName, Set<String> paths, long createTimeMs)
{
+    this.authzObjName = authzObjName;
+    this.paths = paths;
+    this.createTimeMs = createTimeMs;
+  }
+
+  public long getCreateTime() {
+    return createTimeMs;
+  }
+
+  public void setCreateTime(long createTime) {
+    this.createTimeMs = createTime;
+  }
+
+  public String getAuthzObjName() {
+    return authzObjName;
+  }
+
+  public void setAuthzObjName(String authzObjName) {
+    this.authzObjName = authzObjName;
+  }
+
+  public void setPaths(Set<String> paths) {
+    this.paths = paths;
+  }
+
+  public Set<String> getPaths() {
+    return paths;
+  }
+
+  @Override
+  public String toString() {
+    return "MSentryPathsUpdate [authzObj=" + authzObjName + "], paths=[" + paths.toString()
+        + "], createTimeMs=[" + String.valueOf(createTimeMs) + "]";
+  }
+
+  @Override
+  public int hashCode() {
+    return authzObjName == null ? 0 : authzObjName.hashCode();
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj) {
+      return true;
+    }
+
+    if (obj == null) {
+      return false;
+    }
+
+    if (getClass() != obj.getClass()) {
+      return false;
+    }
+
+    MAuthzPathsMapping other = (MAuthzPathsMapping) obj;
+
+    if (authzObjName == null) {
+      if (other.authzObjName != null) {
+        return false;
+      }
+    } else if (!authzObjName.equals(other.authzObjName)) {
+      return false;
+    }
+
+    return true;
+  }
+}

http://git-wip-us.apache.org/repos/asf/sentry/blob/e7ec257a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo
b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo
index b3b9494..fb5470f 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo
@@ -237,6 +237,31 @@
       </field>
      </class>
 
+     <class name="MAuthzPathsMapping" identity-type="datastore" table="AUTHZ_PATHS_MAPPING"
detachable="true">
+       <datastore-identity>
+         <column name="AUTHZ_OBJ_ID"/>
+       </datastore-identity>
+       <!--
+         authzObjName is composed by hive database name, and table name. e.g. "default.tb1".
Since
+         both hive database name, and table name have restrictions to be at most 128 characters
long,
+         384 characters length should be enough for AUTHZ_OBJ_NAM.
+       -->
+       <field name="authzObjName">
+         <column name="AUTHZ_OBJ_NAME" length="384" jdbc-type="VARCHAR"/>
+         <index name="AuthzObjName" unique="true"/>
+       </field>
+       <field name="createTimeMs">
+         <column name="CREATE_TIME_MS" jdbc-type="BIGINT"/>
+       </field>
+       <field name = "paths">
+         <collection element-type="java.lang.String"/>
+             <join/>
+             <element>
+                 <column name="PATHS"/>
+             </element>
+       </field>
+     </class>
+
   </package>
 </jdo>
 

http://git-wip-us.apache.org/repos/asf/sentry/blob/e7ec257a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java
b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java
index b7ef0e9..7dad496 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java
@@ -56,6 +56,7 @@ import org.apache.sentry.core.common.exception.SentryAlreadyExistsException;
 import org.apache.sentry.core.common.exception.SentryGrantDeniedException;
 import org.apache.sentry.core.common.exception.SentryInvalidInputException;
 import org.apache.sentry.core.common.exception.SentryNoSuchObjectException;
+import org.apache.sentry.provider.db.service.model.MAuthzPathsMapping;
 import org.apache.sentry.provider.db.service.model.MSentryGroup;
 import org.apache.sentry.provider.db.service.model.MSentryPrivilege;
 import org.apache.sentry.provider.db.service.model.MSentryRole;
@@ -1986,6 +1987,76 @@ public class SentryStore {
   }
 
   /**
+   * This returns a Mapping of Authz -> [Paths]
+   */
+  public Map<String, Set<String>> retrieveFullPathsImage() {
+    Map<String, Set<String>> retVal = new HashMap<>();
+    boolean rollbackTransaction = true;
+    PersistenceManager pm = null;
+
+    try {
+      pm = openTransaction();
+      Query query = pm.newQuery(MAuthzPathsMapping.class);
+      List<MAuthzPathsMapping> authzToPathsMappings = (List<MAuthzPathsMapping>)
query.execute();
+      for (MAuthzPathsMapping authzToPaths : authzToPathsMappings) {
+        retVal.put(authzToPaths.getAuthzObjName(), authzToPaths.getPaths());
+      }
+
+      rollbackTransaction = false;
+      commitTransaction(pm);
+      return retVal;
+    } finally {
+      if (rollbackTransaction) {
+        rollbackTransaction(pm);
+      }
+    }
+  }
+
+  public CommitContext createAuthzPathsMapping(String hiveObj,
+      Set<String> paths) throws SentryNoSuchObjectException, SentryAlreadyExistsException
{
+
+    boolean rollbackTransaction = true;
+    PersistenceManager pm = null;
+
+    try {
+      pm = openTransaction();
+      createAuthzPathsMappingCore(pm, hiveObj, paths);
+      CommitContext commit = commitUpdateTransaction(pm);
+      rollbackTransaction = false;
+      return commit;
+    } finally {
+      if (rollbackTransaction) {
+        rollbackTransaction(pm);
+      }
+    }
+  }
+
+  private void createAuthzPathsMappingCore(PersistenceManager pm, String authzObj,
+      Set<String> paths) throws SentryAlreadyExistsException {
+
+    MAuthzPathsMapping mAuthzPathsMapping = getMAuthzPathsMapping(pm, authzObj);
+
+    if (mAuthzPathsMapping == null) {
+      mAuthzPathsMapping =
+          new MAuthzPathsMapping(authzObj, paths, System.currentTimeMillis());
+      pm.makePersistent(mAuthzPathsMapping);
+    } else {
+      throw new SentryAlreadyExistsException("AuthzObj: " + authzObj);
+    }
+  }
+
+  /**
+   * Get the MAuthzPathsMapping object from authzObj
+   */
+  public MAuthzPathsMapping getMAuthzPathsMapping(PersistenceManager pm, String authzObj)
{
+    Query query = pm.newQuery(MAuthzPathsMapping.class);
+    query.setFilter("this.authzObjName == t");
+    query.declareParameters("java.lang.String t");
+    query.setUnique(true);
+    return (MAuthzPathsMapping) query.execute(authzObj);
+  }
+
+  /**
    * This thread exists to clean up "orphaned" privilege rows in the database.
    * These rows aren't removed automatically due to the fact that there is
    * a many-to-many mapping between the roles and privileges, and the

http://git-wip-us.apache.org/repos/asf/sentry/blob/e7ec257a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
index bc7fe12..3ef1cf7 100644
--- a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
+++ b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
@@ -2079,6 +2079,18 @@ public class TestSentryStore extends org.junit.Assert {
     }
   }
 
+  @Test
+  public void testAuthzPathsMapping() throws Exception {
+    long seqId = sentryStore.createAuthzPathsMapping("db1.table1", Sets.newHashSet("/user/hive/warehouse/db1.db/table1")).getSequenceId();
+    long actualSeqId = sentryStore.createAuthzPathsMapping("db1.table2", Sets.newHashSet("/user/hive/warehouse/db1.db/table2")).getSequenceId();
+    assertEquals(seqId + 1, actualSeqId);
+
+    Map<String, Set<String>> pathsImage = sentryStore.retrieveFullPathsImage();
+    assertEquals(2, pathsImage.size());
+    assertEquals(Sets.newHashSet("/user/hive/warehouse/db1.db/table1"), pathsImage.get("db1.table1"));
+  }
+
+
   protected static void addGroupsToUser(String user, String... groupNames) {
     policyFile.addGroupsToUser(user, groupNames);
   }


Mime
View raw message