sentry-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kal...@apache.org
Subject sentry git commit: SENTRY-2423: Increase the allocation size for auto-increment of id's for Snapshot tables. (Kalyan Kumar Kalvagadda reviewed by Arjun Mishra and Sergio Pena)
Date Tue, 09 Oct 2018 14:35:54 GMT
Repository: sentry
Updated Branches:
  refs/heads/master 2c9a927a9 -> 74676535b


SENTRY-2423: Increase the allocation size for auto-increment of id's for Snapshot tables.
(Kalyan Kumar Kalvagadda reviewed by Arjun Mishra and Sergio Pena)


Project: http://git-wip-us.apache.org/repos/asf/sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/74676535
Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/74676535
Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/74676535

Branch: refs/heads/master
Commit: 74676535b066b48939ae9ad9c1a1061986450f3d
Parents: 2c9a927
Author: Kalyan Kumar Kalvagadda <kkalyan@cloudera.com>
Authored: Tue Oct 9 09:32:49 2018 -0500
Committer: Kalyan Kumar Kalvagadda <kkalyan@cloudera.com>
Committed: Tue Oct 9 09:32:49 2018 -0500

----------------------------------------------------------------------
 .../sentry/service/common/ServiceConstants.java |  8 ++++++
 .../provider/db/service/model/package.jdo       |  4 +--
 .../db/service/persistent/SentryStore.java      |  3 ++
 .../db/service/persistent/TestSentryStore.java  | 30 ++++++++++++++++++++
 4 files changed, 43 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sentry/blob/74676535/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/service/common/ServiceConstants.java
----------------------------------------------------------------------
diff --git a/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/service/common/ServiceConstants.java
b/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/service/common/ServiceConstants.java
index e90fe2d..092060c 100644
--- a/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/service/common/ServiceConstants.java
+++ b/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/service/common/ServiceConstants.java
@@ -256,6 +256,14 @@ public class ServiceConstants {
      */
     public static final String SENTRY_DB_EXPLICIT_GRANTS_PERMITTED = "sentry.db.explicit.grants.permitted";
     public static final String SENTRY_DB_EXPLICIT_GRANTS_PERMITTED_DEFAULT = "";
+
+    /**
+     * This value sets the allocation size used by datanucleus for the values it auto generates.
+     * This is used when the strategy is explicitly mentioned in JDO. With native(default)
this configuration
+     * is not used.
+     */
+    public static final String SENTRY_DB_VALUE_GENERATION_ALLOCATION_SIZE = "sentry.db.valuegeneration.allocation.size";
+    public static final int SENTRY_DB_VALUE_GENERATION_ALLOCATION_SIZE_DEFAULT = 100;
   }
 
   public static final String SENTRY_ZK_JAAS_NAME = "Sentry";

http://git-wip-us.apache.org/repos/asf/sentry/blob/74676535/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo
----------------------------------------------------------------------
diff --git a/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo
b/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo
index 6539e33..20ec0de 100644
--- a/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo
+++ b/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo
@@ -256,7 +256,7 @@
     </class>
 
     <class name="MAuthzPathsMapping" identity-type="datastore" table="AUTHZ_PATHS_MAPPING"
detachable="true">
-       <datastore-identity>
+       <datastore-identity strategy="increment">
          <column name="AUTHZ_OBJ_ID"/>
        </datastore-identity>
        <index name="AUTHZ_SNAPSHOT_ID_INDEX" unique="false">
@@ -288,7 +288,7 @@
      </class>
 
     <class name="MPath" identity-type="datastore" table="AUTHZ_PATH" detachable="true">
-      <datastore-identity>
+      <datastore-identity strategy="increment">
         <column name="PATH_ID"/>
       </datastore-identity>
       <field name="path">

http://git-wip-us.apache.org/repos/asf/sentry/blob/74676535/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java
----------------------------------------------------------------------
diff --git a/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java
b/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java
index 1722109..7a736ca 100644
--- a/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java
+++ b/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java
@@ -249,6 +249,9 @@ public class SentryStore implements SentryStoreInterface {
     // Disallow operations outside of transactions
     prop.setProperty("datanucleus.NontransactionalRead", "false");
     prop.setProperty("datanucleus.NontransactionalWrite", "false");
+    int allocationSize = conf.getInt(ServerConfig.SENTRY_DB_VALUE_GENERATION_ALLOCATION_SIZE,
ServerConfig.
+            SENTRY_DB_VALUE_GENERATION_ALLOCATION_SIZE_DEFAULT);
+    prop.setProperty("datanucleus.valuegeneration.increment.allocationSize", Integer.toString(allocationSize));
     return prop;
   }
 

http://git-wip-us.apache.org/repos/asf/sentry/blob/74676535/sentry-service/sentry-service-server/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
----------------------------------------------------------------------
diff --git a/sentry-service/sentry-service-server/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
b/sentry-service/sentry-service-server/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
index a299e00..4a9afe3 100644
--- a/sentry-service/sentry-service-server/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
+++ b/sentry-service/sentry-service-server/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
@@ -4571,6 +4571,36 @@ public class TestSentryStore extends org.junit.Assert {
     assertEquals(0, allPrivileges.get(USER3).size());
   }
 
+  @Test
+  public void testPersistFullPathsImageWithHugeData() throws Exception {
+    Map<String, Collection<String>> authzPaths = new HashMap<>();
+    String[] prefixes = {"/user/hive/warehouse"};
+    // Makes sure that authorizable object could be associated
+    // with different paths and can be properly persisted into database.
+    for(int db_index = 1 ; db_index <= 10; db_index++) {
+      String db_name = "db" + db_index;
+      for( int table_index = 1; table_index <= 15; table_index++) {
+        Set<String> paths = Sets.newHashSet();
+        String table_name = "tb" + table_index;
+        String location = "/u/h/w/" + db_name + "/" + table_name;
+        for (int part_index = 1; part_index <= 30; part_index++) {
+          paths.add(location + "/" + part_index);
+        }
+        authzPaths.put(db_name+table_name, paths);
+      }
+    }
+    long notificationID = 110000;
+    sentryStore.persistFullPathsImage(authzPaths, notificationID);
+    PathsUpdate pathsUpdate = sentryStore.retrieveFullPathsImageUpdate(prefixes);
+    long savedNotificationID = sentryStore.getLastProcessedNotificationID();
+    assertEquals(1, pathsUpdate.getImgNum());
+    TPathsDump pathDump = pathsUpdate.toThrift().getPathsDump();
+    assertNotNull(pathDump);
+    Map<Integer, TPathEntry> nodeMap = pathDump.getNodeMap();
+    assertTrue(nodeMap.size() > 0);
+    assertEquals(notificationID, savedNotificationID);
+  }
+
   private TSentryPrivilege toTSentryPrivilege(String action, String scope, String server,
     String dbName, String tableName) {
     TSentryPrivilege privilege = new TSentryPrivilege();


Mime
View raw message