sentry-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ha...@apache.org
Subject sentry git commit: SENTRY-1611: Purge MSentryPerm/PathChange tables (Lei (Eddy) Xu, Reviewed by: Hao Hao and Alexander Kolbasov)
Date Wed, 22 Feb 2017 19:30:45 GMT
Repository: sentry
Updated Branches:
  refs/heads/sentry-ha-redesign 3817f18be -> b596fe9b8


SENTRY-1611: Purge MSentryPerm/PathChange tables (Lei (Eddy) Xu, Reviewed by: Hao Hao and
Alexander Kolbasov)

Change-Id: I59ec5b83abd7f86cadc434e46caf9be142dd3ab6


Project: http://git-wip-us.apache.org/repos/asf/sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/b596fe9b
Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/b596fe9b
Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/b596fe9b

Branch: refs/heads/sentry-ha-redesign
Commit: b596fe9b884db583215a58a3309677590969b71f
Parents: 3817f18
Author: hahao <hao.hao@cloudera.com>
Authored: Wed Feb 22 11:28:49 2017 -0800
Committer: hahao <hao.hao@cloudera.com>
Committed: Wed Feb 22 11:28:49 2017 -0800

----------------------------------------------------------------------
 .../db/service/model/MSentryChange.java         |  24 ++++
 .../db/service/model/MSentryPathChange.java     |   2 +-
 .../db/service/model/MSentryPermChange.java     |   2 +-
 .../db/service/persistent/SentryStore.java      | 117 ++++++++++++++++---
 .../db/service/persistent/TestSentryStore.java  |  33 ++++++
 5 files changed, 162 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sentry/blob/b596fe9b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryChange.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryChange.java
b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryChange.java
new file mode 100644
index 0000000..6011ef4
--- /dev/null
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryChange.java
@@ -0,0 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sentry.provider.db.service.model;
+
+/**
+ * The base class for various delta changes stored in Sentry DB.
+ */
+public interface MSentryChange {
+}

http://git-wip-us.apache.org/repos/asf/sentry/blob/b596fe9b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPathChange.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPathChange.java
b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPathChange.java
index 12e9a09..0ca7fe2 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPathChange.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPathChange.java
@@ -57,7 +57,7 @@ import javax.jdo.annotations.PrimaryKey;
  */
 
 @PersistenceCapable
-public class MSentryPathChange {
+public class MSentryPathChange implements MSentryChange {
 
   @PrimaryKey
   private long changeID;

http://git-wip-us.apache.org/repos/asf/sentry/blob/b596fe9b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPermChange.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPermChange.java
b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPermChange.java
index 3662bfd..476fbcb 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPermChange.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPermChange.java
@@ -55,7 +55,7 @@ import javax.jdo.annotations.PrimaryKey;
  * can re-enhance.
  */
 @PersistenceCapable
-public class MSentryPermChange {
+public class MSentryPermChange implements MSentryChange {
 
   @PrimaryKey
   private long changeID;

http://git-wip-us.apache.org/repos/asf/sentry/blob/b596fe9b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java
b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java
index 4cc123f..2ae9f3b 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java
@@ -425,6 +425,8 @@ public class SentryStore {
               pm.newQuery(MSentryGroup.class).deletePersistentAll();
               pm.newQuery(MSentryUser.class).deletePersistentAll();
               pm.newQuery(MSentryPrivilege.class).deletePersistentAll();
+              pm.newQuery(MSentryPermChange.class).deletePersistentAll();
+              pm.newQuery(MSentryPathChange.class).deletePersistentAll();
               return null;
             }
           });
@@ -435,6 +437,56 @@ public class SentryStore {
   }
 
   /**
+   * Purge a given delta change table, with a specified number of changes to be kept.
+   *
+   * @param cls the class of a perm/path delta change {@link MSentryPermChange} or
+   *            {@link MSentryPathChange}.
+   * @param pm a {@link PersistenceManager} instance.
+   * @param changesToKeep the number of changes the caller want to keep.
+   * @param <T> the type of delta change class.
+   */
+  @VisibleForTesting
+  <T extends MSentryChange> void purgeDeltaChangeTableCore(
+      Class<T> cls, PersistenceManager pm, long changesToKeep) {
+    Preconditions.checkArgument(changesToKeep >= 0,
+        "changes to keep must be a non-negative number");
+    long lastChangedID = getLastProcessedChangeIDCore(pm, cls);
+    long maxIDDeleted = lastChangedID - changesToKeep;
+
+    Query query = pm.newQuery(cls);
+
+    // It is an approximation of "SELECT ... LIMIT CHANGE_TO_KEEP" in SQL, because JDO w/
derby
+    // does not support "LIMIT".
+    // See: http://www.datanucleus.org/products/datanucleus/jdo/jdoql_declarative.html
+    query.setFilter("changeID <= maxChangedIdDeleted");
+    query.declareParameters("long maxChangedIdDeleted");
+    long numDeleted = query.deletePersistentAll(maxIDDeleted);
+    LOGGER.info(String.format("Purged %d of %s to changeID=%d",
+        numDeleted, cls.getSimpleName(), maxIDDeleted));
+  }
+
+  /**
+   * Purge delta change tables, {@link MSentryPermChange} and {@link MSentryPathChange}.
+   */
+  public void purgeDeltaChangeTables() {
+    LOGGER.info("Purging MSentryPathUpdate and MSentyPermUpdate tables.");
+    try {
+      tm.executeTransaction(new TransactionBlock<Object>() {
+        @Override
+        public Object execute(PersistenceManager pm) throws Exception {
+          purgeDeltaChangeTableCore(MSentryPermChange.class, pm, 1);
+          LOGGER.info("MSentryPermChange table has been purged.");
+          purgeDeltaChangeTableCore(MSentryPathChange.class, pm, 1);
+          LOGGER.info("MSentryPathUpdate table has been purged.");
+          return null;
+        }
+      });
+    } catch (Exception e) {
+      LOGGER.error("Delta change cleaning process encounter an error.", e);
+    }
+  }
+
+  /**
    * Alter a given sentry role to grant a privilege.
    *
    * @param grantorPrincipal User name
@@ -3289,34 +3341,34 @@ public class SentryStore {
   }
 
   /**
-   * Get the last processed perm change ID.
+   * Get the last processed change ID for perm/path delta changes.
    *
    * @param pm the PersistenceManager
-   * @return the last processed perm changedID
+   * @param changeCls the class of a delta c
+   *
+   * @return the last processed changedID for the delta changes.
    */
-  private long getLastProcessedPermChangeIDCore(PersistenceManager pm) {
-    Query query = pm.newQuery(MSentryPermChange.class);
-    query.setResult("max(this.changeID)");
+  private <T extends MSentryChange> long getLastProcessedChangeIDCore(
+      PersistenceManager pm, Class<T> changeCls) {
+    Query query = pm.newQuery(changeCls);
+    query.setResult("max(changeID)");
     Long changeID = (Long) query.execute();
-    if (changeID == null) {
-      return EMPTY_CHANGE_ID;
-    } else {
-      return changeID;
-    }
+    return changeID == null ? EMPTY_CHANGE_ID : changeID;
   }
 
   /**
-   * Get the MSentryPermChange object by ChangeID. Internally invoke
-   * getLastProcessedPermChangeIDCore().
+   * Get the last processed change ID for perm delta changes.
    *
-   * @return MSentryPermChange
+   * Internally invoke {@link #getLastProcessedChangeIDCore(PersistenceManager, Class)}
+   *
+   * @return latest perm change ID.
    */
   @VisibleForTesting
   long getLastProcessedPermChangeID() throws Exception {
     return tm.executeTransaction(
       new TransactionBlock<Long>() {
         public Long execute(PersistenceManager pm) throws Exception {
-          return getLastProcessedPermChangeIDCore(pm);
+          return getLastProcessedChangeIDCore(pm, MSentryPermChange.class);
         }
       });
   }
@@ -3348,6 +3400,35 @@ public class SentryStore {
   }
 
   /**
+   * Fetch all {@link MSentryChange} in the database.
+   *
+   * @param cls the class of the Sentry delta change.
+   * @return a list of Sentry delta changes.
+   * @throws Exception
+   */
+  @SuppressWarnings("unchecked")
+  private <T extends MSentryChange> List<T> getMSentryChanges(final Class<T>
cls) throws Exception {
+    return tm.executeTransaction(
+        new TransactionBlock<List<T>>() {
+          public List<T> execute(PersistenceManager pm) throws Exception {
+            Query query = pm.newQuery(cls);
+            return (List<T>) query.execute();
+          }
+        });
+  }
+
+  /**
+   * Fetch all {@link MSentryPermChange} in the database. It should only be used in the tests.
+   *
+   * @return a list of permission changes.
+   * @throws Exception
+   */
+  @VisibleForTesting
+  List<MSentryPermChange> getMSentryPermChanges() throws Exception {
+    return getMSentryChanges(MSentryPermChange.class);
+  }
+
+  /**
    * Get the MSentryPathChange object by ChangeID.
    */
   public MSentryPathChange getMSentryPathChangeByID(final long changeID) throws Exception
{
@@ -3371,6 +3452,14 @@ public class SentryStore {
   }
 
   /**
+   * Fetch all {@link MSentryPathChange} in the database. It should only be used in the tests.
+   */
+  @VisibleForTesting
+  List<MSentryPathChange> getMSentryPathChanges() throws Exception {
+    return getMSentryChanges(MSentryPathChange.class);
+  }
+
+  /**
    * Execute Perm/Path UpdateTransaction and corresponding actual
    * action transaction, e.g dropSentryRole, in a single transaction.
    * The order of the transaction does not matter because there is no

http://git-wip-us.apache.org/repos/asf/sentry/blob/b596fe9b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
index dfaac15..d9fce69 100644
--- a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
+++ b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
@@ -2479,4 +2479,37 @@ public class TestSentryStore extends org.junit.Assert {
     policyFile.write(policyFilePath);
   }
 
+  @Test
+  public void testPurgeDeltaChanges() throws Exception {
+    String role = "purgeRole";
+    String grantor = "g1";
+    String table = "purgeTable";
+
+    assertEquals(0, sentryStore.getMSentryPermChanges().size());
+    assertEquals(0, sentryStore.getMSentryPathChanges().size());
+
+    sentryStore.createSentryRole(role);
+
+    final int numPermChanges = 5;
+    for (int i = 0; i < numPermChanges; i++) {
+      TSentryPrivilege privilege = new TSentryPrivilege();
+      privilege.setPrivilegeScope("Column");
+      privilege.setServerName("server");
+      privilege.setDbName("db");
+      privilege.setTableName(table);
+      privilege.setColumnName("column");
+      privilege.setAction(AccessConstants.SELECT);
+      privilege.setCreateTime(System.currentTimeMillis());
+
+      PermissionsUpdate update = new PermissionsUpdate(i + 1, false);
+      sentryStore.alterSentryRoleGrantPrivilege(grantor, role, privilege, update);
+    }
+    assertEquals(numPermChanges, sentryStore.getMSentryPermChanges().size());
+
+    sentryStore.purgeDeltaChangeTables();
+    assertEquals(1, sentryStore.getMSentryPermChanges().size());
+
+    // TODO: verify MSentryPathChange being purged.
+    // assertEquals(1, sentryStore.getMSentryPathChanges().size());
+  }
 }


Mime
View raw message