sentry-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ha...@apache.org
Subject sentry git commit: SENTRY-1566: Make full Perm/Path snapshot available for NN plugin (Hao Hao, Reviewed by: Alexander Kolbasov and Lei Xu)
Date Fri, 10 Mar 2017 19:15:48 GMT
Repository: sentry
Updated Branches:
  refs/heads/sentry-ha-redesign 731a5d15e -> 8f5c17dbc


SENTRY-1566: Make full Perm/Path snapshot available for NN plugin (Hao Hao, Reviewed by: Alexander Kolbasov and Lei Xu)

Change-Id: I2788046192b71a3988f5d8bf1fa903eff18a7c8d


Project: http://git-wip-us.apache.org/repos/asf/sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/8f5c17db
Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/8f5c17db
Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/8f5c17db

Branch: refs/heads/sentry-ha-redesign
Commit: 8f5c17dbce6417b0c98ba6c2c31fa9c7fd8b222f
Parents: 731a5d1
Author: hahao <hao.hao@cloudera.com>
Authored: Thu Mar 9 15:37:38 2017 -0800
Committer: hahao <hao.hao@cloudera.com>
Committed: Fri Mar 10 11:14:17 2017 -0800

----------------------------------------------------------------------
 .../org/apache/sentry/hdfs/ImageRetriever.java  |  45 ++++
 .../org/apache/sentry/hdfs/PathsUpdate.java     |   3 +-
 .../apache/sentry/hdfs/PathImageRetriever.java  |  72 ++++++
 .../apache/sentry/hdfs/PermImageRetriever.java  |  93 ++++++++
 .../sentry/hdfs/SentryHdfsMetricsUtil.java      |  27 ++-
 .../org/apache/sentry/hdfs/SentryPlugin.java    |  53 +----
 .../org/apache/sentry/hdfs/UpdateForwarder.java |  32 ++-
 .../sentry/hdfs/UpdateablePermissions.java      |   7 +-
 .../apache/sentry/hdfs/TestUpdateForwarder.java |  15 +-
 .../db/service/persistent/PathsImage.java       |  48 ++++
 .../db/service/persistent/PermissionsImage.java |  57 +++++
 .../db/service/persistent/SentryStore.java      | 219 ++++++++++++-------
 .../db/service/persistent/TestSentryStore.java  |  56 ++++-
 13 files changed, 545 insertions(+), 182 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sentry/blob/8f5c17db/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/ImageRetriever.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/ImageRetriever.java b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/ImageRetriever.java
new file mode 100644
index 0000000..0e40756
--- /dev/null
+++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/ImageRetriever.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sentry.hdfs;
+
+import static org.apache.sentry.hdfs.Updateable.Update;
+
+/**
+ * ImageRetriever obtains a complete snapshot of either Sentry Permissions
+ * ({@code PermissionsUpdate}) or Sentry representation of Hive Paths
+ * ({@code PathsUpdate}).
+ * <p>
+ * The snapshot image should represent a consistent state.
+ * The {@link #retrieveFullImage(long)} method obtains such state snapshot from
+ * a persistent storage.
+ * The Snapshots are propagated to a consumer of Sentry, such as HDFS NameNode,
+ * whenever the consumer needs to synchronize its full state.
+ */
+public interface ImageRetriever<K extends Update> {
+
+  /**
+   * Retrieve a complete snapshot of type {@code Update} from a persistent storage.
+   *
+   * @param seqNum
+   * @return a complete snapshot of type {@link Update}, e.g {@link PermissionsUpdate}
+   *         or {@link PathsUpdate}
+   * @throws Exception
+   */
+  K retrieveFullImage(long seqNum) throws Exception;
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/sentry/blob/8f5c17db/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java
index 992c8b7..14e967a 100644
--- a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java
+++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -39,7 +39,6 @@ import org.apache.thrift.TException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-
 /**
  * A wrapper class over the TPathsUpdate thrift generated class. Please see
  * {@link Updateable.Update} for more information

http://git-wip-us.apache.org/repos/asf/sentry/blob/8f5c17db/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathImageRetriever.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathImageRetriever.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathImageRetriever.java
new file mode 100644
index 0000000..16a1604
--- /dev/null
+++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathImageRetriever.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sentry.hdfs;
+
+import com.codahale.metrics.Timer;
+import org.apache.sentry.hdfs.service.thrift.TPathChanges;
+import org.apache.sentry.provider.db.service.persistent.PathsImage;
+import org.apache.sentry.provider.db.service.persistent.SentryStore;
+
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * PathImageRetriever obtains a complete snapshot of Hive Paths from a persistent
+ * storage and translate it into {@code PathsUpdate} that the consumers, such as
+ * HDFS NameNod, can understand.
+ */
+public class PathImageRetriever implements ImageRetriever<PathsUpdate> {
+
+  private final SentryStore sentryStore;
+
+  PathImageRetriever(SentryStore sentryStore) {
+    this.sentryStore = sentryStore;
+  }
+
+  @Override
+  public PathsUpdate retrieveFullImage(long seqNum) throws Exception {
+    try (final Timer.Context timerContext =
+        SentryHdfsMetricsUtil.getRetrievePathFullImageTimer.time()) {
+
+      // Reads a up-to-date complete snapshot of Hive paths from the
+      // persistent storage, along with the sequence number of latest
+      // delta change the snapshot corresponds to.
+      PathsImage pathsImage = sentryStore.retrieveFullPathsImage();
+      long curSeqNum = pathsImage.getCurSeqNum();
+      Map<String, Set<String>> pathImage = pathsImage.getPathImage();
+
+      // Translates the complete Hive paths snapshot into a PathsUpdate.
+      // Adds all <hiveObj, paths> mapping to be included in this paths update.
+      // And label it with the latest delta change sequence number for consumer
+      // to be aware of the next delta change it should continue with.
+      // TODO: use curSeqNum from DB instead of seqNum when doing SENTRY-1613
+      PathsUpdate pathsUpdate = new PathsUpdate(seqNum, true);
+      for (Map.Entry<String, Set<String>> pathEnt : pathImage.entrySet()) {
+        TPathChanges pathChange = pathsUpdate.newPathChange(pathEnt.getKey());
+
+        for (String path : pathEnt.getValue()) {
+          pathChange.addToAddPaths(PathsUpdate.splitPath(path));
+        }
+      }
+
+      SentryHdfsMetricsUtil.getPathChangesHistogram.update(pathsUpdate
+            .getPathChanges().size());
+      return pathsUpdate;
+    }
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/sentry/blob/8f5c17db/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermImageRetriever.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermImageRetriever.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermImageRetriever.java
new file mode 100644
index 0000000..3017c9e
--- /dev/null
+++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermImageRetriever.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sentry.hdfs;
+
+import com.codahale.metrics.Timer;
+import org.apache.sentry.hdfs.service.thrift.TPermissionsUpdate;
+import org.apache.sentry.hdfs.service.thrift.TPrivilegeChanges;
+import org.apache.sentry.hdfs.service.thrift.TRoleChanges;
+import org.apache.sentry.provider.db.service.persistent.PermissionsImage;
+import org.apache.sentry.provider.db.service.persistent.SentryStore;
+
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * PermImageRetriever obtains a complete snapshot of Sentry permission from a persistent
+ * storage and translate it into {@code PermissionsUpdate} that the consumers, such as
+ * HDFS NameNod, can understand.
+ */
+public class PermImageRetriever implements ImageRetriever<PermissionsUpdate> {
+
+  private final SentryStore sentryStore;
+
+  PermImageRetriever(SentryStore sentryStore) {
+    this.sentryStore = sentryStore;
+  }
+
+  @Override
+  public PermissionsUpdate retrieveFullImage(long seqNum) throws Exception {
+    try(Timer.Context timerContext =
+        SentryHdfsMetricsUtil.getRetrievePermFullImageTimer.time()) {
+
+      // Read the most up-to-date snapshot of Sentry perm information,
+      // with a corresponding delta change sequence number.
+      PermissionsImage permImage = sentryStore.retrieveFullPermssionsImage();
+      long curSeqNum = permImage.getCurSeqNum();
+      Map<String, Map<String, String>> privilegeImage =
+          permImage.getPrivilegeImage();
+      Map<String, List<String>> roleImage =
+          permImage.getRoleImage();
+
+      // Translates the complete Sentry permission snapshot into a PermissionsUpdate.
+      // Adds the <hiveObj, <role, privileges>> mapping and the <role, groups> mapping
+      // to be included in the permission update.
+      // And label it with the latest delta change sequence number for consumer
+      // to be aware of the next delta change it should continue with.
+      TPermissionsUpdate tPermUpdate = new TPermissionsUpdate(true, curSeqNum,
+          new HashMap<String, TPrivilegeChanges>(),
+          new HashMap<String, TRoleChanges>());
+
+      for (Map.Entry<String, Map<String, String>> privEnt : privilegeImage.entrySet()) {
+        String authzObj = privEnt.getKey();
+        Map<String,String> privs = privEnt.getValue();
+        tPermUpdate.putToPrivilegeChanges(authzObj, new TPrivilegeChanges(
+        authzObj, privs, new HashMap<String, String>()));
+      }
+
+      for (Map.Entry<String, List<String>> privEnt : roleImage.entrySet()) {
+        String role = privEnt.getKey();
+        List<String> groups = privEnt.getValue();
+        tPermUpdate.putToRoleChanges(role, new TRoleChanges(role, groups,
+            new LinkedList<String>()));
+      }
+
+      PermissionsUpdate permissionsUpdate = new PermissionsUpdate(tPermUpdate);
+      // TODO: use curSeqNum from DB instead of seqNum when doing SENTRY-1567
+      permissionsUpdate.setSeqNum(seqNum);
+      SentryHdfsMetricsUtil.getPrivilegeChangesHistogram.update(
+          tPermUpdate.getPrivilegeChangesSize());
+      SentryHdfsMetricsUtil.getRoleChangesHistogram.update(
+          tPermUpdate.getRoleChangesSize());
+      return permissionsUpdate;
+    }
+  }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/sentry/blob/8f5c17db/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHdfsMetricsUtil.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHdfsMetricsUtil.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHdfsMetricsUtil.java
index e68c708..be14569 100644
--- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHdfsMetricsUtil.java
+++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHdfsMetricsUtil.java
@@ -60,19 +60,30 @@ public class SentryHdfsMetricsUtil {
       MetricRegistry.name(SentryHDFSServiceProcessor.class, "handle-hms-notification",
           "path-changes-size"));
 
-  // Metrics for retrieveFullImage in SentryPlugin.PermImageRetriever
-  // The time used for each retrieveFullImage
-  public static final Timer getRetrieveFullImageTimer = sentryMetrics.getTimer(
-      MetricRegistry.name(SentryPlugin.PermImageRetriever.class, "retrieve-full-image"));
-  // The size of privilege changes for each retrieveFullImage
+  // Metrics for retrievePermFullImage in PermImageRetriever
+  // The time used for each retrievePermFullImage
+  public static final Timer getRetrievePermFullImageTimer = sentryMetrics.getTimer(
+      MetricRegistry.name(PermImageRetriever.class, "retrieve-perm-full-image"));
+  // The size of privilege changes for each retrievePermFullImage
   public static final Histogram getPrivilegeChangesHistogram = sentryMetrics.getHistogram(
-      MetricRegistry.name(SentryPlugin.PermImageRetriever.class, "retrieve-full-image",
+      MetricRegistry.name(PermImageRetriever.class, "retrieve-perm-full-image",
           "privilege-changes-size"));
-  // The size of role changes for each retrieveFullImage call
+  // The size of role changes for each retrievePermFullImage call
   public static final Histogram getRoleChangesHistogram = sentryMetrics.getHistogram(
-      MetricRegistry.name(SentryPlugin.PermImageRetriever.class, "retrieve-full-image",
+      MetricRegistry.name(PermImageRetriever.class, "retrieve-perm-full-image",
           "role-changes-size"));
 
+  // Metrics for retrievePathFullImage in PathImageRetriever
+  // The time used for each retrievePathFullImage
+  public static final Timer getRetrievePathFullImageTimer = sentryMetrics.getTimer(
+      MetricRegistry.name(PathImageRetriever.class, "retrieve-path-full-image"));
+
+  // The size of path changes for each retrievePathFullImage
+  public static final Histogram getPathChangesHistogram = sentryMetrics.getHistogram(
+      MetricRegistry.name(PathImageRetriever.class, "retrieve-path-full-image",
+          "path-changes-size"));
+
+
   // Metrics for notifySentry HMS update in MetaStorePlugin
   // The timer used for each notifySentry
   public static final Timer getNotifyHMSUpdateTimer = sentryMetrics.getTimer(

http://git-wip-us.apache.org/repos/asf/sentry/blob/8f5c17db/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java
index f68f690..029f9d5 100644
--- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java
+++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java
@@ -18,23 +18,17 @@
 
 package org.apache.sentry.hdfs;
 
-import java.util.HashMap;
-import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
 
-import com.codahale.metrics.Timer;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.sentry.core.common.utils.SigUtils;
 import org.apache.sentry.hdfs.ServiceConstants.ServerConfig;
-import org.apache.sentry.hdfs.UpdateForwarder.ExternalImageRetriever;
-import org.apache.sentry.hdfs.service.thrift.TPermissionsUpdate;
 import org.apache.sentry.hdfs.service.thrift.TPrivilegeChanges;
 import org.apache.sentry.hdfs.service.thrift.TRoleChanges;
 import org.apache.sentry.provider.db.SentryPolicyStorePlugin;
-import org.apache.sentry.provider.db.service.persistent.DeltaTransactionBlock;
 import org.apache.sentry.provider.db.service.persistent.SentryStore;
 import org.apache.sentry.provider.db.service.thrift.TAlterSentryRoleAddGroupsRequest;
 import org.apache.sentry.provider.db.service.thrift.TAlterSentryRoleDeleteGroupsRequest;
@@ -43,7 +37,6 @@ import org.apache.sentry.provider.db.service.thrift.TAlterSentryRoleRevokePrivil
 import org.apache.sentry.provider.db.service.thrift.TDropPrivilegesRequest;
 import org.apache.sentry.provider.db.service.thrift.TDropSentryRoleRequest;
 import org.apache.sentry.provider.db.service.thrift.TRenamePrivilegesRequest;
-import org.apache.sentry.provider.db.service.thrift.TSentryAuthorizable;
 import org.apache.sentry.provider.db.service.thrift.TSentryGroup;
 import org.apache.sentry.provider.db.service.thrift.TSentryPrivilege;
 import org.apache.sentry.service.thrift.HMSFollower;
@@ -122,48 +115,6 @@ public class SentryPlugin implements SentryPolicyStorePlugin, SigUtils.SigListen
 
   public static volatile SentryPlugin instance;
 
-  static class PermImageRetriever implements ExternalImageRetriever<PermissionsUpdate> {
-
-    private final SentryStore sentryStore;
-
-    public PermImageRetriever(SentryStore sentryStore) {
-      this.sentryStore = sentryStore;
-    }
-
-    @Override
-    public PermissionsUpdate retrieveFullImage(long currSeqNum) throws Exception {
-      try(Timer.Context timerContext =
-                  SentryHdfsMetricsUtil.getRetrieveFullImageTimer.time()) {
-
-        SentryHdfsMetricsUtil.getRetrieveFullImageTimer.time();
-        Map<String, HashMap<String, String>> privilegeImage = sentryStore.retrieveFullPrivilegeImage();
-        Map<String, LinkedList<String>> roleImage = sentryStore.retrieveFullRoleImage();
-
-        TPermissionsUpdate tPermUpdate = new TPermissionsUpdate(true, currSeqNum,
-                new HashMap<String, TPrivilegeChanges>(),
-                new HashMap<String, TRoleChanges>());
-        for (Map.Entry<String, HashMap<String, String>> privEnt : privilegeImage.entrySet()) {
-          String authzObj = privEnt.getKey();
-          HashMap<String,String> privs = privEnt.getValue();
-          tPermUpdate.putToPrivilegeChanges(authzObj, new TPrivilegeChanges(
-                  authzObj, privs, new HashMap<String, String>()));
-        }
-        for (Map.Entry<String, LinkedList<String>> privEnt : roleImage.entrySet()) {
-          String role = privEnt.getKey();
-          LinkedList<String> groups = privEnt.getValue();
-          tPermUpdate.putToRoleChanges(role, new TRoleChanges(role, groups, new LinkedList<String>()));
-        }
-        PermissionsUpdate permissionsUpdate = new PermissionsUpdate(tPermUpdate);
-        permissionsUpdate.setSeqNum(currSeqNum);
-        SentryHdfsMetricsUtil.getPrivilegeChangesHistogram.update(
-                tPermUpdate.getPrivilegeChangesSize());
-        SentryHdfsMetricsUtil.getRoleChangesHistogram.update(
-                tPermUpdate.getRoleChangesSize());
-        return permissionsUpdate;
-      }
-    }
-  }
-
   private UpdateForwarder<PathsUpdate> pathsUpdater;
   private UpdateForwarder<PermissionsUpdate> permsUpdater;
   // TODO: Each perm change sequence number should be generated during persistence at sentry store.
@@ -202,10 +153,10 @@ public class SentryPlugin implements SentryPolicyStorePlugin, SigUtils.SigListen
     permImageRetriever = new PermImageRetriever(sentryStore);
 
     pathsUpdater = UpdateForwarder.create(conf, new UpdateableAuthzPaths(
-        pathPrefixes), new PathsUpdate(0, false), null, 100, initUpdateRetryDelayMs);
+        pathPrefixes), new PathsUpdate(0, false), null, 100, initUpdateRetryDelayMs, false);
     permsUpdater = UpdateForwarder.create(conf,
         new UpdateablePermissions(permImageRetriever), new PermissionsUpdate(0, false),
-        permImageRetriever, 100, initUpdateRetryDelayMs);
+        permImageRetriever, 100, initUpdateRetryDelayMs, true);
     LOGGER.info("Sentry HDFS plugin initialized !!");
     instance = this;
 

http://git-wip-us.apache.org/repos/asf/sentry/blob/8f5c17db/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateForwarder.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateForwarder.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateForwarder.java
index 6d5c607..22c5769 100644
--- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateForwarder.java
+++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateForwarder.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -37,12 +37,6 @@ import org.slf4j.LoggerFactory;
 public class UpdateForwarder<K extends Updateable.Update> implements
     Updateable<K>, Closeable {
 
-  interface ExternalImageRetriever<K> {
-
-    K retrieveFullImage(long currSeqNum) throws Exception;
-
-  }
-
   private final AtomicLong lastSeenSeqNum = new AtomicLong(0);
   protected final AtomicLong lastCommittedSeqNum = new AtomicLong(0);
   // Updates should be handled in order
@@ -60,7 +54,7 @@ public class UpdateForwarder<K extends Updateable.Update> implements
   // UpdateLog is disabled when getMaxUpdateLogSize() = 0;
   private final int maxUpdateLogSize;
 
-  private final ExternalImageRetriever<K> imageRetreiver;
+  private final ImageRetriever<K> imageRetreiver;
 
   private volatile Updateable<K> updateable;
 
@@ -72,16 +66,16 @@ public class UpdateForwarder<K extends Updateable.Update> implements
   private static final String UPDATABLE_TYPE_NAME = "update_forwarder";
 
   public UpdateForwarder(Configuration conf, Updateable<K> updateable,
-      ExternalImageRetriever<K> imageRetreiver, int maxUpdateLogSize) {
-    this(conf, updateable, imageRetreiver, maxUpdateLogSize, INIT_UPDATE_RETRY_DELAY);
+      ImageRetriever<K> imageRetreiver, int maxUpdateLogSize, boolean shouldInit) {
+    this(conf, updateable, imageRetreiver, maxUpdateLogSize, INIT_UPDATE_RETRY_DELAY, shouldInit);
   }
 
   protected UpdateForwarder(Configuration conf, Updateable<K> updateable, //NOPMD
-      ExternalImageRetriever<K> imageRetreiver, int maxUpdateLogSize,
-      int initUpdateRetryDelay) {
+      ImageRetriever<K> imageRetreiver, int maxUpdateLogSize,
+      int initUpdateRetryDelay, boolean shouldInit) {
     this.maxUpdateLogSize = maxUpdateLogSize;
     this.imageRetreiver = imageRetreiver;
-    if (imageRetreiver != null) {
+    if (shouldInit) {
       spawnInitialUpdater(updateable, initUpdateRetryDelay);
     } else {
       this.updateable = updateable;
@@ -89,17 +83,17 @@ public class UpdateForwarder<K extends Updateable.Update> implements
   }
 
   public static <K extends Updateable.Update> UpdateForwarder<K> create(Configuration conf,
-      Updateable<K> updateable, K update, ExternalImageRetriever<K> imageRetreiver,
-      int maxUpdateLogSize) throws SentryPluginException {
+      Updateable<K> updateable, K update, ImageRetriever<K> imageRetreiver,
+      int maxUpdateLogSize, boolean shouldInit) throws SentryPluginException {
     return create(conf, updateable, update, imageRetreiver, maxUpdateLogSize,
-        INIT_UPDATE_RETRY_DELAY);
+        INIT_UPDATE_RETRY_DELAY, shouldInit);
   }
 
   public static <K extends Updateable.Update> UpdateForwarder<K> create(Configuration conf,
-      Updateable<K> updateable, K update, ExternalImageRetriever<K> imageRetreiver,
-      int maxUpdateLogSize, int initUpdateRetryDelay) throws SentryPluginException {
+      Updateable<K> updateable, K update, ImageRetriever<K> imageRetreiver,
+      int maxUpdateLogSize, int initUpdateRetryDelay, boolean shouldInit) throws SentryPluginException {
     return new UpdateForwarder<K>(conf, updateable, imageRetreiver,
-        maxUpdateLogSize, initUpdateRetryDelay);
+        maxUpdateLogSize, initUpdateRetryDelay, shouldInit);
   }
 
   private void spawnInitialUpdater(final Updateable<K> updateable,

http://git-wip-us.apache.org/repos/asf/sentry/blob/8f5c17db/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateablePermissions.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateablePermissions.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateablePermissions.java
index fe2baa6..03c67d6 100644
--- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateablePermissions.java
+++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateablePermissions.java
@@ -20,16 +20,14 @@ package org.apache.sentry.hdfs;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.locks.ReadWriteLock;
 
-import org.apache.sentry.hdfs.UpdateForwarder.ExternalImageRetriever;
-
 public class UpdateablePermissions implements Updateable<PermissionsUpdate>{
   private static final String UPDATABLE_TYPE_NAME = "perm_update";
 
   private AtomicLong seqNum = new AtomicLong();
-  private final ExternalImageRetriever<PermissionsUpdate> imageRetreiver;
+  private final ImageRetriever<PermissionsUpdate> imageRetreiver;
 
   public UpdateablePermissions(
-      ExternalImageRetriever<PermissionsUpdate> imageRetreiver) {
+      ImageRetriever<PermissionsUpdate> imageRetreiver) {
     this.imageRetreiver = imageRetreiver;
   }
 
@@ -62,5 +60,4 @@ public class UpdateablePermissions implements Updateable<PermissionsUpdate>{
   public String getUpdateableTypeName() {
     return UPDATABLE_TYPE_NAME;
   }
-
 }

http://git-wip-us.apache.org/repos/asf/sentry/blob/8f5c17db/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestUpdateForwarder.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestUpdateForwarder.java b/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestUpdateForwarder.java
index 0f0d0a7..d12b134 100644
--- a/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestUpdateForwarder.java
+++ b/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestUpdateForwarder.java
@@ -27,7 +27,6 @@ import org.apache.thrift.TException;
 import org.junit.Assert;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.sentry.hdfs.UpdateForwarder.ExternalImageRetriever;
 import org.apache.sentry.hdfs.Updateable.Update;
 import org.apache.sentry.service.thrift.ServiceConstants.ServerConfig;
 import org.junit.After;
@@ -134,7 +133,7 @@ public class TestUpdateForwarder {
     }
   }
 
-  static class DummyImageRetreiver implements ExternalImageRetriever<DummyUpdate> {
+  static class DummyImageRetreiver implements ImageRetriever<DummyUpdate> {
 
     private String state;
     public void setState(String state) {
@@ -164,7 +163,7 @@ public class TestUpdateForwarder {
     DummyImageRetreiver imageRetreiver = new DummyImageRetreiver();
     imageRetreiver.setState("a,b,c");
     updateForwarder = UpdateForwarder.create(
-        testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 10);
+        testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 10, true);
     Assert.assertEquals(-2, updateForwarder.getLastUpdatedSeqNum());
     List<DummyUpdate> allUpdates = updateForwarder.getAllUpdatesFrom(0);
     Assert.assertTrue(allUpdates.size() == 1);
@@ -184,7 +183,7 @@ public class TestUpdateForwarder {
     DummyImageRetreiver imageRetreiver = new DummyImageRetreiver();
     imageRetreiver.setState("a,b,c");
     updateForwarder = UpdateForwarder.create(
-        testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 5);
+        testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 5, true);
     updateForwarder.handleUpdateNotification(new DummyUpdate(5, false).setState("d"));
     while(!updateForwarder.areAllUpdatesCommited()) {
       Thread.sleep(100);
@@ -205,7 +204,7 @@ public class TestUpdateForwarder {
     Assume.assumeTrue(!testConf.getBoolean(ServerConfig.SENTRY_HA_ENABLED,
         false));
     updateForwarder = UpdateForwarder.create(
-        testConf, new DummyUpdatable(), new DummyUpdate(), null, 5);
+        testConf, new DummyUpdatable(), new DummyUpdate(), null, 5, false);
     updateForwarder.handleUpdateNotification(new DummyUpdate(-1, true).setState("a"));
     while(!updateForwarder.areAllUpdatesCommited()) {
       Thread.sleep(100);
@@ -232,7 +231,7 @@ public class TestUpdateForwarder {
     DummyImageRetreiver imageRetreiver = new DummyImageRetreiver();
     imageRetreiver.setState("a,b,c");
     updateForwarder = UpdateForwarder.create(
-        testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 5);
+        testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 5, true);
     updateForwarder.handleUpdateNotification(new DummyUpdate(5, false).setState("d"));
     while(!updateForwarder.areAllUpdatesCommited()) {
       Thread.sleep(100);
@@ -282,7 +281,7 @@ public class TestUpdateForwarder {
     DummyImageRetreiver imageRetreiver = new DummyImageRetreiver();
     imageRetreiver.setState("a,b,c");
     updateForwarder = UpdateForwarder.create(
-        testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 5);
+        testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 5, true);
     updateForwarder.handleUpdateNotification(new DummyUpdate(5, false).setState("d"));
     while(!updateForwarder.areAllUpdatesCommited()) {
       Thread.sleep(100);
@@ -328,7 +327,7 @@ public class TestUpdateForwarder {
     DummyImageRetreiver imageRetreiver = new DummyImageRetreiver();
     imageRetreiver.setState("a,b,c");
     updateForwarder = UpdateForwarder.create(
-        testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 5);
+        testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 5, true);
     updateForwarder.handleUpdateNotification(new DummyUpdate(5, false).setState("d"));
     while(!updateForwarder.areAllUpdatesCommited()) {
       Thread.sleep(100);

http://git-wip-us.apache.org/repos/asf/sentry/blob/8f5c17db/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/PathsImage.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/PathsImage.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/PathsImage.java
new file mode 100644
index 0000000..fd56ce2
--- /dev/null
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/PathsImage.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.provider.db.service.persistent;
+
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A container for complete hive paths snapshot.
+ * <p>
+ * It is composed by a hiveObj to Paths mapping and the sequence number/change ID
+ * of latest delta change that the snapshot maps to.
+ */
+public class PathsImage {
+
+  // A full snapshot of hiveObj to Paths mapping.
+  private final Map<String, Set<String>> pathImage;
+  private final long curSeqNum;
+
+  PathsImage(Map<String, Set<String>> pathImage, long curSeqNum) {
+    this.pathImage = pathImage;
+    this.curSeqNum = curSeqNum;
+  }
+
+  public long getCurSeqNum() {
+    return curSeqNum;
+  }
+
+  public Map<String, Set<String>> getPathImage() {
+    return pathImage;
+  }
+}

http://git-wip-us.apache.org/repos/asf/sentry/blob/8f5c17db/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/PermissionsImage.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/PermissionsImage.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/PermissionsImage.java
new file mode 100644
index 0000000..f03e93f
--- /dev/null
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/PermissionsImage.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.provider.db.service.persistent;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A container for complete sentry permission snapshot.
+ * <p>
+ * It is composed by a role to groups mapping, and hiveObj to &lt role, privileges &gt mapping.
+ * It also has the sequence number/change ID of latest delta change that the snapshot maps to.
+ */
+public class PermissionsImage {
+
+  // A full snapshot of sentry role to groups mapping.
+  private final Map<String, List<String>> roleImage;
+
+  // A full snapshot of hiveObj to <role, privileges> mapping.
+  private final Map<String, Map<String, String>> privilegeImage;
+  private final long curSeqNum;
+
+  PermissionsImage(Map<String, List<String>> roleImage,
+          Map<String, Map<String, String>> privilegeImage, long curSeqNum) {
+    this.roleImage = roleImage;
+    this.privilegeImage = privilegeImage;
+    this.curSeqNum = curSeqNum;
+  }
+
+  public long getCurSeqNum() {
+    return curSeqNum;
+  }
+
+  public Map<String, Map<String, String>> getPrivilegeImage() {
+    return privilegeImage;
+  }
+
+  public Map<String, List<String>> getRoleImage() {
+    return roleImage;
+  }
+}

http://git-wip-us.apache.org/repos/asf/sentry/blob/8f5c17db/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java
index a5a835e..6ea6d3f 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java
@@ -2260,107 +2260,158 @@ public class SentryStore {
         ServerConfig.ADMIN_GROUPS, new String[]{}));
   }
 
-  public Map<String, HashMap<String, String>> retrieveFullPrivilegeImage() throws Exception {
+  /**
+   * Retrieves an up-to-date sentry permission snapshot.
+   * <p>
+   * It reads hiveObj to &lt role, privileges &gt mapping from {@link MSentryPrivilege}
+   * table and role to groups mapping from {@link MSentryGroup}.
+   * It also gets the changeID of latest delta update, from {@link MSentryPathChange}, that
+   * the snapshot corresponds to.
+   *
+   * @return a {@link PathsImage} contains the mapping of hiveObj to
+   *         &lt role, privileges &gt and the mapping of role to &lt Groups &gt.
+   *         For empty image returns {@link #EMPTY_CHANGE_ID} and empty maps.
+   * @throws Exception
+   */
+  public PermissionsImage retrieveFullPermssionsImage() throws Exception {
     return tm.executeTransaction(
-      new TransactionBlock<Map<String, HashMap<String, String>>>() {
-        public Map<String, HashMap<String, String>> execute(PersistenceManager pm)
-                throws Exception {
-          Map<String, HashMap<String, String>> retVal = new HashMap<>();
-          Query query = pm.newQuery(MSentryPrivilege.class);
-          QueryParamBuilder paramBuilder = newQueryParamBuilder();
-          paramBuilder
-                  .addNotNull(SERVER_NAME)
-                  .addNotNull(DB_NAME)
-                  .addNull(URI);
+    new TransactionBlock<PermissionsImage>() {
+      public PermissionsImage execute(PersistenceManager pm)
+      throws Exception {
+        // curChangeID could be 0, if Sentry server has been running before
+        // enable SentryPlugin(HDFS Sync feature).
+        long curChangeID = getLastProcessedChangeIDCore(pm, MSentryPermChange.class);
+        Map<String, List<String>> roleImage = retrieveFullRoleImageCore(pm);
+        Map<String, Map<String, String>> privilegeMap = retrieveFullPrivilegeImageCore(pm);
 
-          query.setFilter(paramBuilder.toString());
-          query.setOrdering("serverName ascending, dbName ascending, tableName ascending");
-          @SuppressWarnings("unchecked")
-          List<MSentryPrivilege> privileges =
-                  (List<MSentryPrivilege>) query.executeWithMap(paramBuilder.getArguments());
-          for (MSentryPrivilege mPriv : privileges) {
-            String authzObj = mPriv.getDbName();
-            if (!isNULL(mPriv.getTableName())) {
-              authzObj = authzObj + "." + mPriv.getTableName();
-            }
-            HashMap<String, String> pUpdate = retVal.get(authzObj);
-            if (pUpdate == null) {
-              pUpdate = new HashMap<>();
-              retVal.put(authzObj, pUpdate);
-            }
-            for (MSentryRole mRole : mPriv.getRoles()) {
-              String existingPriv = pUpdate.get(mRole.getRoleName());
-              if (existingPriv == null) {
-                pUpdate.put(mRole.getRoleName(), mPriv.getAction().toUpperCase());
-              } else {
-                pUpdate.put(mRole.getRoleName(), existingPriv + ","
-                        + mPriv.getAction().toUpperCase());
-              }
-            }
-          }
-          return retVal;
+        return new PermissionsImage(roleImage, privilegeMap, curChangeID);
+      }
+    });
+  }
+
+  /**
+   * Retrieves an up-to-date sentry privileges snapshot from {@code MSentryPrivilege} table.
+   * The snapshot is represented by mapping of hiveObj to role privileges.
+   *
+   * @param pm PersistenceManager
+   * @return a mapping of hiveObj to &lt role, privileges &gt
+   * @throws Exception
+   */
+   private Map<String, Map<String, String>> retrieveFullPrivilegeImageCore(PersistenceManager pm)
+        throws Exception {
+
+    Map<String, Map<String, String>> retVal = new HashMap<>();
+    Query query = pm.newQuery(MSentryPrivilege.class);
+    QueryParamBuilder paramBuilder = newQueryParamBuilder();
+    paramBuilder.addNotNull(SERVER_NAME)
+                .addNotNull(DB_NAME)
+                .addNull(URI);
+
+    query.setFilter(paramBuilder.toString());
+    query.setOrdering("serverName ascending, dbName ascending, tableName ascending");
+    @SuppressWarnings("unchecked")
+    List<MSentryPrivilege> privileges =
+            (List<MSentryPrivilege>) query.executeWithMap(paramBuilder.getArguments());
+    for (MSentryPrivilege mPriv : privileges) {
+      String authzObj = mPriv.getDbName();
+      if (!isNULL(mPriv.getTableName())) {
+        authzObj = authzObj + "." + mPriv.getTableName();
+      }
+      Map<String, String> pUpdate = retVal.get(authzObj);
+      if (pUpdate == null) {
+        pUpdate = new HashMap<>();
+        retVal.put(authzObj, pUpdate);
+      }
+      for (MSentryRole mRole : mPriv.getRoles()) {
+        String existingPriv = pUpdate.get(mRole.getRoleName());
+        if (existingPriv == null) {
+          pUpdate.put(mRole.getRoleName(), mPriv.getAction().toUpperCase());
+        } else {
+          pUpdate.put(mRole.getRoleName(), existingPriv + "," + mPriv.getAction().toUpperCase());
         }
-      });
+      }
+    }
+    return retVal;
   }
 
   /**
-   * @return Mapping of Role -> [Groups]
+   * Retrieves an up-to-date sentry role snapshot from {@code MSentryGroup} table.
+   * The snapshot is represented by a role to groups map.
+   *
+   * @param pm PersistenceManager
+   * @return a mapping of Role to &lt Groups &gt
+   * @throws Exception
    */
-  public Map<String, LinkedList<String>> retrieveFullRoleImage() throws Exception {
-    return tm.executeTransaction(
-        new TransactionBlock<Map<String, LinkedList<String>>>() {
-          public Map<String, LinkedList<String>> execute(PersistenceManager pm) throws Exception {
-            Query query = pm.newQuery(MSentryGroup.class);
-            @SuppressWarnings("unchecked")
-            List<MSentryGroup> groups = (List<MSentryGroup>) query.execute();
-            if (groups.isEmpty()) {
-              return Collections.emptyMap();
-            }
+  private Map<String, List<String>> retrieveFullRoleImageCore(PersistenceManager pm)
+          throws Exception {
+    Query query = pm.newQuery(MSentryGroup.class);
+    @SuppressWarnings("unchecked")
+    List<MSentryGroup> groups = (List<MSentryGroup>) query.execute();
+    if (groups.isEmpty()) {
+      return Collections.emptyMap();
+    }
 
-            Map<String, LinkedList<String>> retVal = new HashMap<>();
-            for (MSentryGroup mGroup : groups) {
-              for (MSentryRole role : mGroup.getRoles()) {
-                LinkedList<String> rUpdate = retVal.get(role.getRoleName());
-                if (rUpdate == null) {
-                  rUpdate = new LinkedList<String>();
-                  retVal.put(role.getRoleName(), rUpdate);
-                }
-                rUpdate.add(mGroup.getGroupName());
-              }
-            }
-            return retVal;
-          }
-        });
+    Map<String, List<String>> retVal = new HashMap<>();
+    for (MSentryGroup mGroup : groups) {
+      for (MSentryRole role : mGroup.getRoles()) {
+        List<String> rUpdate = retVal.get(role.getRoleName());
+        if (rUpdate == null) {
+          rUpdate = new LinkedList<>();
+          retVal.put(role.getRoleName(), rUpdate);
+        }
+        rUpdate.add(mGroup.getGroupName());
+      }
+    }
+    return retVal;
+  }
+
+  /**
+   * Retrieves an up-to-date hive paths snapshot.
+   * <p>
+   * It reads hiveObj to paths mapping from {@link MAuthzPathsMapping} table and
+   * gets the changeID of latest delta update, from {@link MSentryPathChange}, that
+   * the snapshot corresponds to.
+   *
+   * @return an up-to-date hive paths snapshot contains mapping of hiveObj to &lt Paths &gt.
+   *         For empty image return {@link #EMPTY_CHANGE_ID} and a empty map.
+   * @throws Exception
+   */
+  public PathsImage retrieveFullPathsImage() throws Exception {
+    return (PathsImage) tm.executeTransaction(
+    new TransactionBlock() {
+      public Object execute(PersistenceManager pm) throws Exception {
+        // curChangeID could be 0 for the first full snapshot fetching
+        // from HMS. It does not have corresponding delta update.
+        long curChangeID = getLastProcessedChangeIDCore(pm, MSentryPathChange.class);
+        Map<String, Set<String>> pathImage = retrieveFullPathsImageCore(pm);
+
+        return new PathsImage(pathImage, curChangeID);
+      }
+    });
   }
 
   /**
-   * This returns a Mapping of Authz -> [Paths]
+   * Retrieves an up-to-date hive paths snapshot from {@code MAuthzPathsMapping} table.
+   * The snapshot is represented by a hiveObj to paths map.
+   *
+   * @return a mapping of hiveObj to &lt Paths &gt.
    */
-  public Map<String, Set<String>> retrieveFullPathsImage() {
-    Map<String, Set<String>> result = new HashMap<>();
-    try {
-      result = (Map<String, Set<String>>) tm.executeTransaction(
-          new TransactionBlock<Object>() {
-            public Object execute(PersistenceManager pm) throws Exception {
-              Map<String, Set<String>> retVal = new HashMap<>();
-              Query query = pm.newQuery(MAuthzPathsMapping.class);
-              List<MAuthzPathsMapping> authzToPathsMappings = (List<MAuthzPathsMapping>) query.execute();
-              for (MAuthzPathsMapping authzToPaths : authzToPathsMappings) {
-                retVal.put(authzToPaths.getAuthzObjName(), authzToPaths.getPaths());
-              }
-              return retVal;
-            }
-          });
-    } catch (Exception e) {
-      LOGGER.error(e.getMessage(), e);
+  private Map<String, Set<String>> retrieveFullPathsImageCore(PersistenceManager pm) {
+    Map<String, Set<String>> retVal = new HashMap<>();
+    Query query = pm.newQuery(MAuthzPathsMapping.class);
+    Iterable<MAuthzPathsMapping> authzToPathsMappings =
+        (Iterable<MAuthzPathsMapping>) query.execute();
+
+    for (MAuthzPathsMapping authzToPaths : authzToPathsMappings) {
+      retVal.put(authzToPaths.getAuthzObjName(), authzToPaths.getPaths());
     }
-    return result;
+    return retVal;
   }
 
   /**
-   * Persist a full hive snapshot into Sentry DB in a single transaction.
+   * Persist an up-to-date hive snapshot into Sentry DB in a single transaction.
    *
-   * @param authzPaths Mapping of hiveObj -> [Paths]
+   * @param authzPaths Mapping of hiveObj to &lt Paths &lt
    * @throws Exception
    */
   public void persistFullPathsImage(final Map<String, Set<String>> authzPaths) throws Exception {

http://git-wip-us.apache.org/repos/asf/sentry/blob/8f5c17db/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
index 91f15c0..75f855c 100644
--- a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
+++ b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
@@ -2188,21 +2188,67 @@ public class TestSentryStore extends org.junit.Assert {
   }
 
   @Test
+  public void testRetrieveFullPermssionsImage() throws Exception {
+
+    // Create roles
+    String roleName1 = "privs-r1", roleName2 = "privs-r2";
+    String groupName1 = "privs-g1";
+    String grantor = "g1";
+    sentryStore.createSentryRole(roleName1);
+    sentryStore.createSentryRole(roleName2);
+
+    // Grant Privileges to the roles
+    TSentryPrivilege privilege1 = new TSentryPrivilege();
+    privilege1.setPrivilegeScope("TABLE");
+    privilege1.setServerName("server1");
+    privilege1.setDbName("db1");
+    privilege1.setTableName("tbl1");
+    privilege1.setAction("SELECT");
+    privilege1.setCreateTime(System.currentTimeMillis());
+    sentryStore.alterSentryRoleGrantPrivilege(grantor, roleName1, privilege1);
+    sentryStore.alterSentryRoleGrantPrivilege(grantor, roleName2, privilege1);
+
+    TSentryPrivilege privilege2 = new TSentryPrivilege();
+    privilege2.setPrivilegeScope("SERVER");
+    privilege2.setServerName("server1");
+    privilege1.setDbName("db2");
+    privilege1.setAction("ALL");
+    privilege2.setCreateTime(System.currentTimeMillis());
+    sentryStore.alterSentryRoleGrantPrivilege(grantor, roleName2, privilege2);
+
+    // Grant roles to the groups
+    Set<TSentryGroup> groups = Sets.newHashSet();
+    TSentryGroup group = new TSentryGroup();
+    group.setGroupName(groupName1);
+    groups.add(group);
+    sentryStore.alterSentryRoleAddGroups(grantor, roleName1, groups);
+    sentryStore.alterSentryRoleAddGroups(grantor, roleName2, groups);
+
+    PermissionsImage permImage = sentryStore.retrieveFullPermssionsImage();
+    Map<String, Map<String, String>> privs = permImage.getPrivilegeImage();
+    Map<String, List<String>> roles = permImage.getRoleImage();
+    assertEquals(2, privs.get("db1.tbl1").size());
+    assertEquals(2, roles.size());
+  }
+
+  @Test
   public void testAuthzPathsMapping() throws Exception {
     sentryStore.createAuthzPathsMapping("db1.table1", Sets.newHashSet("/user/hive/warehouse/db1.db/table1"));
     sentryStore.createAuthzPathsMapping("db1.table2", Sets.newHashSet("/user/hive/warehouse/db1.db/table2"));
 
-    Map<String, Set<String>> pathsImage = sentryStore.retrieveFullPathsImage();
-    assertEquals(2, pathsImage.size());
-    assertEquals(Sets.newHashSet("/user/hive/warehouse/db1.db/table1"), pathsImage.get("db1.table1"));
+    PathsImage pathsImage = sentryStore.retrieveFullPathsImage();
+    Map<String, Set<String>> pathImage = pathsImage.getPathImage();
+    assertEquals(2, pathImage.size());
+    assertEquals(Sets.newHashSet("/user/hive/warehouse/db1.db/table1"), pathImage.get("db1.table1"));
 
     Map<String, Set<String>> authzPaths = new HashMap<>();
     authzPaths.put("db2.table1", Sets.newHashSet("/user/hive/warehouse/db2.db/table1"));
     authzPaths.put("db2.table2", Sets.newHashSet("/user/hive/warehouse/db2.db/table2"));
     sentryStore.persistFullPathsImage(authzPaths);
     pathsImage = sentryStore.retrieveFullPathsImage();
-    assertEquals(4, pathsImage.size());
-    assertEquals(Sets.newHashSet("/user/hive/warehouse/db2.db/table1"), pathsImage.get("db2.table1"));
+    pathImage = pathsImage.getPathImage();
+    assertEquals(4, pathImage.size());
+    assertEquals(Sets.newHashSet("/user/hive/warehouse/db2.db/table1"), pathImage.get("db2.table1"));
   }
 
   public void testQueryParamBuilder() {


Mime
View raw message