sentry-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vam...@apache.org
Subject sentry git commit: SENTRY-1931: NameNode only gets full snapshot once (Sergio Pena, Reviewed by: Alexander Kolbasov, Vamsee Yarlagadda)
Date Fri, 08 Sep 2017 01:17:47 GMT
Repository: sentry
Updated Branches:
  refs/heads/master c302da781 -> 398a8a937


SENTRY-1931: NameNode only gets full snapshot once (Sergio Pena, Reviewed by: Alexander Kolbasov,
Vamsee Yarlagadda)


Project: http://git-wip-us.apache.org/repos/asf/sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/398a8a93
Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/398a8a93
Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/398a8a93

Branch: refs/heads/master
Commit: 398a8a9378cb43fea7f66153df917b9eb3abd40c
Parents: c302da7
Author: Vamsee Yarlagadda <vamsee@cloudera.com>
Authored: Thu Sep 7 18:15:57 2017 -0700
Committer: Vamsee Yarlagadda <vamsee@cloudera.com>
Committed: Thu Sep 7 18:15:57 2017 -0700

----------------------------------------------------------------------
 .../src/main/java/org/apache/sentry/hdfs/DeltaRetriever.java   | 6 ++++--
 .../src/main/java/org/apache/sentry/hdfs/PathsUpdate.java      | 5 +++++
 .../main/java/org/apache/sentry/hdfs/PermissionsUpdate.java    | 5 +++++
 .../src/main/java/org/apache/sentry/hdfs/Updateable.java       | 2 ++
 .../main/java/org/apache/sentry/hdfs/UpdateableAuthzPaths.java | 6 +++++-
 .../main/java/org/apache/sentry/hdfs/DBUpdateForwarder.java    | 2 +-
 .../main/java/org/apache/sentry/hdfs/PathDeltaRetriever.java   | 3 ++-
 .../main/java/org/apache/sentry/hdfs/PermDeltaRetriever.java   | 2 +-
 .../java/org/apache/sentry/hdfs/TestDBUpdateForwarder.java     | 2 +-
 .../test/java/org/apache/sentry/hdfs/TestDeltaRetriever.java   | 6 ++++--
 10 files changed, 30 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sentry/blob/398a8a93/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/DeltaRetriever.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/DeltaRetriever.java
b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/DeltaRetriever.java
index 4503950..3b7fcef 100644
--- a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/DeltaRetriever.java
+++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/DeltaRetriever.java
@@ -29,7 +29,7 @@ import static org.apache.sentry.hdfs.Updateable.Update;
  * Sentry permissions are represented as {@link PermissionsUpdate} and HMS Paths
  * are represented as {@link PathsUpdate}. The delta update contains change
  * from a state to another.
- * The {@link #retrieveDelta(long)} method obtains such delta update from a persistent storage.
+ * The {@link #retrieveDelta(long,long)} method obtains such delta update from a persistent
storage.
  * Delta update is propagated to a consumer of Sentry, such as HDFS NameNode whenever
  * the consumer needs to synchronize the update.
  */
@@ -38,13 +38,15 @@ public interface DeltaRetriever<K extends Update> {
   /**
    * Retrieves all delta updates of type {@link Update} newer than or equal to
    * the given sequence number/change ID (inclusive) from a persistent storage.
+   * An image number may be used by the implementation to request deltas for a specific image.
    * An empty collection can be returned.
    *
    * @param seqNum the given seq number
+   * @param imgNum the given img number
    * @return delta updates of type K
    * @throws Exception when there is an error in operation on persistent storage
    */
-  List<K> retrieveDelta(long seqNum) throws Exception;
+  List<K> retrieveDelta(long seqNum, long imgNum) throws Exception;
 
   /**
    * Checks if there the delta update is available, given the sequence number/change

http://git-wip-us.apache.org/repos/asf/sentry/blob/398a8a93/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java
b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java
index ccc10ef..3996747 100644
--- a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java
+++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java
@@ -100,6 +100,11 @@ public class PathsUpdate implements Updateable.Update {
     return tPathsUpdate.getImgNum();
   }
 
+  @Override
+  public void setImgNum(long imgNum) {
+    tPathsUpdate.setImgNum(imgNum);
+  }
+
   public TPathsUpdate toThrift() {
     return tPathsUpdate;
   }

http://git-wip-us.apache.org/repos/asf/sentry/blob/398a8a93/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PermissionsUpdate.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PermissionsUpdate.java
b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PermissionsUpdate.java
index 7aa60a3..392f9a6 100644
--- a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PermissionsUpdate.java
+++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PermissionsUpdate.java
@@ -68,6 +68,11 @@ public class PermissionsUpdate implements Updateable.Update {
   }
 
   @Override
+  public void setImgNum(long imgNum) {
+    throw new UnsupportedOperationException("setImgNum not used");
+  }
+
+  @Override
   public boolean hasFullImage() {
     return tPermUpdate.isHasfullImage();
   }

http://git-wip-us.apache.org/repos/asf/sentry/blob/398a8a93/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/Updateable.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/Updateable.java
b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/Updateable.java
index 795aac7..bbf3f13 100644
--- a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/Updateable.java
+++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/Updateable.java
@@ -40,6 +40,8 @@ public interface Updateable<K extends Updateable.Update> {
 
     long getImgNum();
 
+    void setImgNum(long imgNum);
+
     byte[] serialize() throws IOException;
 
     void deserialize(byte data[]) throws IOException;

http://git-wip-us.apache.org/repos/asf/sentry/blob/398a8a93/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPaths.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPaths.java
b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPaths.java
index 819b6b2..cd97939 100644
--- a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPaths.java
+++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPaths.java
@@ -84,7 +84,11 @@ public class UpdateableAuthzPaths implements AuthzPaths, Updateable<PathsUpdate>
           lock.writeLock().lock();
         }
         seqNum.set(update.getSeqNum());
-        imgNum.set(update.getImgNum());
+
+        // Update the image ID only if the update has a new one
+        if (imgNum.get() < update.getImgNum()) {
+          imgNum.set(update.getImgNum());
+        }
         LOG.debug("##### Updated paths seq Num [{}] img Num [{}]", seqNum.get(), imgNum.get());
       }
     } finally {

http://git-wip-us.apache.org/repos/asf/sentry/blob/398a8a93/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/DBUpdateForwarder.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/DBUpdateForwarder.java
b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/DBUpdateForwarder.java
index 5d74421..bb85c13 100644
--- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/DBUpdateForwarder.java
+++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/DBUpdateForwarder.java
@@ -103,7 +103,7 @@ class DBUpdateForwarder<K extends Updateable.Update> {
     // Checks if newer deltas exist in the persistent storage.
     // If there are, return the list of delta updates.
     if (seqNum > SEQUENCE_NUMBER_UPDATE_UNINITIALIZED && deltaRetriever.isDeltaAvailable(seqNum))
{
-      List<K> deltas = deltaRetriever.retrieveDelta(seqNum);
+      List<K> deltas = deltaRetriever.retrieveDelta(seqNum, imgNum);
       if (!deltas.isEmpty()) {
         LOGGER.info("Newer delta updates are found up to sequence number: {}", curSeqNum);
         return deltas;

http://git-wip-us.apache.org/repos/asf/sentry/blob/398a8a93/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathDeltaRetriever.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathDeltaRetriever.java
b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathDeltaRetriever.java
index fda7455..b2e45f9 100644
--- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathDeltaRetriever.java
+++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathDeltaRetriever.java
@@ -44,7 +44,7 @@ public class PathDeltaRetriever implements DeltaRetriever<PathsUpdate>
{
   }
 
   @Override
-  public List<PathsUpdate> retrieveDelta(long seqNum) throws Exception {
+  public List<PathsUpdate> retrieveDelta(long seqNum, long imgNum) throws Exception
{
     try (final Context timerContext =
                  SentryHdfsMetricsUtil.getDeltaPathChangesTimer.time()) {
       List<MSentryPathChange> mSentryPathChanges =
@@ -66,6 +66,7 @@ public class PathDeltaRetriever implements DeltaRetriever<PathsUpdate>
{
         PathsUpdate pathsUpdate = new PathsUpdate();
         pathsUpdate.JSONDeserialize(mSentryPathChange.getPathChange());
         pathsUpdate.setSeqNum(changeID);
+        pathsUpdate.setImgNum(imgNum);
         updates.add(pathsUpdate);
       }
       return updates;

http://git-wip-us.apache.org/repos/asf/sentry/blob/398a8a93/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermDeltaRetriever.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermDeltaRetriever.java
b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermDeltaRetriever.java
index df6a0b0..6974d37 100644
--- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermDeltaRetriever.java
+++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermDeltaRetriever.java
@@ -44,7 +44,7 @@ public class PermDeltaRetriever implements DeltaRetriever<PermissionsUpdate>
{
   }
 
   @Override
-  public List<PermissionsUpdate> retrieveDelta(long seqNum) throws Exception {
+  public List<PermissionsUpdate> retrieveDelta(long seqNum, long imgNum) throws Exception
{
     try (final Context timerContext =
                  SentryHdfsMetricsUtil.getDeltaPermChangesTimer.time()) {
       Collection<MSentryPermChange> mSentryPermChanges =

http://git-wip-us.apache.org/repos/asf/sentry/blob/398a8a93/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestDBUpdateForwarder.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestDBUpdateForwarder.java
b/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestDBUpdateForwarder.java
index 8fbc100..830d00e 100644
--- a/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestDBUpdateForwarder.java
+++ b/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestDBUpdateForwarder.java
@@ -117,7 +117,7 @@ public class TestDBUpdateForwarder {
     Mockito.when(imageRetriever.getLatestImageID()).thenReturn(1L);
     Mockito.when(deltaRetriever.getLatestDeltaID()).thenReturn(3L);
     Mockito.when(deltaRetriever.isDeltaAvailable(2L)).thenReturn(true);
-    Mockito.when(deltaRetriever.retrieveDelta(2L))
+    Mockito.when(deltaRetriever.retrieveDelta(2L, 1L))
         .thenReturn(Arrays.asList(new PathsUpdate(3, 1, false)));
 
     List<PathsUpdate> updates = updater.getAllUpdatesFrom(2, 1);

http://git-wip-us.apache.org/repos/asf/sentry/blob/398a8a93/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestDeltaRetriever.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestDeltaRetriever.java
b/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestDeltaRetriever.java
index 63b8caf..60696cc 100644
--- a/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestDeltaRetriever.java
+++ b/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestDeltaRetriever.java
@@ -44,7 +44,7 @@ public class TestDeltaRetriever {
         .thenReturn(Collections.<MSentryPathChange>emptyList());
 
     PathDeltaRetriever deltaRetriever = new PathDeltaRetriever(sentryStoreMock);
-    List<PathsUpdate> pathsUpdates = deltaRetriever.retrieveDelta(1);
+    List<PathsUpdate> pathsUpdates = deltaRetriever.retrieveDelta(1, 1);
 
     assertTrue(pathsUpdates.isEmpty());
   }
@@ -63,12 +63,14 @@ public class TestDeltaRetriever {
         .thenReturn(deltaPathChanges);
 
     deltaRetriever = new PathDeltaRetriever(sentryStoreMock);
-    pathsUpdates = deltaRetriever.retrieveDelta(1);
+    pathsUpdates = deltaRetriever.retrieveDelta(1, 3);
 
     assertEquals(2, pathsUpdates.size());
     assertEquals(1, pathsUpdates.get(0).getSeqNum());
     assertEquals(true, pathsUpdates.get(0).hasFullImage());
+    assertEquals(3, pathsUpdates.get(0).getImgNum());
     assertEquals(2, pathsUpdates.get(1).getSeqNum());
     assertEquals(false, pathsUpdates.get(1).hasFullImage());
+    assertEquals(3, pathsUpdates.get(1).getImgNum());
   }
 }


Mime
View raw message