sentry-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ha...@apache.org
Subject sentry git commit: SENTRY-1428: Only leader should follow HMS updates (Alexander Kolbasov, Reviewed by: Hao Hao)
Date Fri, 20 Jan 2017 19:33:18 GMT
Repository: sentry
Updated Branches:
  refs/heads/sentry-ha-redesign 79cbd178a -> c4d3b5eae


SENTRY-1428: Only leader should follow HMS updates (Alexander Kolbasov, Reviewed by: Hao Hao)

Change-Id: I158608d917241fe01ad7a12b2c4ad8e633b6a23d


Project: http://git-wip-us.apache.org/repos/asf/sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/c4d3b5ea
Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/c4d3b5ea
Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/c4d3b5ea

Branch: refs/heads/sentry-ha-redesign
Commit: c4d3b5eae26f615cee02f7e77c642ff6dd7f0706
Parents: 79cbd17
Author: hahao <hao.hao@cloudera.com>
Authored: Fri Jan 20 11:31:26 2017 -0800
Committer: hahao <hao.hao@cloudera.com>
Committed: Fri Jan 20 11:31:26 2017 -0800

----------------------------------------------------------------------
 .../sentry/service/thrift/HMSFollower.java      | 57 ++++++++++----------
 .../sentry/service/thrift/SentryService.java    |  2 +-
 2 files changed, 30 insertions(+), 29 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sentry/blob/c4d3b5ea/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
index f3cefd6..749c2ce 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
@@ -49,11 +49,11 @@ import java.util.List;
 import static org.apache.sentry.binding.hive.conf.HiveAuthzConf.AuthzConfVars.AUTHZ_SYNC_CREATE_WITH_POLICY_STORE;
 import static org.apache.sentry.binding.hive.conf.HiveAuthzConf.AuthzConfVars.AUTHZ_SYNC_DROP_WITH_POLICY_STORE;
 
-/*
-HMSFollower is the thread which follows the Hive MetaStore state changes from Sentry.
-It gets the full update and notification logs from HMS and applies it to
-update permissions stored in Sentry using SentryStore and also update the <obj,path>
state
-stored for HDFS- Sentry sync.
+/**
+ * HMSFollower is the thread which follows the Hive MetaStore state changes from Sentry.
+ * It gets the full update and notification logs from HMS and applies it to
+ * update permissions stored in Sentry using SentryStore and also update the &lt obj,path
&gt state
+ * stored for HDFS-Sentry sync.
  */
 @SuppressWarnings("PMD")
 public class HMSFollower implements Runnable {
@@ -63,19 +63,22 @@ public class HMSFollower implements Runnable {
   private static boolean connectedToHMS = false;
   private HiveMetaStoreClient client;
   private SentryKerberosContext kerberosContext;
-  private Configuration authzConf;
+  private final Configuration authzConf;
   private boolean kerberos;
-  private SentryStore sentryStore;
+  private final SentryStore sentryStore;
   private String hiveInstance;
-  final static int maxRetriesForLogin = 3;
-  final static int maxRetriesForConnection = 3;
+  private static final int maxRetriesForLogin = 3;
+  private static final int maxRetriesForConnection = 3;
 
   private volatile UpdateableAuthzPaths authzPaths;
   private boolean needHiveSnapshot = true;
+  private final LeaderStatusMonitor leaderMonitor;
 
-  HMSFollower(Configuration conf) throws Exception { //TODO: Handle any possible exceptions
or throw specific exceptions
+  HMSFollower(Configuration conf, LeaderStatusMonitor leaderMonitor)
+          throws Exception {
     LOGGER.info("HMSFollower is being initialized");
     authzConf = conf;
+    this.leaderMonitor = leaderMonitor;
     sentryStore = new SentryStore(authzConf);
     //TODO: Initialize currentEventID from Sentry db
     currentEventID = 0;
@@ -86,6 +89,7 @@ public class HMSFollower implements Runnable {
     this.authzConf = conf;
     this.sentryStore = sentryStore;
     this.hiveInstance = hiveInstance;
+    this.leaderMonitor = null;
   }
 
   @VisibleForTesting
@@ -93,10 +97,10 @@ public class HMSFollower implements Runnable {
     return connectedToHMS;
   }
 
-  /*
-  Returns HMS Client if successful, returns null if HMS is not ready yet to take connections
-  Throws @LoginException if Kerberos context creation failed using Sentry's kerberos credentials
-  Throws @MetaException if there was a problem on creating an HMSClient
+  /**
+   * Returns HMS Client if successful, returns null if HMS is not ready yet to take connections
+   * Throws @LoginException if Kerberos context creation failed using Sentry's kerberos credentials
+   * Throws @MetaException if there was a problem on creating an HMSClient
    */
   private HiveMetaStoreClient getMetaStoreClient(Configuration conf)
       throws LoginException, MetaException {
@@ -179,6 +183,7 @@ public class HMSFollower implements Runnable {
     return client;
   }
 
+  @Override
   public void run() {
     if (client == null) {
       try {
@@ -196,8 +201,13 @@ public class HMSFollower implements Runnable {
       }
     }
 
+    // Only the leader should listen to HMS updates
+    if ((leaderMonitor != null) && !leaderMonitor.isLeader()) {
+      return;
+    }
+
     try {
-      if (isNeedHiveSnapshot()) {
+      if (needHiveSnapshot) {
         // TODO: expose time used for full update in the metrics
 
         // To ensure point-in-time snapshot consistency, need to make sure
@@ -262,7 +272,8 @@ public class HMSFollower implements Runnable {
 
     try {
       updateInitializer = new FullUpdateInitializer(client, authzConf);
-      HMSFollower.this.authzPaths = updateInitializer.createInitialUpdate();
+      // TODO - do we need to save returned authz path?
+      updateInitializer.createInitialUpdate();
       // TODO: notify HDFS plugin
       LOGGER.info("#### Hive full update initialization complete !!");
     } finally {
@@ -276,24 +287,14 @@ public class HMSFollower implements Runnable {
     }
   }
 
-  private boolean isNeedHiveSnapshot() {
-    // An indicator that in request of a full hive update.
-
-    // TODO: Will need to get Hive snapshot if the Notification ID
-    // we are requesting has been rolled over in the NotificationLog
-    // table of Hive
-    return needHiveSnapshot;
-  }
-
   private boolean syncWithPolicyStore(HiveAuthzConf.AuthzConfVars syncConfVar) {
     return "true"
         .equalsIgnoreCase((authzConf.get(syncConfVar.getVar(), "true")));
   }
 
-  /*
-  Throws SentryInvalidHMSEventException if Notification event contains insufficient information
+  /**
+   * Throws SentryInvalidHMSEventException if Notification event contains insufficient information
    */
-
   void processNotificationEvents(List<NotificationEvent> events) throws
       SentryInvalidHMSEventException, SentryInvalidInputException {
     SentryJSONMessageDeserializer deserializer = new SentryJSONMessageDeserializer();

http://git-wip-us.apache.org/repos/asf/sentry/blob/c4d3b5ea/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java
b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java
index a89d3f6..be59c1e 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java
@@ -158,7 +158,7 @@ public class SentryService implements Callable, SigUtils.SigListener {
     if (notificationLogEnabled) {
       try {
         hmsFollowerExecutor = Executors.newScheduledThreadPool(1);
-        hmsFollowerExecutor.scheduleAtFixedRate(new HMSFollower(conf),
+        hmsFollowerExecutor.scheduleAtFixedRate(new HMSFollower(conf, leaderMonitor),
                 60000, 500, TimeUnit.MILLISECONDS);
       } catch (Exception e) {
         //TODO: Handle


Mime
View raw message