sentry-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ak...@apache.org
Subject sentry git commit: SENTRY-1649: Initialize HMSFollower when sentry server actually starts (Na Li, reviewed by: Alex Kolbasov)
Date Sat, 06 May 2017 03:24:40 GMT
Repository: sentry
Updated Branches:
  refs/heads/sentry-ha-redesign ab94c439a -> 18998c04e


SENTRY-1649: Initialize HMSFollower when sentry server actually starts (Na Li, reviewed by:
Alex Kolbasov)


Project: http://git-wip-us.apache.org/repos/asf/sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/18998c04
Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/18998c04
Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/18998c04

Branch: refs/heads/sentry-ha-redesign
Commit: 18998c04ed84895d5fc5f3b292ef3e2595005a63
Parents: ab94c43
Author: Alexander Kolbasov <akolb@cloudera.com>
Authored: Fri May 5 20:22:29 2017 -0700
Committer: Alexander Kolbasov <akolb@cloudera.com>
Committed: Fri May 5 20:22:29 2017 -0700

----------------------------------------------------------------------
 .../sentry/service/thrift/HMSFollower.java      |  24 ++-
 .../sentry/service/thrift/SentryService.java    | 170 +++++++++++++------
 .../service/thrift/SentryServiceUtil.java       |  41 +++++
 .../sentry/service/thrift/ServiceConstants.java |   2 +-
 .../dbprovider/AbstractTestWithDbProvider.java  |   4 +-
 .../TestDbSentryOnFailureHookLoading.java       |  10 +-
 .../tests/e2e/hdfs/TestHDFSIntegrationBase.java |   7 +-
 .../e2e/hdfs/TestHDFSIntegrationEnd2End.java    |  20 ++-
 8 files changed, 208 insertions(+), 70 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sentry/blob/18998c04/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
index 99549bc..e271ea4 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
@@ -68,10 +68,11 @@ import static org.apache.sentry.hdfs.Updateable.Update;
  * stored for HDFS-Sentry sync.
  */
 @SuppressWarnings("PMD")
-public class HMSFollower implements Runnable {
+public class HMSFollower implements Runnable, AutoCloseable {
   private static final Logger LOGGER = LoggerFactory.getLogger(HMSFollower.class);
-
   private long currentEventID;
+  // Copy from Hive
+  public static String CONF_METASTORE_URI = "hive.metastore.uris";
   // Track the latest eventId of the event that has been logged. So we don't log the same
message
   private long lastLoggedEventId = SentryStore.EMPTY_CHANGE_ID;
   private static boolean connectedToHMS = false;
@@ -85,8 +86,7 @@ public class HMSFollower implements Runnable {
   private boolean needHiveSnapshot = true;
   private final LeaderStatusMonitor leaderMonitor;
 
-  HMSFollower(Configuration conf, SentryStore store, LeaderStatusMonitor leaderMonitor)
-          throws Exception {
+  HMSFollower(Configuration conf, SentryStore store, LeaderStatusMonitor leaderMonitor) throws
Exception {
     LOGGER.info("HMSFollower is being initialized");
     authzConf = conf;
     this.leaderMonitor = leaderMonitor;
@@ -111,6 +111,12 @@ public class HMSFollower implements Runnable {
     return connectedToHMS;
   }
 
+  @Override
+  public void close() {
+    // Close any outstanding connections to HMS
+    closeHMSConnection();
+  }
+
   /**
    * Returns HMS Client if successful, returns null if HMS is not ready yet to take connections
    * Throws @LoginException if Kerberos context creation failed using Sentry's kerberos credentials
@@ -122,6 +128,12 @@ public class HMSFollower implements Runnable {
       return client;
     }
 
+    // Do not create client if the metastre URI in the configuration is missing
+    if (conf.get(CONF_METASTORE_URI, "").isEmpty()) {
+      // Come back later with real Hive URI
+      return null;
+    }
+
     final HiveConf hiveConf = new HiveConf();
     hiveInstance = hiveConf.get(HiveAuthzConf.AuthzConfVars.AUTHZ_SERVER_NAME.getVar());
 
@@ -149,7 +161,7 @@ public class HMSFollower implements Runnable {
           "Kerberos principal should have 3 parts: " + principal);
 
       keytab = Preconditions.checkNotNull(conf.get(ServiceConstants.ServerConfig.KEY_TAB),
-          ServiceConstants.ServerConfig.KEY_TAB + " is required");
+              ServiceConstants.ServerConfig.KEY_TAB + " is required");
       File keytabFile = new File(keytab);
       Preconditions.checkState(keytabFile.isFile() && keytabFile.canRead(),
           "Keytab " + keytab + " does not exist or is not readable.");
@@ -558,7 +570,7 @@ public class HMSFollower implements Runnable {
     } catch (SentryNoSuchObjectException e) {
       LOGGER.info("Drop Sentry privilege ignored as there are no privileges on the table:
%s.%s", dbName, tableName);
     } catch (Exception e) {
-      throw new SentryInvalidInputException("Could not process Create table event. Event:
" + event.toString(), e);
+      throw new SentryInvalidInputException("Could not process Drop table event. Event: "
+ event.toString(), e);
     }
   }
 

http://git-wip-us.apache.org/repos/asf/sentry/blob/18998c04/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java
b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java
index 132db63..3ea7ebf 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java
@@ -70,6 +70,7 @@ import com.google.common.base.Preconditions;
 
 import static org.apache.sentry.core.common.utils.SigUtils.registerSigListener;
 
+// Enable signal handler for HA leader/follower status if configured
 public class SentryService implements Callable, SigUtils.SigListener {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(SentryService.class);
@@ -89,6 +90,7 @@ public class SentryService implements Callable, SigUtils.SigListener {
   private final String keytab;
   private final ExecutorService serviceExecutor;
   private ScheduledExecutorService hmsFollowerExecutor = null;
+  private HMSFollower hmsFollower = null;
   private Future serviceStatus;
   private TServer thriftServer;
   private Status status;
@@ -101,8 +103,7 @@ public class SentryService implements Callable, SigUtils.SigListener {
     {@link HMSFollower}.
    */
   private final SentryStore sentryStore;
-  private final ScheduledExecutorService sentryStoreCleanService =
-      Executors.newSingleThreadScheduledExecutor();
+  private ScheduledExecutorService sentryStoreCleanService;
   private final LeaderStatusMonitor leaderMonitor;
   private final boolean notificationLogEnabled;
 
@@ -165,21 +166,6 @@ public class SentryService implements Callable, SigUtils.SigListener
{
     notificationLogEnabled = conf.getBoolean(ServerConfig.SENTRY_NOTIFICATION_LOG_ENABLED,
         ServerConfig.SENTRY_NOTIFICATION_LOG_ENABLED_DEFAULT);
 
-    if (notificationLogEnabled) {
-      try {
-        long initDelay = conf.getLong(ServerConfig.SENTRY_HMSFOLLOWER_INIT_DELAY_MILLS,
-                ServerConfig.SENTRY_HMSFOLLOWER_INIT_DELAY_MILLS_DEFAULT);
-        long period = conf.getLong(ServerConfig.SENTRY_HMSFOLLOWER_INTERVAL_MILLS,
-                ServerConfig.SENTRY_HMSFOLLOWER_INTERVAL_MILLS_DEFAULT);
-        hmsFollowerExecutor = Executors.newScheduledThreadPool(1);
-        hmsFollowerExecutor.scheduleAtFixedRate(new HMSFollower(conf, sentryStore, leaderMonitor),
-                initDelay, period, TimeUnit.MILLISECONDS);
-      } catch (Exception e) {
-        //TODO: Handle
-        LOGGER.error("Could not start HMSFollower");
-      }
-    }
-
     status = Status.NOT_STARTED;
 
     // Enable signal handler for HA leader/follower status if configured
@@ -192,25 +178,6 @@ public class SentryService implements Callable, SigUtils.SigListener
{
         LOGGER.error("Failed to register signal", e);
       }
     }
-
-    // If SENTRY_STORE_CLEAN_PERIOD_SECONDS is set to positive, the background SentryStore
cleaning
-    // thread is enabled. Currently, it only purges the delta changes {@link MSentryChange}
in
-    // the sentry store.
-    long storeCleanPeriodSecs = conf.getLong(
-        ServerConfig.SENTRY_STORE_CLEAN_PERIOD_SECONDS,
-        ServerConfig.SENTRY_STORE_CLEAN_PERIOD_SECONDS_DEFAULT);
-    if (storeCleanPeriodSecs > 0) {
-      Runnable storeCleaner = new Runnable() {
-        @Override
-        public void run() {
-          if (leaderMonitor.isLeader()) {
-            sentryStore.purgeDeltaChangeTables();
-          }
-        }
-      };
-      sentryStoreCleanService.scheduleWithFixedDelay(
-          storeCleaner, 0, storeCleanPeriodSecs, TimeUnit.SECONDS);
-    }
   }
 
   @Override
@@ -243,6 +210,10 @@ public class SentryService implements Callable, SigUtils.SigListener
{
   }
 
   private void runServer() throws Exception {
+
+    startSentryStoreCleaner(conf);
+    startHMSFollower(conf);
+
     Iterable<String> processorFactories = ConfUtilties.CLASS_SPLITTER
         .split(conf.get(ServerConfig.PROCESSOR_FACTORIES,
             ServerConfig.PROCESSOR_FACTORIES_DEFAULT).trim());
@@ -282,7 +253,7 @@ public class SentryService implements Callable, SigUtils.SigListener {
       TSaslServerTransport.Factory saslTransportFactory = new TSaslServerTransport.Factory();
       saslTransportFactory.addServerDefinition(AuthMethod.KERBEROS
           .getMechanismName(), principalParts[0], principalParts[1],
-          ServerConfig.SASL_PROPERTIES, new GSSCallback(conf));
+              ServerConfig.SASL_PROPERTIES, new GSSCallback(conf));
       transportFactory = saslTransportFactory;
     } else {
       transportFactory = new TTransportFactory();
@@ -298,6 +269,110 @@ public class SentryService implements Callable, SigUtils.SigListener
{
     thriftServer.serve();
   }
 
+  private void startHMSFollower(Configuration conf) throws Exception{
+    if (!notificationLogEnabled) {
+      return;
+    }
+
+    Preconditions.checkState(hmsFollower == null);
+    Preconditions.checkState(hmsFollowerExecutor == null);
+
+    try {
+      hmsFollower = new HMSFollower(conf, sentryStore, leaderMonitor);
+    } catch (Exception ex) {
+      LOGGER.error("Could not create HMSFollower", ex);
+      throw ex;
+    }
+
+    long initDelay = conf.getLong(ServerConfig.SENTRY_HMSFOLLOWER_INIT_DELAY_MILLS,
+            ServerConfig.SENTRY_HMSFOLLOWER_INIT_DELAY_MILLS_DEFAULT);
+    long period = conf.getLong(ServerConfig.SENTRY_HMSFOLLOWER_INTERVAL_MILLS,
+            ServerConfig.SENTRY_HMSFOLLOWER_INTERVAL_MILLS_DEFAULT);
+    try {
+      hmsFollowerExecutor = Executors.newScheduledThreadPool(1);
+      hmsFollowerExecutor.scheduleAtFixedRate(hmsFollower,
+              initDelay, period, TimeUnit.MILLISECONDS);
+    } catch (IllegalArgumentException e) {
+      LOGGER.error(String.format("Could not start HMSFollower due to illegal argument. period
is %s ms", period), e);
+      throw e;
+    }
+  }
+
+  private void stopHMSFollower(Configuration conf) {
+    if (!notificationLogEnabled) {
+      return;
+    }
+
+    Preconditions.checkNotNull(hmsFollowerExecutor);
+    Preconditions.checkNotNull(hmsFollower);
+
+    // use follower scheduling interval as timeout for shutting down its executor as
+    // such scheduling interval should be an upper bound of how long the task normally takes
to finish
+    long timeoutValue = conf.getLong(ServerConfig.SENTRY_HMSFOLLOWER_INTERVAL_MILLS,
+            ServerConfig.SENTRY_HMSFOLLOWER_INTERVAL_MILLS_DEFAULT);
+    try {
+      SentryServiceUtil.shutdownAndAwaitTermination(hmsFollowerExecutor, "hmsFollowerExecutor",
+              timeoutValue, TimeUnit.MILLISECONDS, LOGGER);
+    } finally {
+      hmsFollowerExecutor = null;
+      try {
+        // close connections
+        hmsFollower.close();
+      } catch (RuntimeException ex) {
+        LOGGER.error("HMSFollower.close() failed", ex);
+      } finally {
+        hmsFollower = null;
+      }
+    }
+  }
+
+  private void startSentryStoreCleaner(Configuration conf) {
+    Preconditions.checkState(sentryStoreCleanService == null);
+
+    // If SENTRY_STORE_CLEAN_PERIOD_SECONDS is set to positive, the background SentryStore
cleaning
+    // thread is enabled. Currently, it only purges the delta changes {@link MSentryChange}
in
+    // the sentry store.
+    long storeCleanPeriodSecs = conf.getLong(
+            ServerConfig.SENTRY_STORE_CLEAN_PERIOD_SECONDS,
+            ServerConfig.SENTRY_STORE_CLEAN_PERIOD_SECONDS_DEFAULT);
+    if (storeCleanPeriodSecs <= 0) {
+      return;
+    }
+
+    try {
+      Runnable storeCleaner = new Runnable() {
+        @Override
+        public void run() {
+          if (leaderMonitor.isLeader()) {
+            sentryStore.purgeDeltaChangeTables();
+          }
+        }
+      };
+
+      sentryStoreCleanService = Executors.newSingleThreadScheduledExecutor();
+      sentryStoreCleanService.scheduleWithFixedDelay(
+              storeCleaner, 0, storeCleanPeriodSecs, TimeUnit.SECONDS);
+
+      LOGGER.info("sentry store cleaner is scheduled with interval %d seconds", storeCleanPeriodSecs);
+    }
+    catch(IllegalArgumentException e){
+      LOGGER.error("Could not start SentryStoreCleaner due to illegal argument", e);
+      sentryStoreCleanService = null;
+    }
+  }
+
+  private void stopSentryStoreCleaner() {
+    Preconditions.checkNotNull(sentryStoreCleanService);
+
+    try {
+      SentryServiceUtil.shutdownAndAwaitTermination(sentryStoreCleanService, "sentryStoreCleanService",
+              10, TimeUnit.SECONDS, LOGGER);
+    }
+    finally {
+      sentryStoreCleanService = null;
+    }
+  }
+
   private void addSentryServiceGauge() {
     SentryMetrics.getInstance().addSentryServiceGauges(this);
   }
@@ -312,7 +387,6 @@ public class SentryService implements Callable, SigUtils.SigListener {
       sentryWebServer = new SentryWebServer(listenerList, webServerPort, conf);
       sentryWebServer.start();
     }
-
   }
 
   private void stopSentryWebServer() throws Exception{
@@ -369,20 +443,10 @@ public class SentryService implements Callable, SigUtils.SigListener
{
     } else {
       LOGGER.info("Sentry web service is already stopped...");
     }
-    if(hmsFollowerExecutor != null) {
-      hmsFollowerExecutor.shutdown();
-    }
-    sentryStoreCleanService.shutdown();
-    try {
-      if (!sentryStoreCleanService.awaitTermination(10, TimeUnit.SECONDS)) {
-        sentryStoreCleanService.shutdownNow();
-        if (!sentryStoreCleanService.awaitTermination(10, TimeUnit.SECONDS)) {
-          LOGGER.error("DeltaCleanerService did not terminate");
-        }
-      }
-    } catch (InterruptedException ie) {
-      sentryStoreCleanService.shutdownNow();
-    }
+
+    stopHMSFollower(conf);
+    stopSentryStoreCleaner();
+
     if (exception != null) {
       exception.ifExceptionThrow();
     }
@@ -536,4 +600,4 @@ public class SentryService implements Callable, SigUtils.SigListener {
     // Become follower
     leaderMonitor.deactivate();
   }
-}
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/sentry/blob/18998c04/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceUtil.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceUtil.java
b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceUtil.java
index ce73358..4019e61 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceUtil.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceUtil.java
@@ -21,7 +21,10 @@ package org.apache.sentry.service.thrift;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
 
+import com.google.common.base.Preconditions;
 import org.apache.commons.lang.StringUtils;
 import org.apache.sentry.core.common.utils.SentryConstants;
 import org.apache.sentry.core.common.utils.KeyValue;
@@ -31,6 +34,7 @@ import org.apache.sentry.provider.db.service.thrift.TSentryPrivilege;
 import org.apache.sentry.service.thrift.ServiceConstants.PrivilegeScope;
 
 import com.google.common.collect.Lists;
+import org.slf4j.Logger;
 
 public final class SentryServiceUtil {
 
@@ -151,6 +155,43 @@ public final class SentryServiceUtil {
     return SentryConstants.AUTHORIZABLE_JOINER.join(privileges);
   }
 
+  /**
+   * Gracefully shut down an Executor service.
+   * <p>
+   * This code is based on the Javadoc example for the Executor service.
+   * <p>
+   * First call shutdown to reject incoming tasks, and then call
+   * shutdownNow, if necessary, to cancel any lingering tasks.
+   *
+   * @param pool the executor service to shut down
+   * @param poolName the name of the executor service to shut down to make it easy for debugging
+   * @param timeout the timeout interval to wait for its termination
+   * @param unit the unit of the timeout
+   * @param logger the logger to log the error message if it cannot terminate. It could be
null
+   */
+  static void shutdownAndAwaitTermination(ExecutorService pool, String poolName,
+                       long timeout, TimeUnit unit, Logger logger) {
+    Preconditions.checkNotNull(pool);
+
+    pool.shutdown(); // Disable new tasks from being submitted
+    try {
+      // Wait a while for existing tasks to terminate
+      if (!pool.awaitTermination(timeout, unit)) {
+        pool.shutdownNow(); // Cancel currently executing tasks
+        // Wait a while for tasks to respond to being cancelled
+        if ((!pool.awaitTermination(timeout, unit)) && (logger != null)) {
+          logger.error("Executor service {} did not terminate",
+              StringUtils.defaultIfBlank(poolName, "null"));
+        }
+      }
+    } catch (InterruptedException ignored) {
+      // (Re-)Cancel if current thread also interrupted
+      pool.shutdownNow();
+      // Preserve interrupt status
+      Thread.currentThread().interrupt();
+    }
+  }
+
   private SentryServiceUtil() {
     // Make constructor private to avoid instantiation
   }

http://git-wip-us.apache.org/repos/asf/sentry/blob/18998c04/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java
b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java
index 834ed41..638c079 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java
@@ -169,7 +169,7 @@ public class ServiceConstants {
 
     // InitialDelay and period time for HMSFollower thread.
     public static final String SENTRY_HMSFOLLOWER_INIT_DELAY_MILLS = "sentry.hmsfollower.init.delay.mills";
-    public static final long SENTRY_HMSFOLLOWER_INIT_DELAY_MILLS_DEFAULT = 60000;
+    public static final long SENTRY_HMSFOLLOWER_INIT_DELAY_MILLS_DEFAULT = 0;
     public static final String SENTRY_HMSFOLLOWER_INTERVAL_MILLS = "sentry.hmsfollower.interval.mills";
     public static final long SENTRY_HMSFOLLOWER_INTERVAL_MILLS_DEFAULT = 500;
 

http://git-wip-us.apache.org/repos/asf/sentry/blob/18998c04/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/AbstractTestWithDbProvider.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/AbstractTestWithDbProvider.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/AbstractTestWithDbProvider.java
index 2ebe561..17a2d1e 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/AbstractTestWithDbProvider.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/AbstractTestWithDbProvider.java
@@ -43,8 +43,6 @@ import org.apache.sentry.tests.e2e.hive.hiveserver.HiveServerFactory;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
-import static org.junit.Assume.assumeNotNull;
-
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Maps;
 import com.google.common.io.Files;
@@ -107,7 +105,7 @@ public abstract class AbstractTestWithDbProvider extends AbstractTestWithHiveSer
       properties.put(ClientConfig.SERVER_RPC_PORT,
           String.valueOf(server.getAddress().getPort()));
     }
-    assumeNotNull(context);
+
     context = AbstractTestWithHiveServer.createContext(properties);
     policyFile
         .setUserGroupMapping(StaticUserGroup.getStaticMapping())

http://git-wip-us.apache.org/repos/asf/sentry/blob/18998c04/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbSentryOnFailureHookLoading.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbSentryOnFailureHookLoading.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbSentryOnFailureHookLoading.java
index 212c465..aa99595 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbSentryOnFailureHookLoading.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbSentryOnFailureHookLoading.java
@@ -36,9 +36,12 @@ import org.junit.After;
 import org.junit.Assume;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class TestDbSentryOnFailureHookLoading extends AbstractTestWithDbProvider {
 
+  private static final Logger LOGGER = LoggerFactory.getLogger(TestDbSentryOnFailureHookLoading.class);
 
   @BeforeClass
   public static void setup() throws Exception {
@@ -265,7 +268,12 @@ public class TestDbSentryOnFailureHookLoading extends AbstractTestWithDbProvider
     }
     if(dbName != null) {
       Assert.assertNotNull("Database object is null for op: " + expectedOp, DummySentryOnFailureHook.db);
-      Assert.assertTrue(dbName.equalsIgnoreCase(DummySentryOnFailureHook.db.getName()));
+
+      String failureHookDbName = DummySentryOnFailureHook.db.getName();
+      LOGGER.debug("dbName: {}; DummySentryOnFailureHook.db.getName(): {}", dbName, failureHookDbName);
+      if(!failureHookDbName.equalsIgnoreCase("*")) {
+        Assert.assertTrue(dbName.equalsIgnoreCase(failureHookDbName));
+      }
     }
   }
 }

http://git-wip-us.apache.org/repos/asf/sentry/blob/18998c04/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java
index b5247d0..32acc65 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java
@@ -147,8 +147,11 @@ public abstract class TestHDFSIntegrationBase {
   protected static SentrySrv sentryServer;
   protected static boolean testSentryHA = false;
   protected static final long STALE_THRESHOLD = 5000;
-  protected static final long CACHE_REFRESH = 100; //Default is 500, but we want it to be
low
-  // in our tests so that changes reflect soon
+
+  // we want to make sure the cache is updated in our tests so that changes reflect soon
+  protected static final long CACHE_REFRESH =
+          ServerConfig.SENTRY_HMSFOLLOWER_INIT_DELAY_MILLS_DEFAULT +
+                  ServerConfig.SENTRY_HMSFOLLOWER_INTERVAL_MILLS_DEFAULT * 2;
 
   protected static String fsURI;
   protected static int hmsPort;

http://git-wip-us.apache.org/repos/asf/sentry/blob/18998c04/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationEnd2End.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationEnd2End.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationEnd2End.java
index 1ace07c..548dcf8 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationEnd2End.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationEnd2End.java
@@ -44,6 +44,8 @@ public class TestHDFSIntegrationEnd2End extends TestHDFSIntegrationBase
{
   private static final Logger LOGGER = LoggerFactory
       .getLogger(TestHDFSIntegrationEnd2End.class);
 
+  private static String adminRole = "admin_role";
+
   @Ignore
   @Test
   public void testEnd2End() throws Throwable {
@@ -374,8 +376,10 @@ public class TestHDFSIntegrationEnd2End extends TestHDFSIntegrationBase
{
   }
 
   //SENTRY-780
+  @Ignore // SENTRY-1750 HMSFollower does not handle view update correctly. Enable this test
once SENTRY-1750 is fixed
   @Test
   public void testViews() throws Throwable {
+    LOGGER.info("testViews starts");
     String dbName= "db1";
 
     tmpHDFSDir = new Path("/tmp/external");
@@ -407,6 +411,7 @@ public class TestHDFSIntegrationEnd2End extends TestHDFSIntegrationBase
{
 
     stmt.close();
     conn.close();
+    LOGGER.info("testViews ends");
   }
 
   /*
@@ -414,11 +419,13 @@ TODO:SENTRY-819
 */
   @Test
   public void testAllColumn() throws Throwable {
+    LOGGER.info("testAllColumn starts");
     String dbName = "db2";
+    String userRole = "col1_role";
 
     tmpHDFSDir = new Path("/tmp/external");
     dbNames = new String[]{dbName};
-    roles = new String[]{"admin_role", "col_role"};
+    roles = new String[]{"admin_role", userRole};
     admin = StaticUserGroup.ADMIN1;
 
     Connection conn;
@@ -438,9 +445,9 @@ TODO:SENTRY-819
     stmt.execute("alter table p1 add partition (month=1, day=1)");
     loadDataTwoCols(stmt);
 
-    stmt.execute("create role col_role");
-    stmt.execute("grant select(c1,c2) on p1 to role col_role");
-    stmt.execute("grant role col_role to group "+ StaticUserGroup.USERGROUP1);
+    stmt.execute("create role " + userRole);
+    stmt.execute("grant select(c1,c2) on p1 to role " + userRole);
+    stmt.execute("grant role " + userRole + " to group "+ StaticUserGroup.USERGROUP1);
     Thread.sleep(100);
 
     //User with privileges on all columns of the data cannot still read the HDFS files
@@ -448,15 +455,19 @@ TODO:SENTRY-819
 
     stmt.close();
     conn.close();
+    LOGGER.info("testAllColumn ends");
   }
 
   @Test
   public void testColumnPrivileges() throws Throwable {
+    LOGGER.info("testColumnPrivileges starts");
     String dbName = "db2";
 
     tmpHDFSDir = new Path("/tmp/external");
     dbNames = new String[]{dbName};
+      dbNames = new String[]{dbName};
     roles = new String[]{"admin_role", "tab_role", "db_role", "col_role"};
+    roles = new String[]{adminRole, "tab_role", "db_role", "col_role"};
     admin = StaticUserGroup.ADMIN1;
 
     Connection conn;
@@ -513,6 +524,7 @@ TODO:SENTRY-819
 
     stmt.close();
     conn.close();
+    LOGGER.info("testColumnPrivileges ends");
   }
 
 


Mime
View raw message