sentry-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ha...@apache.org
Subject [2/2] sentry git commit: SENTRY-1499: Add feature flag for using NotificationLog (Hao Hao, Reviewed by: Sravya Tirukkovalur, Kalyan Kalvagadda, Alexander Kolbasov and Vamsee Yarlagadda)
Date Tue, 10 Jan 2017 00:11:21 GMT
SENTRY-1499: Add feature flag for using NotificationLog (Hao Hao, Reviewed by: Sravya Tirukkovalur, Kalyan Kalvagadda, Alexander Kolbasov and Vamsee Yarlagadda)

Change-Id: I8e817357b9c833f0aed0e3732b159c866d952f97


Project: http://git-wip-us.apache.org/repos/asf/sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/b34a736a
Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/b34a736a
Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/b34a736a

Branch: refs/heads/sentry-ha-redesign
Commit: b34a736a64a60790651b39fd77d0a9ce39cd3c2a
Parents: 5e6da0c
Author: hahao <hao.hao@cloudera.com>
Authored: Mon Jan 9 15:52:29 2017 -0800
Committer: hahao <hao.hao@cloudera.com>
Committed: Mon Jan 9 16:01:19 2017 -0800

----------------------------------------------------------------------
 .../SentryMetastorePostEventListener.java       | 523 ++++++++++---------
 ...tastorePostEventListenerNotificationLog.java | 395 ++++++++++++++
 .../sentry/service/thrift/SentryService.java    |  22 +-
 .../sentry/service/thrift/ServiceConstants.java |   2 +
 .../AbstractTestWithStaticConfiguration.java    |  16 +-
 .../TestSentryListenerInBuiltDeserializer.java  |   1 +
 .../TestSentryListenerSentryDeserializer.java   |   1 +
 7 files changed, 702 insertions(+), 258 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sentry/blob/b34a736a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryMetastorePostEventListener.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryMetastorePostEventListener.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryMetastorePostEventListener.java
index 75190c1..fdb6df4 100644
--- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryMetastorePostEventListener.java
+++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryMetastorePostEventListener.java
@@ -17,16 +17,15 @@
  */
 package org.apache.sentry.binding.metastore;
 
-import java.util.concurrent.TimeUnit;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.MetaStoreEventListener;
-import org.apache.hadoop.hive.metastore.RawStore;
-import org.apache.hadoop.hive.metastore.RawStoreProxy;
-import org.apache.hadoop.hive.metastore.TableType;
 import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.NotificationEvent;
+import org.apache.hadoop.hive.metastore.api.Partition;
 import org.apache.hadoop.hive.metastore.events.AddPartitionEvent;
 import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent;
 import org.apache.hadoop.hive.metastore.events.AlterTableEvent;
@@ -35,360 +34,386 @@ import org.apache.hadoop.hive.metastore.events.CreateTableEvent;
 import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent;
 import org.apache.hadoop.hive.metastore.events.DropPartitionEvent;
 import org.apache.hadoop.hive.metastore.events.DropTableEvent;
-import org.apache.hive.hcatalog.common.HCatConstants;
-import org.apache.sentry.binding.metastore.messaging.json.SentryJSONMessageFactory;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.sentry.core.common.exception.SentryUserException;
+import org.apache.sentry.binding.hive.conf.HiveAuthzConf;
+import org.apache.sentry.binding.hive.conf.HiveAuthzConf.AuthzConfVars;
+import org.apache.sentry.core.common.Authorizable;
+import org.apache.sentry.core.model.db.Database;
+import org.apache.sentry.core.model.db.Server;
+import org.apache.sentry.core.model.db.Table;
 import org.apache.sentry.provider.db.SentryMetastoreListenerPlugin;
+import org.apache.sentry.provider.db.service.thrift.SentryPolicyServiceClient;
+import org.apache.sentry.service.thrift.SentryServiceClientFactory;
+import org.apache.sentry.service.thrift.ServiceConstants.ConfUtilties;
+import org.apache.sentry.service.thrift.ServiceConstants.ServerConfig;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.commons.lang3.builder.ToStringBuilder;
-/*
-A HMS listener class which should ideally go into the transaction which persists the Hive metadata.
-This class writes all DDL events to the NotificationLog through rawstore.addNotificationEvent(event)
-This class is very similar to DbNotificationListener, except:
-1. It uses a custom SentryJSONMessageFactory which adds additional information to the message part of the event
- to avoid another round trip from the clients
-2. It handles the cases where actual operation has failed, and hence skips writing to the notification log.
-3. Has additional validations to make sure event has the required information.
-
-This can be replaced with DbNotificationListener in future and sentry's message factory can be plugged in if:
-- HIVE-14011 is fixed: Make MessageFactory truly pluggable
-- 2 and 3 above are handled in DbNotificationListener
-*/
 
+/**
+ * SentryMetastorePostEventListener class is HMS plugin for listening to
+ * all DDL events and deliver those events to Sentry server. This class
+ * sends all DDL events to the Sentry server through thrift API.
+ *
+ * In case any actual event fails, skipping deliver the event to Sentry server.
+ */
 public class SentryMetastorePostEventListener extends MetaStoreEventListener {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(SentryMetastoreListenerPlugin.class);
-  private RawStore rs;
-  private HiveConf hiveConf;
-  SentryJSONMessageFactory messageFactory;
-
-  private static SentryMetastorePostEventListener.CleanerThread cleaner = null;
-
-  //Same as DbNotificationListener to make the transition back easy
-  private synchronized void init(HiveConf conf) {
-    try {
-      this.rs = RawStoreProxy.getProxy(conf, conf, conf.getVar(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL), 999999);
-    } catch (MetaException var3) {
-      LOGGER.error("Unable to connect to raw store, notifications will not be tracked", var3);
-      this.rs = null;
-    }
+  private final HiveAuthzConf authzConf;
+  private final Server server;
 
-    if(cleaner == null && this.rs != null) {
-      cleaner = new SentryMetastorePostEventListener.CleanerThread(conf, this.rs);
-      cleaner.start();
-    }
-  }
+  private List<SentryMetastoreListenerPlugin> sentryPlugins = new ArrayList<SentryMetastoreListenerPlugin>();
 
   public SentryMetastorePostEventListener(Configuration config) {
     super(config);
-    // The code in MetastoreUtils.getMetaStoreListeners() that calls this looks for a constructor
-    // with a Configuration parameter, so we have to declare config as Configuration.  But it
-    // actually passes a HiveConf, which we need.  So we'll do this ugly down cast.
+
     if (!(config instanceof HiveConf)) {
-      String error = "Could not initialize Plugin - Configuration is not an instanceof HiveConf";
-      LOGGER.error(error);
-      throw new RuntimeException(error);
+        String error = "Could not initialize Plugin - Configuration is not an instanceof HiveConf";
+        LOGGER.error(error);
+        throw new RuntimeException(error);
+    }
+
+    authzConf = HiveAuthzConf.getAuthzConf((HiveConf)config);
+    server = new Server(authzConf.get(AuthzConfVars.AUTHZ_SERVER_NAME.getVar()));
+    Iterable<String> pluginClasses = ConfUtilties.CLASS_SPLITTER
+        .split(config.get(ServerConfig.SENTRY_METASTORE_PLUGINS, ServerConfig.SENTRY_METASTORE_PLUGINS_DEFAULT).trim());
+
+    try {
+      for (String pluginClassStr : pluginClasses) {
+        Class<?> clazz = config.getClassByName(pluginClassStr);
+        if (!SentryMetastoreListenerPlugin.class.isAssignableFrom(clazz)) {
+          throw new IllegalArgumentException("Class \\"
+              + pluginClassStr + "\\ is not a "
+              + SentryMetastoreListenerPlugin.class.getName());
+        }
+        SentryMetastoreListenerPlugin plugin = (SentryMetastoreListenerPlugin) clazz
+            .getConstructor(Configuration.class, Configuration.class)
+            .newInstance(config, authzConf);
+        sentryPlugins.add(plugin);
+      }
+    } catch (Exception e) {
+      LOGGER.error("Could not initialize HMS Plugin: SentryMetastorePostEventListener !!", e);
+      throw new RuntimeException(e);
     }
-    hiveConf = (HiveConf)config;
-    messageFactory = new SentryJSONMessageFactory();
-    init(hiveConf);
   }
 
   @Override
-  public void onCreateDatabase(CreateDatabaseEvent dbEvent)
-          throws MetaException {
+  public void onCreateTable (CreateTableEvent tableEvent) throws MetaException {
 
-    // do not write to Notification log if the operation has failed
-    if (!dbEvent.getStatus()) {
-      LOGGER.info("Skipping writing to NotificationLog as the Create database event failed");
+    // don't sync paths/privileges if the operation has failed
+    if (!tableEvent.getStatus()) {
+      LOGGER.debug("Skip sync paths/privileges with Sentry server for onCreateTable event," +
+        " since the operation failed. \n");
       return;
     }
 
-    String location = dbEvent.getDatabase().getLocationUri();
-    if (location == null || location.isEmpty()) {
-      throw new SentryMalformedEventException("CreateDatabaseEvent has invalid location", dbEvent);
+    if (tableEvent.getTable().getSd().getLocation() != null) {
+      String authzObj = tableEvent.getTable().getDbName() + "."
+          + tableEvent.getTable().getTableName();
+      String path = tableEvent.getTable().getSd().getLocation();
+      for (SentryMetastoreListenerPlugin plugin : sentryPlugins) {
+        plugin.addPath(authzObj, path);
+      }
     }
-    String dbName = dbEvent.getDatabase().getName();
-    if (dbName == null || dbName.isEmpty()) {
-      throw new SentryMalformedEventException("CreateDatabaseEvent has invalid dbName", dbEvent);
+
+    // drop the privileges on the given table, in case if anything was left
+    // behind during the drop
+    if (!syncWithPolicyStore(AuthzConfVars.AUTHZ_SYNC_CREATE_WITH_POLICY_STORE)) {
+      return;
     }
 
-    NotificationEvent event = new NotificationEvent(0L, now(), HCatConstants.HCAT_CREATE_DATABASE_EVENT,
-            messageFactory.buildCreateDatabaseMessage(dbEvent.getDatabase()).toString());
-    event.setDbName(dbName);
-    this.enqueue(event);
+    dropSentryTablePrivilege(tableEvent.getTable().getDbName(),
+        tableEvent.getTable().getTableName());
   }
 
   @Override
-  public void onDropDatabase(DropDatabaseEvent dbEvent) throws MetaException {
+  public void onDropTable(DropTableEvent tableEvent) throws MetaException {
 
-    // do not write to Notification log if the operation has failed
-    if (!dbEvent.getStatus()) {
-      LOGGER.info("Skipping writing to NotificationLog as the Drop database event failed");
+    // don't sync paths/privileges if the operation has failed
+    if (!tableEvent.getStatus()) {
+      LOGGER.debug("Skip syncing paths/privileges with Sentry server for onDropTable event," +
+        " since the operation failed. \n");
+      return;
+    }
+
+    if (tableEvent.getTable().getSd().getLocation() != null) {
+      String authzObj = tableEvent.getTable().getDbName() + "."
+          + tableEvent.getTable().getTableName();
+      for (SentryMetastoreListenerPlugin plugin : sentryPlugins) {
+        plugin.removeAllPaths(authzObj, null);
+      }
+    }
+    // drop the privileges on the given table
+    if (!syncWithPolicyStore(AuthzConfVars.AUTHZ_SYNC_DROP_WITH_POLICY_STORE)) {
       return;
     }
 
-    String dbName = dbEvent.getDatabase().getName();
-    if (dbName == null || dbName.isEmpty()) {
-      throw new SentryMalformedEventException("DropDatabaseEvent has invalid dbName", dbEvent);
+    if (!tableEvent.getStatus()) {
+      return;
     }
 
-    NotificationEvent event = new NotificationEvent(0L, now(), HCatConstants.HCAT_DROP_DATABASE_EVENT,
-            messageFactory.buildDropDatabaseMessage(dbEvent.getDatabase()).toString());
-    event.setDbName(dbName);
-    this.enqueue(event);
+    dropSentryTablePrivilege(tableEvent.getTable().getDbName(),
+        tableEvent.getTable().getTableName());
   }
 
   @Override
-  public void onCreateTable (CreateTableEvent tableEvent) throws MetaException {
+  public void onCreateDatabase(CreateDatabaseEvent dbEvent)
+      throws MetaException {
 
-    // do not write to Notification log if the operation has failed
-    if (!tableEvent.getStatus()) {
-      LOGGER.info("Skipping writing to NotificationLog as the Create table event failed");
+    // don't sync paths/privileges if the operation has failed
+    if (!dbEvent.getStatus()) {
+      LOGGER.debug("Skip syncing paths/privileges with Sentry server for onCreateDatabase event," +
+        " since the operation failed. \n");
       return;
     }
 
-    String dbName = tableEvent.getTable().getDbName();
-    if (dbName == null || dbName.isEmpty()) {
-      throw new SentryMalformedEventException("CreateTableEvent has invalid dbName", tableEvent);
-    }
-    String tableName = tableEvent.getTable().getTableName();
-    if (tableName == null || tableName.isEmpty()) {
-      throw new SentryMalformedEventException("CreateTableEvent has invalid tableName", tableEvent);
-    }
-    // Create table event should also contain a location.
-    // But, Create view also generates a Create table event, but it does not have a location.
-    // Create view is identified by the tableType. But turns out tableType is not set in some cases.
-    // We assume that tableType is set for all create views.
-    //TODO: Location can be null/empty, handle that in HMSFollower
-    String tableType = tableEvent.getTable().getTableType();
-    if(!(tableType != null && tableType.equals(TableType.VIRTUAL_VIEW.name()))) {
-        if (tableType == null) {
-        LOGGER.warn("TableType is null, assuming it is not TableType.VIRTUAL_VIEW: tableEvent", tableEvent);
-      }
-      String location = tableEvent.getTable().getSd().getLocation();
-      if (location == null || location.isEmpty()) {
-        throw new SentryMalformedEventException("CreateTableEvent has invalid location", tableEvent);
+    if (dbEvent.getDatabase().getLocationUri() != null) {
+      String authzObj = dbEvent.getDatabase().getName();
+      String path = dbEvent.getDatabase().getLocationUri();
+      for (SentryMetastoreListenerPlugin plugin : sentryPlugins) {
+        plugin.addPath(authzObj, path);
       }
     }
-    NotificationEvent event = new NotificationEvent(0L, now(), HCatConstants.HCAT_CREATE_TABLE_EVENT,
-            messageFactory.buildCreateTableMessage(tableEvent.getTable()).toString());
-    event.setDbName(dbName);
-    event.setTableName(tableName);
-    this.enqueue(event);
+    // drop the privileges on the database, in case anything left behind during
+    // last drop db
+    if (!syncWithPolicyStore(AuthzConfVars.AUTHZ_SYNC_CREATE_WITH_POLICY_STORE)) {
+      return;
+    }
+
+    dropSentryDbPrivileges(dbEvent.getDatabase().getName());
   }
 
+  /**
+   * Drop the privileges on the database. Note that child tables will be
+   * dropped individually by client, so we just need to handle the removing
+   * the db privileges. The table drop should cleanup the table privileges.
+   */
   @Override
-  public void onDropTable(DropTableEvent tableEvent) throws MetaException {
+  public void onDropDatabase(DropDatabaseEvent dbEvent) throws MetaException {
 
-    // do not write to Notification log if the operation has failed
-    if (!tableEvent.getStatus()) {
-      LOGGER.info("Skipping writing to NotificationLog as the Drop table event failed");
+    // don't sync paths/privileges if the operation has failed
+    if (!dbEvent.getStatus()) {
+      LOGGER.debug("Skip syncing paths/privileges with Sentry server for onDropDatabase event," + " since the operation failed. \n");
       return;
     }
 
-    String dbName = tableEvent.getTable().getDbName();
-    if (dbName == null || dbName.isEmpty()) {
-      throw new SentryMalformedEventException("DropTableEvent has invalid dbName", tableEvent);
+    String authzObj = dbEvent.getDatabase().getName();
+    for (SentryMetastoreListenerPlugin plugin : sentryPlugins) {
+      List<String> tNames = dbEvent.getHandler().get_all_tables(authzObj);
+      plugin.removeAllPaths(authzObj, tNames);
     }
-    String tableName = tableEvent.getTable().getTableName();
-    if (tableName == null || tableName.isEmpty()) {
-      throw new SentryMalformedEventException("DropTableEvent has invalid tableName", tableEvent);
+    if (!syncWithPolicyStore(AuthzConfVars.AUTHZ_SYNC_DROP_WITH_POLICY_STORE)) {
+      return;
     }
 
-    NotificationEvent event = new NotificationEvent(0L, now(), HCatConstants.HCAT_DROP_TABLE_EVENT,
-            messageFactory.buildDropTableMessage(tableEvent.getTable()).toString());
-    event.setDbName(dbName);
-    event.setTableName(tableName);
-    this.enqueue(event);
+    dropSentryDbPrivileges(dbEvent.getDatabase().getName());
   }
 
+  /**
+   * Adjust the privileges when table is renamed
+   */
   @Override
   public void onAlterTable (AlterTableEvent tableEvent) throws MetaException {
 
-    // do not write to Notification log if the operation has failed
+    // don't sync privileges if the operation has failed
     if (!tableEvent.getStatus()) {
-      LOGGER.info("Skipping writing to NotificationLog as the Alter table event failed");
+      LOGGER.debug("Skip syncing privileges with Sentry server for onAlterTable event," +
+        " since the operation failed. \n");
       return;
     }
 
-    String dbName = tableEvent.getNewTable().getDbName();
-    if (dbName == null || dbName.isEmpty()) {
-      throw new SentryMalformedEventException("AlterTableEvent's newTable has invalid dbName", tableEvent);
-    }
-    String tableName = tableEvent.getNewTable().getTableName();
-    if (tableName == null || tableName.isEmpty()) {
-      throw new SentryMalformedEventException("AlterTableEvent's newTable has invalid tableName", tableEvent);
-    }
-    dbName = tableEvent.getOldTable().getDbName();
-    if (dbName == null || dbName.isEmpty()) {
-      throw new SentryMalformedEventException("AlterTableEvent's oldTable has invalid dbName", tableEvent);
+    String oldLoc = null, newLoc = null;
+
+    org.apache.hadoop.hive.metastore.api.Table oldTal = tableEvent.getOldTable();
+    org.apache.hadoop.hive.metastore.api.Table newTal = tableEvent.getNewTable();
+
+    if(oldTal != null && oldTal.getSd() !=null) {
+      oldLoc = oldTal.getSd().getLocation();
     }
-    tableName = tableEvent.getOldTable().getTableName();
-    if (tableName == null || tableName.isEmpty()) {
-      throw new SentryMalformedEventException("AlterTableEvent's oldTable has invalid tableName", tableEvent);
+    if (newTal != null && newTal.getSd() != null) {
+      newLoc = newTal.getSd().getLocation();
     }
-    //Alter view also generates an alter table event, but it does not have a location
-    //TODO: Handle this case in Sentry
-    if(!tableEvent.getOldTable().getTableType().equals(TableType.VIRTUAL_VIEW.name())) {
-      String location = tableEvent.getNewTable().getSd().getLocation();
-      if (location == null || location.isEmpty()) {
-        throw new SentryMalformedEventException("AlterTableEvent's newTable has invalid location", tableEvent);
-      }
-      location = tableEvent.getOldTable().getSd().getLocation();
-      if (location == null || location.isEmpty()) {
-        throw new SentryMalformedEventException("AlterTableEvent's oldTable has invalid location", tableEvent);
-      }
+    if(oldLoc != null && newLoc != null && !oldLoc.equals(newLoc)) {
+      String oldDbName = tableEvent.getOldTable().getDbName();
+      String oldTbName = tableEvent.getOldTable().getTableName();
+      String newTbName = tableEvent.getNewTable().getTableName();
+      String newDbName = tableEvent.getNewTable().getDbName();
+      renameSentryTablePrivilege(oldDbName, oldTbName, oldLoc, newDbName, newTbName, newLoc);
     }
-
-    NotificationEvent event = new NotificationEvent(0L, now(), HCatConstants.HCAT_ALTER_TABLE_EVENT,
-            messageFactory.buildAlterTableMessage(tableEvent.getOldTable(), tableEvent.getNewTable()).toString());
-    event.setDbName(tableEvent.getNewTable().getDbName());
-    event.setTableName(tableEvent.getNewTable().getTableName());
-    this.enqueue(event);
   }
 
   @Override
   public void onAlterPartition(AlterPartitionEvent partitionEvent)
-          throws MetaException {
+      throws MetaException {
 
-    // do not write to Notification log if the operation has failed
+    // don't sync privileges if the operation has failed
     if (!partitionEvent.getStatus()) {
-      LOGGER.info("Skipping writing to NotificationLog as the Alter partition event failed");
+      LOGGER.debug("Skip syncing privileges with Sentry server for onAlterPartition event," +
+        " since the operation failed. \n");
       return;
     }
 
-    String dbName = partitionEvent.getNewPartition().getDbName();
-    if (dbName == null || dbName.isEmpty()) {
-      throw new SentryMalformedEventException("AlterPartitionEvent's newPartition has invalid dbName", partitionEvent);
+    String oldLoc = null, newLoc = null;
+    if (partitionEvent.getOldPartition() != null) {
+      oldLoc = partitionEvent.getOldPartition().getSd().getLocation();
     }
-    String tableName = partitionEvent.getNewPartition().getTableName();
-    if (tableName == null || tableName.isEmpty()) {
-      throw new SentryMalformedEventException("AlterPartitionEvent's newPartition has invalid tableName", partitionEvent);
+    if (partitionEvent.getNewPartition() != null) {
+      newLoc = partitionEvent.getNewPartition().getSd().getLocation();
     }
 
-    //TODO: Need more validations, but it is tricky as there are many variations and validations change for each one
-    // Alter partition Location
-    // Alter partition property
-    // Any more?
-
-    NotificationEvent event = new NotificationEvent(0L, now(), HCatConstants.HCAT_ALTER_PARTITION_EVENT,
-            messageFactory.buildAlterPartitionMessage(partitionEvent.getOldPartition(), partitionEvent.getNewPartition()).toString());
-
-    event.setDbName(partitionEvent.getNewPartition().getDbName());
-    event.setTableName(partitionEvent.getNewPartition().getTableName());
-    this.enqueue(event);
+    if (oldLoc != null && newLoc != null && !oldLoc.equals(newLoc)) {
+      String authzObj =
+          partitionEvent.getOldPartition().getDbName() + "."
+              + partitionEvent.getOldPartition().getTableName();
+      for (SentryMetastoreListenerPlugin plugin : sentryPlugins) {
+        plugin.renameAuthzObject(authzObj, oldLoc,
+            authzObj, newLoc);
+      }
+    }
   }
 
   @Override
   public void onAddPartition(AddPartitionEvent partitionEvent)
-          throws MetaException {
+      throws MetaException {
 
-    // do not write to Notification log if the operation has failed
+    // don't sync path if the operation has failed
     if (!partitionEvent.getStatus()) {
-      LOGGER.info("Skipping writing to NotificationLog as the Add partition event failed");
+      LOGGER.debug("Skip syncing path with Sentry server for onAddPartition event," + " since the operation failed. \n");
       return;
     }
 
-    String dbName = partitionEvent.getTable().getDbName();
-    if (dbName == null || dbName.isEmpty()) {
-      throw new SentryMalformedEventException("AddPartitionEvent has invalid dbName", partitionEvent);
-    }
-    String tableName = partitionEvent.getTable().getTableName();
-    if (tableName == null || tableName.isEmpty()) {
-      throw new SentryMalformedEventException("AddPartitionEvent's newPartition has invalid tableName", partitionEvent);
+    for (Partition part : partitionEvent.getPartitions()) {
+      if (part.getSd() != null && part.getSd().getLocation() != null) {
+        String authzObj = part.getDbName() + "." + part.getTableName();
+        String path = part.getSd().getLocation();
+        for (SentryMetastoreListenerPlugin plugin : sentryPlugins) {
+          plugin.addPath(authzObj, path);
+        }
+      }
     }
-
-    //TODO: Need more validations?
-
-    NotificationEvent event = new NotificationEvent(0L, now(), HCatConstants.HCAT_ADD_PARTITION_EVENT,
-            messageFactory.buildAddPartitionMessage(partitionEvent.getTable(), partitionEvent.getPartitions()).toString());
-
-    event.setDbName(partitionEvent.getTable().getDbName());
-    event.setTableName(partitionEvent.getTable().getTableName());
-    this.enqueue(event);
+    super.onAddPartition(partitionEvent);
   }
 
   @Override
   public void onDropPartition(DropPartitionEvent partitionEvent)
-          throws MetaException {
+      throws MetaException {
 
-    // do not write to Notification log if the operation has failed
+    // don't sync path if the operation has failed
     if (!partitionEvent.getStatus()) {
-      LOGGER.info("Skipping writing to NotificationLog as the Drop partition event failed");
+      LOGGER.debug("Skip syncing path with Sentry server for onDropPartition event," +
+        " since the operation failed. \n");
       return;
     }
 
-    NotificationEvent event = new NotificationEvent(0L, now(), HCatConstants.HCAT_DROP_PARTITION_EVENT,
-            messageFactory.buildDropPartitionMessage(partitionEvent.getTable(), partitionEvent.getPartition()).toString());
-    //TODO: Why is this asymmetric with add partitions(s)?
-    // Seems like adding multiple partitions generate a single event
-    // where as single partition drop generated an event?
-
-    event.setDbName(partitionEvent.getTable().getDbName());
-    event.setTableName(partitionEvent.getTable().getTableName());
-    this.enqueue(event);
+    String authzObj = partitionEvent.getTable().getDbName() + "."
+        + partitionEvent.getTable().getTableName();
+    String path = partitionEvent.getPartition().getSd().getLocation();
+    for (SentryMetastoreListenerPlugin plugin : sentryPlugins) {
+      plugin.removePath(authzObj, path);
+    }
+    super.onDropPartition(partitionEvent);
   }
 
-  private int now() {
-    long millis = System.currentTimeMillis();
-    millis /= 1000;
-    if (millis > Integer.MAX_VALUE) {
-      LOGGER.warn("We've passed max int value in seconds since the epoch, " +
-          "all notification times will be the same!");
-      return Integer.MAX_VALUE;
+  private SentryPolicyServiceClient getSentryServiceClient()
+      throws MetaException {
+    try {
+      return SentryServiceClientFactory.create(authzConf);
+    } catch (Exception e) {
+      throw new MetaException("Failed to connect to Sentry service "
+          + e.getMessage());
     }
-    return (int)millis;
   }
 
-  //Same as DbNotificationListener to make the transition back easy
-  private void enqueue(NotificationEvent event) {
-    if(this.rs != null) {
-      this.rs.addNotificationEvent(event);
-    } else {
-      LOGGER.warn("Dropping event " + event + " since notification is not running.");
+  private void dropSentryDbPrivileges(String dbName) throws MetaException {
+    List<Authorizable> authorizableTable = new ArrayList<Authorizable>();
+    authorizableTable.add(server);
+    authorizableTable.add(new Database(dbName));
+    try {
+      dropSentryPrivileges(authorizableTable);
+    } catch (SentryUserException e) {
+      throw new MetaException("Failed to remove Sentry policies for drop DB "
+          + dbName + " Error: " + e.getMessage());
+    } catch (IOException e) {
+      throw new MetaException("Failed to find local user " + e.getMessage());
     }
+
   }
 
-  //Same as DbNotificationListener to make the transition back easy
-  private static class CleanerThread extends Thread {
-    private RawStore rs;
-    private int ttl;
+  private void dropSentryTablePrivilege(String dbName, String tabName)
+      throws MetaException {
+    List<Authorizable> authorizableTable = new ArrayList<Authorizable>();
+    authorizableTable.add(server);
+    authorizableTable.add(new Database(dbName));
+    authorizableTable.add(new Table(tabName));
 
-    CleanerThread(HiveConf conf, RawStore rs) {
-      super("CleanerThread");
-      this.rs = rs;
-      this.setTimeToLive(conf.getTimeVar(HiveConf.ConfVars.METASTORE_EVENT_DB_LISTENER_TTL, TimeUnit.SECONDS));
-      this.setDaemon(true);
+    try {
+      dropSentryPrivileges(authorizableTable);
+    } catch (SentryUserException e) {
+      throw new MetaException(
+          "Failed to remove Sentry policies for drop table " + dbName + "."
+              + tabName + " Error: " + e.getMessage());
+    } catch (IOException e) {
+      throw new MetaException("Failed to find local user " + e.getMessage());
     }
 
-    public void run() {
-      while(true) {
-        this.rs.cleanNotificationEvents(this.ttl);
+  }
+  private void dropSentryPrivileges(
+      List<? extends Authorizable> authorizableTable)
+      throws SentryUserException, IOException, MetaException {
+    String requestorUserName = UserGroupInformation.getCurrentUser()
+        .getShortUserName();
+    SentryPolicyServiceClient sentryClient = getSentryServiceClient();
+    sentryClient.dropPrivileges(requestorUserName, authorizableTable);
+
+    // Close the connection after dropping privileges is done.
+    sentryClient.close();
+  }
 
-        try {
-          Thread.sleep(60000L);
-        } catch (InterruptedException var2) {
-          LOGGER.info("Cleaner thread sleep interupted", var2);
-        }
+  private void renameSentryTablePrivilege(String oldDbName, String oldTabName,
+      String oldPath, String newDbName, String newTabName, String newPath)
+      throws MetaException {
+    List<Authorizable> oldAuthorizableTable = new ArrayList<Authorizable>();
+    oldAuthorizableTable.add(server);
+    oldAuthorizableTable.add(new Database(oldDbName));
+    oldAuthorizableTable.add(new Table(oldTabName));
+
+    List<Authorizable> newAuthorizableTable = new ArrayList<Authorizable>();
+    newAuthorizableTable.add(server);
+    newAuthorizableTable.add(new Database(newDbName));
+    newAuthorizableTable.add(new Table(newTabName));
+
+    if (!oldTabName.equalsIgnoreCase(newTabName)
+        && syncWithPolicyStore(AuthzConfVars.AUTHZ_SYNC_ALTER_WITH_POLICY_STORE)) {
+
+      SentryPolicyServiceClient sentryClient = getSentryServiceClient();
+
+      try {
+        String requestorUserName = UserGroupInformation.getCurrentUser()
+            .getShortUserName();
+        sentryClient.renamePrivileges(requestorUserName, oldAuthorizableTable, newAuthorizableTable);
+      } catch (SentryUserException e) {
+        throw new MetaException(
+            "Failed to remove Sentry policies for rename table " + oldDbName
+            + "." + oldTabName + "to " + newDbName + "." + newTabName
+            + " Error: " + e.getMessage());
+      } catch (IOException e) {
+        throw new MetaException("Failed to find local user " + e.getMessage());
+      } finally {
+
+        // Close the connection after renaming privileges is done.
+        sentryClient.close();
       }
     }
-
-    public void setTimeToLive(long configTtl) {
-      if(configTtl > 2147483647L) {
-        this.ttl = 2147483647;
-      } else {
-        this.ttl = (int)configTtl;
-      }
-
+    // The HDFS plugin needs to know if it's a path change (set location)
+    for (SentryMetastoreListenerPlugin plugin : sentryPlugins) {
+      plugin.renameAuthzObject(oldDbName + "." + oldTabName, oldPath,
+                newDbName + "." + newTabName, newPath);
     }
   }
-  private class SentryMalformedEventException extends MetaException {
-    SentryMalformedEventException(String msg, Object event) {
-      //toString is not implemented in Event classes,
-      // hence using reflection to print the details of the Event object.
-      super(msg + "Event: " + ToStringBuilder.reflectionToString(event));
-    }
+
+  private boolean syncWithPolicyStore(AuthzConfVars syncConfVar) {
+    return Boolean.parseBoolean(authzConf.get(syncConfVar.getVar(), "true"));
   }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/sentry/blob/b34a736a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryMetastorePostEventListenerNotificationLog.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryMetastorePostEventListenerNotificationLog.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryMetastorePostEventListenerNotificationLog.java
new file mode 100644
index 0000000..58470d6
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryMetastorePostEventListenerNotificationLog.java
@@ -0,0 +1,395 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sentry.binding.metastore;
+
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.base.Strings;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.MetaStoreEventListener;
+import org.apache.hadoop.hive.metastore.RawStore;
+import org.apache.hadoop.hive.metastore.RawStoreProxy;
+import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.NotificationEvent;
+import org.apache.hadoop.hive.metastore.events.AddPartitionEvent;
+import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent;
+import org.apache.hadoop.hive.metastore.events.AlterTableEvent;
+import org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent;
+import org.apache.hadoop.hive.metastore.events.CreateTableEvent;
+import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent;
+import org.apache.hadoop.hive.metastore.events.DropPartitionEvent;
+import org.apache.hadoop.hive.metastore.events.DropTableEvent;
+import org.apache.hive.hcatalog.common.HCatConstants;
+import org.apache.sentry.binding.metastore.messaging.json.SentryJSONMessageFactory;
+import org.apache.sentry.provider.db.SentryMetastoreListenerPlugin;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.lang3.builder.ToStringBuilder;
+/*
+A HMS listener class which should ideally go into the transaction which persists the Hive metadata.
+This class writes all DDL events to the NotificationLog through rawstore.addNotificationEvent(event)
+This class is very similar to DbNotificationListener, except:
+1. It uses a custom SentryJSONMessageFactory which adds additional information to the message part of the event
+ to avoid another round trip from the clients
+2. It handles the cases where actual operation has failed, and hence skips writing to the notification log.
+3. Has additional validations to make sure event has the required information.
+
+This can be replaced with DbNotificationListener in future and sentry's message factory can be plugged in if:
+- HIVE-14011 is fixed: Make MessageFactory truly pluggable
+- 2 and 3 above are handled in DbNotificationListener
+*/
+
+public class SentryMetastorePostEventListenerNotificationLog extends MetaStoreEventListener {
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(SentryMetastoreListenerPlugin.class);
+  private RawStore rs;
+  private HiveConf hiveConf;
+  SentryJSONMessageFactory messageFactory;
+
+  private static SentryMetastorePostEventListenerNotificationLog.CleanerThread cleaner = null;
+
+  //Same as DbNotificationListener to make the transition back easy
+  private synchronized void init(HiveConf conf) {
+    try {
+      this.rs = RawStoreProxy.getProxy(conf, conf, conf.getVar(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL), 999999);
+    } catch (MetaException var3) {
+      LOGGER.error("Unable to connect to raw store, notifications will not be tracked", var3);
+      this.rs = null;
+    }
+
+    if(cleaner == null && this.rs != null) {
+      cleaner = new SentryMetastorePostEventListenerNotificationLog.CleanerThread(conf, this.rs);
+      cleaner.start();
+    }
+  }
+
+  public SentryMetastorePostEventListenerNotificationLog(Configuration config) {
+    super(config);
+    // The code in MetastoreUtils.getMetaStoreListeners() that calls this looks for a constructor
+    // with a Configuration parameter, so we have to declare config as Configuration.  But it
+    // actually passes a HiveConf, which we need.  So we'll do this ugly down cast.
+    if (!(config instanceof HiveConf)) {
+      String error = "Could not initialize Plugin - Configuration is not an instanceof HiveConf";
+      LOGGER.error(error);
+      throw new RuntimeException(error);
+    }
+    hiveConf = (HiveConf)config;
+    messageFactory = new SentryJSONMessageFactory();
+    init(hiveConf);
+  }
+
+  @Override
+  public void onCreateDatabase(CreateDatabaseEvent dbEvent)
+          throws MetaException {
+
+    // do not write to Notification log if the operation has failed
+    if (!dbEvent.getStatus()) {
+      LOGGER.info("Skipping writing to NotificationLog as the Create database event failed");
+      return;
+    }
+
+    String location = dbEvent.getDatabase().getLocationUri();
+    if (Strings.isNullOrEmpty(location)) {
+      throw new SentryMalformedEventException("CreateDatabaseEvent has invalid location", dbEvent);
+    }
+    String dbName = dbEvent.getDatabase().getName();
+    if (Strings.isNullOrEmpty(dbName)) {
+      throw new SentryMalformedEventException("CreateDatabaseEvent has invalid dbName", dbEvent);
+    }
+
+    NotificationEvent event = new NotificationEvent(0L, now(), HCatConstants.HCAT_CREATE_DATABASE_EVENT,
+            messageFactory.buildCreateDatabaseMessage(dbEvent.getDatabase()).toString());
+    event.setDbName(dbName);
+    this.enqueue(event);
+  }
+
+  @Override
+  public void onDropDatabase(DropDatabaseEvent dbEvent) throws MetaException {
+
+    // do not write to Notification log if the operation has failed
+    if (!dbEvent.getStatus()) {
+      LOGGER.info("Skipping writing to NotificationLog as the Drop database event failed");
+      return;
+    }
+
+    String dbName = dbEvent.getDatabase().getName();
+    if (dbName == null || dbName.isEmpty()) {
+      throw new SentryMalformedEventException("DropDatabaseEvent has invalid dbName", dbEvent);
+    }
+
+    NotificationEvent event = new NotificationEvent(0L, now(), HCatConstants.HCAT_DROP_DATABASE_EVENT,
+            messageFactory.buildDropDatabaseMessage(dbEvent.getDatabase()).toString());
+    event.setDbName(dbName);
+    this.enqueue(event);
+  }
+
+  @Override
+  public void onCreateTable (CreateTableEvent tableEvent) throws MetaException {
+
+    // do not write to Notification log if the operation has failed
+    if (!tableEvent.getStatus()) {
+      LOGGER.info("Skipping writing to NotificationLog as the Create table event failed");
+      return;
+    }
+
+    String dbName = tableEvent.getTable().getDbName();
+    if (dbName == null || dbName.isEmpty()) {
+      throw new SentryMalformedEventException("CreateTableEvent has invalid dbName", tableEvent);
+    }
+    String tableName = tableEvent.getTable().getTableName();
+    if (tableName == null || tableName.isEmpty()) {
+      throw new SentryMalformedEventException("CreateTableEvent has invalid tableName", tableEvent);
+    }
+    // Create table event should also contain a location.
+    // But, Create view also generates a Create table event, but it does not have a location.
+    // Create view is identified by the tableType. But turns out tableType is not set in some cases.
+    // We assume that tableType is set for all create views.
+    //TODO: Location can be null/empty, handle that in HMSFollower
+    String tableType = tableEvent.getTable().getTableType();
+    if(!(tableType != null && tableType.equals(TableType.VIRTUAL_VIEW.name()))) {
+        if (tableType == null) {
+        LOGGER.warn("TableType is null, assuming it is not TableType.VIRTUAL_VIEW: tableEvent", tableEvent);
+      }
+      String location = tableEvent.getTable().getSd().getLocation();
+      if (location == null || location.isEmpty()) {
+        throw new SentryMalformedEventException("CreateTableEvent has invalid location", tableEvent);
+      }
+    }
+    NotificationEvent event = new NotificationEvent(0L, now(), HCatConstants.HCAT_CREATE_TABLE_EVENT,
+            messageFactory.buildCreateTableMessage(tableEvent.getTable()).toString());
+    event.setDbName(dbName);
+    event.setTableName(tableName);
+    this.enqueue(event);
+  }
+
+  @Override
+  public void onDropTable(DropTableEvent tableEvent) throws MetaException {
+
+    // do not write to Notification log if the operation has failed
+    if (!tableEvent.getStatus()) {
+      LOGGER.info("Skipping writing to NotificationLog as the Drop table event failed");
+      return;
+    }
+
+    String dbName = tableEvent.getTable().getDbName();
+    if (dbName == null || dbName.isEmpty()) {
+      throw new SentryMalformedEventException("DropTableEvent has invalid dbName", tableEvent);
+    }
+    String tableName = tableEvent.getTable().getTableName();
+    if (tableName == null || tableName.isEmpty()) {
+      throw new SentryMalformedEventException("DropTableEvent has invalid tableName", tableEvent);
+    }
+
+    NotificationEvent event = new NotificationEvent(0L, now(), HCatConstants.HCAT_DROP_TABLE_EVENT,
+            messageFactory.buildDropTableMessage(tableEvent.getTable()).toString());
+    event.setDbName(dbName);
+    event.setTableName(tableName);
+    this.enqueue(event);
+  }
+
+  @Override
+  public void onAlterTable (AlterTableEvent tableEvent) throws MetaException {
+
+    // do not write to Notification log if the operation has failed
+    if (!tableEvent.getStatus()) {
+      LOGGER.info("Skipping writing to NotificationLog as the Alter table event failed");
+      return;
+    }
+
+    String dbName = tableEvent.getNewTable().getDbName();
+    if (dbName == null || dbName.isEmpty()) {
+      throw new SentryMalformedEventException("AlterTableEvent's newTable has invalid dbName", tableEvent);
+    }
+    String tableName = tableEvent.getNewTable().getTableName();
+    if (tableName == null || tableName.isEmpty()) {
+      throw new SentryMalformedEventException("AlterTableEvent's newTable has invalid tableName", tableEvent);
+    }
+    dbName = tableEvent.getOldTable().getDbName();
+    if (dbName == null || dbName.isEmpty()) {
+      throw new SentryMalformedEventException("AlterTableEvent's oldTable has invalid dbName", tableEvent);
+    }
+    tableName = tableEvent.getOldTable().getTableName();
+    if (tableName == null || tableName.isEmpty()) {
+      throw new SentryMalformedEventException("AlterTableEvent's oldTable has invalid tableName", tableEvent);
+    }
+    //Alter view also generates an alter table event, but it does not have a location
+    //TODO: Handle this case in Sentry
+    if(!tableEvent.getOldTable().getTableType().equals(TableType.VIRTUAL_VIEW.name())) {
+      String location = tableEvent.getNewTable().getSd().getLocation();
+      if (location == null || location.isEmpty()) {
+        throw new SentryMalformedEventException("AlterTableEvent's newTable has invalid location", tableEvent);
+      }
+      location = tableEvent.getOldTable().getSd().getLocation();
+      if (location == null || location.isEmpty()) {
+        throw new SentryMalformedEventException("AlterTableEvent's oldTable has invalid location", tableEvent);
+      }
+    }
+
+    NotificationEvent event = new NotificationEvent(0L, now(), HCatConstants.HCAT_ALTER_TABLE_EVENT,
+            messageFactory.buildAlterTableMessage(tableEvent.getOldTable(), tableEvent.getNewTable()).toString());
+    event.setDbName(tableEvent.getNewTable().getDbName());
+    event.setTableName(tableEvent.getNewTable().getTableName());
+    this.enqueue(event);
+  }
+
+  @Override
+  public void onAlterPartition(AlterPartitionEvent partitionEvent)
+          throws MetaException {
+
+    // do not write to Notification log if the operation has failed
+    if (!partitionEvent.getStatus()) {
+      LOGGER.info("Skipping writing to NotificationLog as the Alter partition event failed");
+      return;
+    }
+
+    String dbName = partitionEvent.getNewPartition().getDbName();
+    if (dbName == null || dbName.isEmpty()) {
+      throw new SentryMalformedEventException("AlterPartitionEvent's newPartition has invalid dbName", partitionEvent);
+    }
+    String tableName = partitionEvent.getNewPartition().getTableName();
+    if (tableName == null || tableName.isEmpty()) {
+      throw new SentryMalformedEventException("AlterPartitionEvent's newPartition has invalid tableName", partitionEvent);
+    }
+
+    //TODO: Need more validations, but it is tricky as there are many variations and validations change for each one
+    // Alter partition Location
+    // Alter partition property
+    // Any more?
+
+    NotificationEvent event = new NotificationEvent(0L, now(), HCatConstants.HCAT_ALTER_PARTITION_EVENT,
+            messageFactory.buildAlterPartitionMessage(partitionEvent.getOldPartition(), partitionEvent.getNewPartition()).toString());
+
+    event.setDbName(partitionEvent.getNewPartition().getDbName());
+    event.setTableName(partitionEvent.getNewPartition().getTableName());
+    this.enqueue(event);
+  }
+
+  @Override
+  public void onAddPartition(AddPartitionEvent partitionEvent)
+          throws MetaException {
+
+    // do not write to Notification log if the operation has failed
+    if (!partitionEvent.getStatus()) {
+      LOGGER.info("Skipping writing to NotificationLog as the Add partition event failed");
+      return;
+    }
+
+    String dbName = partitionEvent.getTable().getDbName();
+    if (dbName == null || dbName.isEmpty()) {
+      throw new SentryMalformedEventException("AddPartitionEvent has invalid dbName", partitionEvent);
+    }
+    String tableName = partitionEvent.getTable().getTableName();
+    if (tableName == null || tableName.isEmpty()) {
+      throw new SentryMalformedEventException("AddPartitionEvent's newPartition has invalid tableName", partitionEvent);
+    }
+
+    //TODO: Need more validations?
+
+    NotificationEvent event = new NotificationEvent(0L, now(), HCatConstants.HCAT_ADD_PARTITION_EVENT,
+            messageFactory.buildAddPartitionMessage(partitionEvent.getTable(), partitionEvent.getPartitions()).toString());
+
+    event.setDbName(partitionEvent.getTable().getDbName());
+    event.setTableName(partitionEvent.getTable().getTableName());
+    this.enqueue(event);
+  }
+
+  @Override
+  public void onDropPartition(DropPartitionEvent partitionEvent)
+          throws MetaException {
+
+    // do not write to Notification log if the operation has failed
+    if (!partitionEvent.getStatus()) {
+      LOGGER.info("Skipping writing to NotificationLog as the Drop partition event failed");
+      return;
+    }
+
+    NotificationEvent event = new NotificationEvent(0L, now(), HCatConstants.HCAT_DROP_PARTITION_EVENT,
+            messageFactory.buildDropPartitionMessage(partitionEvent.getTable(), partitionEvent.getPartition()).toString());
+    //TODO: Why is this asymmetric with add partitions(s)?
+    // Seems like adding multiple partitions generate a single event
+    // where as single partition drop generated an event?
+
+    event.setDbName(partitionEvent.getTable().getDbName());
+    event.setTableName(partitionEvent.getTable().getTableName());
+    this.enqueue(event);
+  }
+
+  private int now() {
+    long millis = System.currentTimeMillis();
+    millis /= 1000;
+    if (millis > Integer.MAX_VALUE) {
+      LOGGER.warn("We've passed max int value in seconds since the epoch, " +
+          "all notification times will be the same!");
+      return Integer.MAX_VALUE;
+    }
+    return (int)millis;
+  }
+
+  //Same as DbNotificationListener to make the transition back easy
+  private void enqueue(NotificationEvent event) {
+    if(this.rs != null) {
+      this.rs.addNotificationEvent(event);
+    } else {
+      LOGGER.warn("Dropping event " + event + " since notification is not running.");
+    }
+  }
+
+  //Same as DbNotificationListener to make the transition back easy
+  private static class CleanerThread extends Thread {
+    private RawStore rs;
+    private int ttl;
+
+    CleanerThread(HiveConf conf, RawStore rs) {
+      super("CleanerThread");
+      this.rs = rs;
+      this.setTimeToLive(conf.getTimeVar(HiveConf.ConfVars.METASTORE_EVENT_DB_LISTENER_TTL, TimeUnit.SECONDS));
+      this.setDaemon(true);
+    }
+
+    public void run() {
+      while(true) {
+        this.rs.cleanNotificationEvents(this.ttl);
+
+        try {
+          Thread.sleep(60000L);
+        } catch (InterruptedException var2) {
+          LOGGER.info("Cleaner thread sleep interupted", var2);
+        }
+      }
+    }
+
+    public void setTimeToLive(long configTtl) {
+      if(configTtl > 2147483647L) {
+        this.ttl = 2147483647;
+      } else {
+        this.ttl = (int)configTtl;
+      }
+
+    }
+  }
+  private class SentryMalformedEventException extends MetaException {
+    SentryMalformedEventException(String msg, Object event) {
+      //toString is not implemented in Event classes,
+      // hence using reflection to print the details of the Event object.
+      super(msg + "Event: " + ToStringBuilder.reflectionToString(event));
+    }
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/sentry/blob/b34a736a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java
index bd4ada4..a89d3f6 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java
@@ -95,6 +95,7 @@ public class SentryService implements Callable, SigUtils.SigListener {
   private SentryWebServer sentryWebServer;
   private final long maxMessageSize;
   private final LeaderStatusMonitor leaderMonitor;
+  private final boolean notificationLogEnabled;
 
   public SentryService(Configuration conf) throws Exception {
     this.conf = conf;
@@ -150,14 +151,21 @@ public class SentryService implements Callable, SigUtils.SigListener {
     });
     this.leaderMonitor = LeaderStatusMonitor.getLeaderStatusMonitor(conf);
     webServerPort = conf.getInt(ServerConfig.SENTRY_WEB_PORT, ServerConfig.SENTRY_WEB_PORT_DEFAULT);
-    try {
-      hmsFollowerExecutor = Executors.newScheduledThreadPool(1);
-      hmsFollowerExecutor.scheduleAtFixedRate(new HMSFollower(conf),
-              60000, 500, TimeUnit.MILLISECONDS);
-    } catch(Exception e) {
-      //TODO: Handle
-      LOGGER.error("Could not start HMSFollower");
+
+    notificationLogEnabled = conf.getBoolean(ServerConfig.SENTRY_NOTIFICATION_LOG_ENABLED,
+        ServerConfig.SENTRY_NOTIFICATION_LOG_ENABLED_DEFAULT);
+
+    if (notificationLogEnabled) {
+      try {
+        hmsFollowerExecutor = Executors.newScheduledThreadPool(1);
+        hmsFollowerExecutor.scheduleAtFixedRate(new HMSFollower(conf),
+                60000, 500, TimeUnit.MILLISECONDS);
+      } catch (Exception e) {
+        //TODO: Handle
+        LOGGER.error("Could not start HMSFollower");
+      }
     }
+
     status = Status.NOT_STARTED;
 
     // Enable signal handler for HA leader/follower status if configured

http://git-wip-us.apache.org/repos/asf/sentry/blob/b34a736a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java
index 139d038..866ebc6 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java
@@ -165,6 +165,8 @@ public class ServiceConstants {
             .put("javax.jdo.option.Multithreaded", "true")
             .build();
 
+    public static final String SENTRY_NOTIFICATION_LOG_ENABLED = "sentry.otification.log.enabled";
+    public static final boolean SENTRY_NOTIFICATION_LOG_ENABLED_DEFAULT = false;
     public static final String SENTRY_WEB_ENABLE = "sentry.service.web.enable";
     public static final Boolean SENTRY_WEB_ENABLE_DEFAULT = false;
     public static final String SENTRY_WEB_PORT = "sentry.service.web.port";

http://git-wip-us.apache.org/repos/asf/sentry/blob/b34a736a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java
index 61b24fa..47be188 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java
@@ -54,6 +54,7 @@ import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.sentry.binding.hive.SentryHiveAuthorizationTaskFactoryImpl;
+import org.apache.sentry.binding.metastore.SentryMetastorePostEventListenerNotificationLog;
 import org.apache.sentry.binding.metastore.SentryMetastorePostEventListener;
 import org.apache.sentry.core.model.db.DBModelAction;
 import org.apache.sentry.core.model.db.DBModelAuthorizable;
@@ -156,6 +157,7 @@ public abstract class AbstractTestWithStaticConfiguration {
   private static final String EXTERNAL_SENTRY_SERVICE = "sentry.e2etest.external.sentry";
   protected static final String EXTERNAL_HIVE_LIB = "sentry.e2etest.hive.lib";
   private static final String ENABLE_SENTRY_HA = "sentry.e2etest.enable.service.ha";
+  private static final String ENABLE_NOTIFICATION_LOG = "sentry.e2etest.enable.notification.log";
 
   protected static boolean policyOnHdfs = false;
   protected static boolean useSentryService = false;
@@ -179,6 +181,7 @@ public abstract class AbstractTestWithStaticConfiguration {
   protected static SentrySrv sentryServer;
   protected static Configuration sentryConf;
   protected static boolean enableSentryHA = false;
+  protected static boolean enableNotificationLog = false;
   protected static Context context;
   protected final String semanticException = "SemanticException No valid privileges";
 
@@ -312,6 +315,10 @@ public abstract class AbstractTestWithStaticConfiguration {
       enableSentryHA = true;
     }
 
+    if ("true".equalsIgnoreCase(System.getProperty(ENABLE_NOTIFICATION_LOG, "false"))) {
+      enableNotificationLog = true;
+    }
+
     if (enableHiveConcurrency) {
       properties.put(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "true");
       properties.put(HiveConf.ConfVars.HIVE_TXN_MANAGER.varname,
@@ -514,8 +521,13 @@ public abstract class AbstractTestWithStaticConfiguration {
         properties.put(HiveConf.ConfVars.METASTORE_EVENT_LISTENERS.varname,
                 "org.apache.hive.hcatalog.listener.DbNotificationListener");
       } else {
-        properties.put(HiveConf.ConfVars.METASTORE_EVENT_LISTENERS.varname,
-                SentryMetastorePostEventListener.class.getName());
+        if (enableNotificationLog) {
+          properties.put(HiveConf.ConfVars.METASTORE_EVENT_LISTENERS.varname,
+              SentryMetastorePostEventListenerNotificationLog.class.getName());
+        } else {
+          properties.put(HiveConf.ConfVars.METASTORE_EVENT_LISTENERS.varname,
+              SentryMetastorePostEventListener.class.getName());
+        }
         properties.put("hcatalog.message.factory.impl.json",
             "org.apache.sentry.binding.metastore.messaging.json.SentryJSONMessageFactory");
       }

http://git-wip-us.apache.org/repos/asf/sentry/blob/b34a736a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerInBuiltDeserializer.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerInBuiltDeserializer.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerInBuiltDeserializer.java
index c4be62d..6ab3391 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerInBuiltDeserializer.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerInBuiltDeserializer.java
@@ -31,6 +31,7 @@ public class TestSentryListenerInBuiltDeserializer extends TestDBNotificationLis
   public static void setupTestStaticConfiguration() throws Exception {
     setMetastoreListener = true;
     useDbNotificationListener = false;
+    enableNotificationLog = true;
     beforeClass();
   }
 }

http://git-wip-us.apache.org/repos/asf/sentry/blob/b34a736a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerSentryDeserializer.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerSentryDeserializer.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerSentryDeserializer.java
index 6f1886f..86a8964 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerSentryDeserializer.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerSentryDeserializer.java
@@ -53,6 +53,7 @@ public class TestSentryListenerSentryDeserializer extends AbstractMetastoreTestW
   public static void setupTestStaticConfiguration() throws Exception {
     setMetastoreListener = true;
     useDbNotificationListener = false;
+    enableNotificationLog = true;
     AbstractMetastoreTestWithStaticConfiguration.setupTestStaticConfiguration();
     setupClass();
   }


Mime
View raw message