sentry-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ak...@apache.org
Subject [21/50] [abbrv] sentry git commit: SENTRY-1769 Refactor HMSFollower Class (Kalyan Kumar Kalvagadda reviewed by Vamsee Yarlagadda, Na Li, Sergio Pena and Alexander Kolbasov)
Date Fri, 28 Jul 2017 17:06:39 GMT
http://git-wip-us.apache.org/repos/asf/sentry/blob/8b897d8e/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestNotificationProcessor.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestNotificationProcessor.java
b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestNotificationProcessor.java
new file mode 100644
index 0000000..c6c9448
--- /dev/null
+++ b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestNotificationProcessor.java
@@ -0,0 +1,465 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sentry.service.thrift;
+
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.NotificationEvent;
+import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hive.hcatalog.messaging.HCatEventMessage;
+import org.apache.sentry.binding.metastore.messaging.json.SentryJSONMessageFactory;
+import org.apache.sentry.hdfs.Updateable;
+import org.apache.sentry.provider.db.service.persistent.SentryStore;
+import org.apache.sentry.provider.db.service.thrift.TSentryAuthorizable;
+import org.junit.After;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+// TODO 1. More tests should be added here.
+// TODO 2. Tests using actual sentry store where.
+@SuppressWarnings("unused")
+public class TestNotificationProcessor {
+
+  private static final SentryStore sentryStore = Mockito.mock(SentryStore.class);
+  private final static String hiveInstance = "server2";
+  private final static Configuration conf = new Configuration();
+  private final SentryJSONMessageFactory messageFactory = new SentryJSONMessageFactory();
+  private NotificationProcessor notificationProcessor;
+
+  @BeforeClass
+  public static void setup() {
+    conf.set("sentry.hive.sync.create", "true");
+    conf.set("sentry.hive.sync.drop", "true");
+  }
+
+  @After
+  public void resetConf() {
+    conf.set("sentry.hive.sync.create", "true");
+    conf.set("sentry.hive.sync.drop", "true");
+    reset(sentryStore);
+  }
+
+  @Test
+  /*
+    Makes sure that appropriate sentry store methods are invoked when create database event
is
+    processed.
+
+    Also, checks the hive sync configuration.
+   */
+  public void testCreateDatabase() throws Exception {
+    long seqNum = 1;
+    String dbName = "db1";
+    String uriPrefix = "hdfs:///";
+    String location = "user/hive/warehouse";
+    NotificationEvent notificationEvent;
+    TSentryAuthorizable authorizable;
+    notificationProcessor = new NotificationProcessor(sentryStore,
+        hiveInstance, conf);
+
+    // Create notification event
+    notificationEvent = new NotificationEvent(seqNum, 0,
+        HCatEventMessage.EventType.CREATE_DATABASE.toString(),
+        messageFactory.buildCreateDatabaseMessage(new Database(dbName,
+            null, uriPrefix + location, null)).toString());
+
+    notificationProcessor.processNotificationEvent(notificationEvent);
+
+    authorizable = new TSentryAuthorizable(hiveInstance);
+    authorizable.setServer(hiveInstance);
+    authorizable.setDb("db1");
+    //noinspection unchecked
+    verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(),
+        Mockito.anyCollection(), Mockito.any(Updateable.Update.class));
+
+    verify(sentryStore, times(1)).dropPrivilege(authorizable,
+        NotificationProcessor.getPermUpdatableOnDrop(authorizable));
+    reset(sentryStore);
+
+    //Change the configuration and make sure that exiting privileges are not dropped
+    notificationProcessor.setSyncStoreOnCreate(false);
+    dbName = "db2";
+    notificationEvent = new NotificationEvent(1, 0,
+        HCatEventMessage.EventType.CREATE_DATABASE.toString(),
+        messageFactory.buildCreateDatabaseMessage(new Database(dbName,
+            null, "hdfs:///db2", null)).toString());
+
+    notificationProcessor.processNotificationEvent(notificationEvent);
+
+    authorizable = new TSentryAuthorizable(hiveInstance);
+    authorizable.setServer(hiveInstance);
+    authorizable.setDb(dbName);
+
+    //noinspection unchecked
+    verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(),
+        Mockito.anyCollection(), Mockito.any(Updateable.Update.class));
+    //making sure that privileges are not dropped
+    verify(sentryStore, times(0)).dropPrivilege(authorizable,
+        NotificationProcessor.getPermUpdatableOnDrop(authorizable));
+
+  }
+
+  @Test
+  /*
+    Makes sure that appropriate sentry store methods are invoked when drop database event
is
+    processed.
+
+    Also, checks the hive sync configuration.
+   */
+  public void testDropDatabase() throws Exception {
+    String dbName = "db1";
+
+    notificationProcessor = new NotificationProcessor(sentryStore,
+        hiveInstance, conf);
+
+    // Create notification event
+    NotificationEvent notificationEvent = new NotificationEvent(1, 0,
+        HCatEventMessage.EventType.DROP_DATABASE.toString(),
+        messageFactory.buildDropDatabaseMessage(new Database(dbName, null,
+            "hdfs:///db1", null)).toString());
+
+    notificationProcessor.processNotificationEvent(notificationEvent);
+
+    TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance);
+    authorizable.setServer(hiveInstance);
+    authorizable.setDb("db1");
+
+    //noinspection unchecked
+    verify(sentryStore, times(1)).deleteAuthzPathsMapping(Mockito.anyString(),
+        Mockito.anyCollection(), Mockito.any(Updateable.Update.class));
+    verify(sentryStore, times(1)).dropPrivilege(authorizable,
+        NotificationProcessor.getPermUpdatableOnDrop(authorizable));
+    reset(sentryStore);
+
+    // Change the configuration and make sure that exiting privileges are not dropped
+    notificationProcessor.setSyncStoreOnDrop(false);
+    dbName = "db2";
+    // Create notification event
+    notificationEvent = new NotificationEvent(1, 0,
+        HCatEventMessage.EventType.DROP_DATABASE.toString(),
+        messageFactory.buildDropDatabaseMessage(new Database(dbName, null,
+            "hdfs:///db2", null)).toString());
+
+    notificationProcessor.processNotificationEvent(notificationEvent);
+
+    authorizable = new TSentryAuthorizable(hiveInstance);
+    authorizable.setServer(hiveInstance);
+    authorizable.setDb(dbName);
+
+    //noinspection unchecked
+    verify(sentryStore, times(1)).deleteAuthzPathsMapping(Mockito.anyString(),
+        Mockito.anyCollection(), Mockito.any(Updateable.Update.class));
+    verify(sentryStore, times(0)).dropPrivilege(authorizable,
+        NotificationProcessor.getPermUpdatableOnDrop(authorizable));
+  }
+
+  @Test
+  /*
+    Makes sure that appropriate sentry store methods are invoked when create table event
is
+    processed.
+
+    Also, checks the hive sync configuration.
+   */
+  public void testCreateTable() throws Exception {
+    String dbName = "db1";
+    String tableName = "table1";
+
+    notificationProcessor = new NotificationProcessor(sentryStore,
+        hiveInstance, conf);
+
+    // Create notification event
+    StorageDescriptor sd = new StorageDescriptor();
+    sd.setLocation("hdfs:///db1.db/table1");
+    NotificationEvent notificationEvent =
+        new NotificationEvent(1, 0, HCatEventMessage.EventType.CREATE_TABLE.toString(),
+            messageFactory.buildCreateTableMessage(new Table(tableName,
+                dbName, null, 0, 0, 0, sd, null, null, null, null, null)).toString());
+
+    notificationProcessor.processNotificationEvent(notificationEvent);
+
+    TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance);
+    authorizable.setServer(hiveInstance);
+    authorizable.setDb("db1");
+    authorizable.setTable(tableName);
+
+    //noinspection unchecked
+    verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(),
+        Mockito.anyCollection(), Mockito.any(Updateable.Update.class));
+
+    verify(sentryStore, times(1)).dropPrivilege(authorizable,
+        NotificationProcessor.getPermUpdatableOnDrop(authorizable));
+    reset(sentryStore);
+
+    // Change the configuration and make sure that existing privileges are not dropped
+    notificationProcessor.setSyncStoreOnCreate(false);
+
+    // Create notification event
+    dbName = "db2";
+    tableName = "table2";
+    sd = new StorageDescriptor();
+    sd.setLocation("hdfs:///db1.db/table2");
+    notificationEvent =
+        new NotificationEvent(1, 0, HCatEventMessage.EventType.CREATE_TABLE.toString(),
+            messageFactory.buildCreateTableMessage(new Table(tableName,
+                dbName, null, 0, 0, 0, sd, null, null, null, null, null)).toString());
+
+    notificationProcessor.processNotificationEvent(notificationEvent);
+
+    authorizable = new TSentryAuthorizable(hiveInstance);
+    authorizable.setServer(hiveInstance);
+    authorizable.setDb(dbName);
+    authorizable.setTable(tableName);
+
+    //noinspection unchecked
+    verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(),
+        Mockito.anyCollection(), Mockito.any(Updateable.Update.class));
+    // Making sure that privileges are not dropped
+    verify(sentryStore, times(0)).dropPrivilege(authorizable,
+        NotificationProcessor.getPermUpdatableOnDrop(authorizable));
+  }
+
+  @Test
+  /*
+    Makes sure that appropriate sentry store methods are invoked when drop table event is
+    processed.
+
+    Also, checks the hive sync configuration.
+   */
+  public void testDropTable() throws Exception {
+    String dbName = "db1";
+    String tableName = "table1";
+
+    Configuration authConf = new Configuration();
+    notificationProcessor = new NotificationProcessor(sentryStore,
+        hiveInstance, authConf);
+
+    // Create notification event
+    StorageDescriptor sd = new StorageDescriptor();
+    sd.setLocation("hdfs:///db1.db/table1");
+    NotificationEvent notificationEvent = new NotificationEvent(1, 0,
+        HCatEventMessage.EventType.DROP_TABLE.toString(),
+        messageFactory.buildDropTableMessage(new Table(tableName,
+            dbName, null, 0, 0, 0, sd, null, null, null, null, null)).toString());
+
+    notificationProcessor.processNotificationEvent(notificationEvent);
+
+    TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance);
+    authorizable.setServer(hiveInstance);
+    authorizable.setDb("db1");
+    authorizable.setTable(tableName);
+
+    verify(sentryStore, times(1)).deleteAllAuthzPathsMapping(Mockito.anyString(),
+        Mockito.any(Updateable.Update.class));
+
+    verify(sentryStore, times(1)).dropPrivilege(authorizable,
+        NotificationProcessor.getPermUpdatableOnDrop(authorizable));
+  }
+
+  @Test
+  /*
+    Makes sure that appropriate sentry store methods are invoked when alter tables event
is
+    processed.
+   */
+  public void testAlterTable() throws Exception {
+    String dbName = "db1";
+    String tableName = "table1";
+
+    String newDbName = "db1";
+    String newTableName = "table2";
+
+    Configuration authConf = new Configuration();
+    notificationProcessor = new NotificationProcessor(sentryStore,
+        hiveInstance, authConf);
+
+    // Create notification event
+    StorageDescriptor sd = new StorageDescriptor();
+    sd.setLocation("hdfs:///db1.db/table1");
+    NotificationEvent notificationEvent = new NotificationEvent(1, 0,
+        HCatEventMessage.EventType.ALTER_TABLE.toString(),
+        messageFactory.buildAlterTableMessage(
+            new Table(tableName, dbName, null, 0, 0, 0, sd, null, null, null, null, null),
+            new Table(newTableName, newDbName, null, 0, 0, 0, sd, null, null, null, null,
null))
+            .toString());
+    notificationEvent.setDbName(newDbName);
+    notificationEvent.setTableName(newTableName);
+
+    notificationProcessor.processNotificationEvent(notificationEvent);
+
+    TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance);
+    authorizable.setServer(hiveInstance);
+    authorizable.setDb(dbName);
+    authorizable.setTable(tableName);
+
+    TSentryAuthorizable newAuthorizable = new TSentryAuthorizable(hiveInstance);
+    authorizable.setServer(hiveInstance);
+    newAuthorizable.setDb(newDbName);
+    newAuthorizable.setTable(newTableName);
+
+    verify(sentryStore, times(1)).renameAuthzObj(Mockito.anyString(), Mockito.anyString(),
+        Mockito.any(Updateable.Update.class));
+
+    verify(sentryStore, times(1)).renamePrivilege(authorizable, newAuthorizable,
+        NotificationProcessor.getPermUpdatableOnRename(authorizable, newAuthorizable));
+  }
+
+  @Test
+  /*
+    Makes sure that appropriate sentry store methods are invoked when alter tables event
is
+    processed.
+   */
+  public void testRenameTableWithLocationUpdate() throws Exception {
+    String dbName = "db1";
+    String tableName = "table1";
+
+    String newDbName = "db1";
+    String newTableName = "table2";
+
+    Configuration authConf = new Configuration();
+    notificationProcessor = new NotificationProcessor(sentryStore,
+        hiveInstance, authConf);
+
+    // Create notification event
+    StorageDescriptor sd = new StorageDescriptor();
+    sd.setLocation("hdfs:///db1.db/table1");
+    StorageDescriptor new_sd = new StorageDescriptor();
+    new_sd.setLocation("hdfs:///db1.db/table2");
+    NotificationEvent notificationEvent = new NotificationEvent(1, 0,
+        HCatEventMessage.EventType.ALTER_TABLE.toString(),
+        messageFactory.buildAlterTableMessage(
+            new Table(tableName, dbName, null, 0, 0, 0, sd, null, null, null, null, null),
+            new Table(newTableName, newDbName, null, 0, 0, 0, new_sd, null, null, null, null,
null))
+            .toString());
+    notificationEvent.setDbName(newDbName);
+    notificationEvent.setTableName(newTableName);
+
+    notificationProcessor.processNotificationEvent(notificationEvent);
+
+    TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance);
+    authorizable.setServer(hiveInstance);
+    authorizable.setDb(dbName);
+    authorizable.setTable(tableName);
+
+    TSentryAuthorizable newAuthorizable = new TSentryAuthorizable(hiveInstance);
+    authorizable.setServer(hiveInstance);
+    newAuthorizable.setDb(newDbName);
+    newAuthorizable.setTable(newTableName);
+
+    verify(sentryStore, times(1)).renameAuthzPathsMapping(Mockito.anyString(), Mockito.anyString(),
+        Mockito.anyString(), Mockito.anyString(), Mockito.any(Updateable.Update.class));
+
+    verify(sentryStore, times(1)).renamePrivilege(authorizable, newAuthorizable,
+        NotificationProcessor.getPermUpdatableOnRename(authorizable, newAuthorizable));
+  }
+
+  @Test
+  /*
+    Test to made sure that sentry store is not invoked when invalid alter table event is
+    processed.
+   */
+  public void testAlterTableWithInvalidEvent() throws Exception {
+    String dbName = "db1";
+    String tableName1 = "table1";
+    String tableName2 = "table2";
+    long inputEventId = 1;
+    NotificationEvent notificationEvent;
+    List<FieldSchema> partCols;
+    StorageDescriptor sd;
+    Mockito.doNothing().when(sentryStore).persistLastProcessedNotificationID(Mockito.anyLong());
+    //noinspection unchecked
+    Mockito.doNothing().when(sentryStore).addAuthzPathsMapping(Mockito.anyString(),
+        Mockito.anyCollection(), Mockito.any(Updateable.Update.class));
+
+    Configuration authConf = new Configuration();
+    notificationProcessor = new NotificationProcessor(sentryStore,
+        hiveInstance, authConf);
+
+    // Create a table
+    sd = new StorageDescriptor();
+    sd.setLocation("hdfs://db1.db/table1");
+    partCols = new ArrayList<>();
+    partCols.add(new FieldSchema("ds", "string", ""));
+    Table table = new Table(tableName1, dbName, null, 0, 0, 0, sd, partCols,
+        null, null, null, null);
+    notificationEvent = new NotificationEvent(inputEventId, 0,
+        HCatEventMessage.EventType.CREATE_TABLE.toString(),
+        messageFactory.buildCreateTableMessage(table).toString());
+    notificationEvent.setDbName(dbName);
+    notificationEvent.setTableName(tableName1);
+    inputEventId += 1;
+    // Process the notification
+    notificationProcessor.processNotificationEvent(notificationEvent);
+    // Make sure that addAuthzPathsMapping was invoked once to handle CREATE_TABLE notification
+    // and persistLastProcessedNotificationID was not invoked.
+    //noinspection unchecked
+    verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(),
+        Mockito.anyCollection(), Mockito.any(Updateable.Update.class));
+    reset(sentryStore);
+
+    // Create alter table notification with out actually changing anything.
+    // This notification should not be processed by sentry server
+    // Notification should be persisted explicitly
+    notificationEvent = new NotificationEvent(1, 0,
+        HCatEventMessage.EventType.ALTER_TABLE.toString(),
+        messageFactory.buildAlterTableMessage(
+            new Table(tableName1, dbName, null, 0, 0, 0, sd, null, null, null, null, null),
+            new Table(tableName1, dbName, null, 0, 0, 0, sd, null,
+                null, null, null, null)).toString());
+    notificationEvent.setDbName(dbName);
+    notificationEvent.setTableName(tableName1);
+    inputEventId += 1;
+    // Process the notification
+    notificationProcessor.processNotificationEvent(notificationEvent);
+    // Make sure that renameAuthzObj and deleteAuthzPathsMapping were  not invoked
+    // to handle CREATE_TABLE notification
+    // and persistLastProcessedNotificationID is explicitly invoked
+    verify(sentryStore, times(0)).renameAuthzObj(Mockito.anyString(), Mockito.anyString(),
+        Mockito.any(Updateable.Update.class));
+    //noinspection unchecked
+    verify(sentryStore, times(0)).deleteAuthzPathsMapping(Mockito.anyString(),
+        Mockito.anyCollection(), Mockito.any(Updateable.Update.class));
+    reset(sentryStore);
+
+    // Create a table
+    sd = new StorageDescriptor();
+    sd.setLocation("hdfs://db1.db/table2");
+    partCols = new ArrayList<>();
+    partCols.add(new FieldSchema("ds", "string", ""));
+    Table table1 = new Table(tableName2, dbName, null, 0, 0, 0, sd,
+        partCols, null, null, null, null);
+    notificationEvent = new NotificationEvent(inputEventId, 0,
+        HCatEventMessage.EventType.CREATE_TABLE.toString(),
+        messageFactory.buildCreateTableMessage(table1).toString());
+    notificationEvent.setDbName(dbName);
+    notificationEvent.setTableName(tableName2);
+    // Process the notification
+    notificationProcessor.processNotificationEvent(notificationEvent);
+    // Make sure that addAuthzPathsMapping was invoked once to handle CREATE_TABLE notification
+    // and persistLastProcessedNotificationID was not invoked.
+    //noinspection unchecked
+    verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(),
+        Mockito.anyCollection(), Mockito.any(Updateable.Update.class));
+  }
+}

http://git-wip-us.apache.org/repos/asf/sentry/blob/8b897d8e/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestSentryHmsClient.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestSentryHmsClient.java
b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestSentryHmsClient.java
new file mode 100644
index 0000000..3cc6541
--- /dev/null
+++ b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestSentryHmsClient.java
@@ -0,0 +1,470 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.service.thrift;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Random;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.IMetaStoreClient.NotificationFilter;
+import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.NotificationEvent;
+import org.apache.hadoop.hive.metastore.api.NotificationEventResponse;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hive.hcatalog.messaging.HCatEventMessage;
+import org.apache.sentry.binding.metastore.messaging.json.SentryJSONMessageFactory;
+import org.apache.sentry.provider.db.service.persistent.PathsImage;
+import org.apache.thrift.TException;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+import javax.security.auth.login.LoginException;
+
+/**
+ * Test mocks HiveMetaStoreClient class and tests SentryHmsClient.
+ */
+public class TestSentryHmsClient {
+
+  private static final Configuration conf = new Configuration();
+  private static final SentryJSONMessageFactory messageFactory = new SentryJSONMessageFactory();
+  private static SentryHmsClient client;
+  private static MockHMSClientFactory hiveConnectionFactory;
+
+  /**
+   * Create mock database with the given name
+   *
+   * @param name Database name
+   * @return Mock database object
+   */
+  private static Database makeDb(String name) {
+    Database db = Mockito.mock(Database.class);
+    Mockito.when(db.getName()).thenReturn(name);
+    Mockito.when(db.getLocationUri()).thenReturn("hdfs:///" + name);
+    return db;
+  }
+
+  /**
+   * Create mock table
+   *
+   * @param dbName db for this table
+   * @param tableName name of the table
+   * @return mock table object
+   */
+  private static Table makeTable(String dbName, String tableName) {
+    Table table = Mockito.mock(Table.class);
+    Mockito.when(table.getDbName()).thenReturn(dbName);
+    Mockito.when(table.getTableName()).thenReturn(tableName);
+    StorageDescriptor sd = Mockito.mock(StorageDescriptor.class);
+    Mockito.when(sd.getLocation()).thenReturn(
+        String.format("hdfs:///%s/%s", dbName, tableName));
+    Mockito.when(table.getSd()).thenReturn(sd);
+    return table;
+  }
+
+  /**
+   * Create mock partition
+   *
+   * @param dbName database for this partition
+   * @param tableName table for this partition
+   * @param partName partition name
+   * @return mock partition object
+   */
+  private static Partition makePartition(String dbName, String tableName, String partName)
{
+    Partition partition = Mockito.mock(Partition.class);
+    StorageDescriptor sd = Mockito.mock(StorageDescriptor.class);
+    Mockito.when(sd.getLocation()).thenReturn(
+        String.format("hdfs:///%s/%s/%s", dbName, tableName, partName));
+    Mockito.when(partition.getSd()).thenReturn(sd);
+    return partition;
+  }
+
+  /**
+   * Creates create database notification
+   *
+   * @return NotificationEvent
+   */
+  private static NotificationEvent getCreateDatabaseNotification(long id) {
+    Random rand = new Random();
+    int n = rand.nextInt(100) + 1;
+    String dbName = "db" + n;
+    return new NotificationEvent(id, 0, HCatEventMessage.EventType.CREATE_DATABASE.toString(),
+        messageFactory
+            .buildCreateDatabaseMessage(new Database(dbName, null, "hdfs:///" + dbName, null))
+            .toString());
+  }
+
+  /**
+   * Creates drop database notification
+   *
+   * @return NotificationEvent
+   */
+  private static NotificationEvent getDropDatabaseNotification(long id) {
+    Random rand = new Random();
+    int n = rand.nextInt(100) + 1;
+    String dbName = "db" + n;
+    return new NotificationEvent(id, 0, HCatEventMessage.EventType.DROP_DATABASE.toString(),
+        messageFactory
+            .buildDropDatabaseMessage(new Database(dbName, null, "hdfs:///" + dbName, null))
+            .toString());
+  }
+
+  @BeforeClass
+  static public void initialize() throws IOException, LoginException {
+    hiveConnectionFactory = new MockHMSClientFactory();
+    client = new SentryHmsClient(conf, (HiveConnectionFactory)hiveConnectionFactory);
+  }
+
+  /**
+   * Creating snapshot when SentryHmsClient is not connected to HMS
+   */
+  @Test
+  public void testSnapshotCreationWithOutClientConnected() throws Exception {
+    // Make sure that client is not connected
+    Assert.assertFalse(client.isConnected());
+    PathsImage snapshotInfo = client.getFullSnapshot();
+    Assert.assertTrue(snapshotInfo.getPathImage().isEmpty());
+  }
+
+  /**
+   * Creating snapshot when HMS doesn't have any data
+   */
+  @Test
+  public void testSnapshotCreationWithNoHmsData() throws Exception {
+    MockClient mockClient = new MockClient(new HiveSnapshot(), 1);
+    client.setClient(mockClient.client);
+    // Make sure that client is connected
+    Assert.assertTrue(client.isConnected());
+    PathsImage snapshotInfo = client.getFullSnapshot();
+    Assert.assertTrue(snapshotInfo.getPathImage().isEmpty());
+  }
+
+  /**
+   * Creating a snapshot when there is data but there are updates to HMS data mean while
+   */
+  @Test
+  public void testSnapshotCreationWhenDataIsActivelyUpdated() throws Exception {
+    HiveTable tab21 = new HiveTable("tab21");
+    HiveTable tab31 = new HiveTable("tab31").add("part311").add("part312");
+    HiveDb db3 = new HiveDb("db3", Lists.newArrayList(tab31));
+    HiveDb db2 = new HiveDb("db2", Lists.newArrayList(tab21));
+    HiveDb db1 = new HiveDb("db1");
+    HiveSnapshot snap = new HiveSnapshot().add(db1).add(db2).add(db3);
+    final MockClient mockClient = new MockClient(snap, 1);
+
+    client.setClient(mockClient.client);
+    hiveConnectionFactory.setClient(mockClient);
+    // Make sure that client is connected
+    Assert.assertTrue(client.isConnected());
+    PathsImage snapshotInfo = client.getFullSnapshot();
+    // Make sure that snapshot is not empty
+    Assert.assertTrue(!snapshotInfo.getPathImage().isEmpty());
+
+    Mockito.when(mockClient.client.getCurrentNotificationEventId()).
+        thenAnswer(new Answer<CurrentNotificationEventId>() {
+          @Override
+          public CurrentNotificationEventId answer(InvocationOnMock invocation)
+              throws Throwable {
+            return new CurrentNotificationEventId(mockClient.incrementNotificationEventId());
+          }
+
+        });
+
+    snapshotInfo = client.getFullSnapshot();
+    Assert.assertTrue(snapshotInfo.getPathImage().isEmpty());
+  }
+
+  /**
+   * Creating a snapshot when there is data in HMS.
+   */
+  @Test
+  public void testSnapshotCreationSuccess() throws Exception {
+    HiveTable tab21 = new HiveTable("tab21");
+    HiveTable tab31 = new HiveTable("tab31");
+    HiveDb db3 = new HiveDb("db3", Lists.newArrayList(tab31));
+    HiveDb db2 = new HiveDb("db2", Lists.newArrayList(tab21));
+    HiveDb db1 = new HiveDb("db1");
+    HiveSnapshot snap = new HiveSnapshot().add(db1).add(db2).add(db3);
+    MockClient mockClient = new MockClient(snap, 1);
+    Mockito.when(mockClient.client.getCurrentNotificationEventId()).
+        thenReturn(new CurrentNotificationEventId(mockClient.eventId));
+    client.setClient(mockClient.client);
+    hiveConnectionFactory.setClient(mockClient);
+    // Make sure that client is connected
+    Assert.assertTrue(client.isConnected());
+
+    PathsImage snapshotInfo = client.getFullSnapshot();
+    Assert.assertEquals(5, snapshotInfo.getPathImage().size());
+    Assert.assertEquals(Sets.newHashSet("db1"), snapshotInfo.getPathImage().get("db1"));
+    Assert.assertEquals(Sets.newHashSet("db2"), snapshotInfo.getPathImage().get("db2"));
+    Assert.assertEquals(Sets.newHashSet("db3"), snapshotInfo.getPathImage().get("db3"));
+    Assert.assertEquals(Sets.newHashSet("db2/tab21"),
+        snapshotInfo.getPathImage().get("db2.tab21"));
+    Assert.assertEquals(Sets.newHashSet("db3/tab31"), snapshotInfo.getPathImage().get("db3.tab31"));
+    Assert.assertEquals(snapshotInfo.getId(), mockClient.eventId);
+
+  }
+
+  /**
+   * Test scenario when there is no HMS connection
+   * Getting new notifications
+   */
+  @Test
+  public void testGetNewNotificationsWithOutClientConnected() throws Exception {
+    HiveTable tab21 = new HiveTable("tab21");
+    HiveTable tab31 = new HiveTable("tab31");
+    HiveDb db3 = new HiveDb("db3", Lists.newArrayList(tab31));
+    HiveDb db2 = new HiveDb("db2", Lists.newArrayList(tab21));
+    HiveDb db1 = new HiveDb("db1");
+    client.setClient(null);
+    HiveSnapshot snap = new HiveSnapshot().add(db1).add(db2).add(db3);
+    MockClient mockClient = new MockClient(snap, 100);
+    Mockito.when(mockClient.client.getCurrentNotificationEventId()).
+        thenReturn(new CurrentNotificationEventId(mockClient.eventId));
+    // Make sure that client is not connected
+    Assert.assertTrue(!client.isConnected());
+    Collection<NotificationEvent> events = client.getNotifications(100);
+    Assert.assertTrue(events.isEmpty());
+
+  }
+
+  /**
+   * Test scenario where there are no notifications
+   * Getting new notifications
+   */
+  @Test
+  public void testGetNewNotificationsWithNoHmsUpdates() throws Exception {
+    HiveTable tab21 = new HiveTable("tab21");
+    HiveTable tab31 = new HiveTable("tab31");
+    HiveDb db3 = new HiveDb("db3", Lists.newArrayList(tab31));
+    HiveDb db2 = new HiveDb("db2", Lists.newArrayList(tab21));
+    HiveDb db1 = new HiveDb("db1");
+    HiveSnapshot snap = new HiveSnapshot().add(db1).add(db2).add(db3);
+    MockClient mockClient = new MockClient(snap, 100);
+    Mockito.when(mockClient.client.getCurrentNotificationEventId()).
+        thenReturn(new CurrentNotificationEventId(mockClient.eventId));
+    client.setClient(mockClient.client);
+    hiveConnectionFactory.setClient(mockClient);
+    // Make sure that client is connected
+    Assert.assertTrue(client.isConnected());
+    Collection<NotificationEvent> events = client.getNotifications(100);
+    Assert.assertTrue(events.isEmpty());
+  }
+
+  /**
+   * Test scenario where there are notifications
+   * Getting new notifications
+   */
+  @Test
+  public void testGetNewNotificationsSuccess() throws Exception {
+    final MockClient mockClient = new MockClient(new HiveSnapshot(), 100);
+    client.setClient(mockClient.client);
+    hiveConnectionFactory.setClient(mockClient);
+    // Make sure that client is connected
+    Assert.assertTrue(client.isConnected());
+
+    Mockito.when(mockClient.client.getCurrentNotificationEventId()).
+        thenAnswer(new Answer<CurrentNotificationEventId>() {
+          @Override
+          public CurrentNotificationEventId answer(InvocationOnMock invocation)
+              throws Throwable {
+            return new CurrentNotificationEventId(mockClient.incrementNotificationEventId());
+          }
+        });
+    Mockito.when(mockClient.client.getNextNotification(Mockito.anyLong(), Mockito.anyInt(),
+        Mockito.any(NotificationFilter.class))).
+        thenAnswer(new Answer<NotificationEventResponse>() {
+          @Override
+          public NotificationEventResponse answer(InvocationOnMock invocation)
+              throws Throwable {
+            long id = 1;
+            List<NotificationEvent> events = new ArrayList<>();
+            events.add(getCreateDatabaseNotification(id++));
+            events.add(getDropDatabaseNotification(id++));
+            return new NotificationEventResponse(events);
+          }
+        });
+
+    Collection<NotificationEvent> events = client.getNotifications(100);
+    long id = 1;
+    for (NotificationEvent event : events) {
+      Assert.assertEquals(event.getEventId(), id++);
+    }
+    Assert.assertTrue(events.size() == 2);
+  }
+
+  /**
+   * Representation of a Hive table. A table has a name and a list of partitions.
+   */
+  private static class HiveTable {
+
+    private final String name;
+    private final List<String> partitions;
+
+    HiveTable(String name) {
+      this.name = name;
+      this.partitions = new ArrayList<>();
+    }
+
+    HiveTable add(String partition) {
+      partitions.add(partition);
+      return this;
+    }
+  }
+
+  /**
+   * Representation of a Hive database. A database has a name and a list of tables
+   */
+  private static class HiveDb {
+
+    final String name;
+    Collection<HiveTable> tables;
+
+    @SuppressWarnings("SameParameterValue")
+    HiveDb(String name) {
+      this.name = name;
+      tables = new ArrayList<>();
+    }
+
+    HiveDb(String name, Collection<HiveTable> tables) {
+      this.name = name;
+      this.tables = tables;
+      if (this.tables == null) {
+        this.tables = new ArrayList<>();
+      }
+    }
+
+    void add(HiveTable table) {
+      this.tables.add(table);
+    }
+  }
+
+  /**
+   * Representation of a full Hive snapshot. A snapshot is collection of databases
+   */
+  private static class HiveSnapshot {
+
+    final List<HiveDb> databases = new ArrayList<>();
+
+    HiveSnapshot() {
+    }
+
+    HiveSnapshot(Collection<HiveDb> dblist) {
+      if (dblist != null) {
+        databases.addAll(dblist);
+      }
+    }
+
+    HiveSnapshot add(HiveDb db) {
+      this.databases.add(db);
+      return this;
+    }
+  }
+
+  /**
+   * Mock for HMSClientFactory
+   */
+  private static class MockHMSClientFactory implements HiveConnectionFactory {
+
+    private HiveMetaStoreClient mClient;
+
+    public MockHMSClientFactory() {
+      mClient  = null;
+    }
+
+    void setClient(MockClient mockClient) {
+      this.mClient = mockClient.client;
+    }
+    @Override
+    public HMSClient connect() throws IOException, InterruptedException, MetaException {
+      return new HMSClient(mClient);
+    }
+
+    @Override
+    public void close() throws Exception {
+    }
+  }
+
+  /**
+   * Convert Hive snapshot to mock client that will return proper values
+   * for the snapshot.
+   */
+  private static class MockClient {
+
+    public HiveMetaStoreClient client;
+    public long eventId;
+
+    MockClient(HiveSnapshot snapshot, long eventId) throws TException {
+      this.eventId = eventId;
+      client = Mockito.mock(HiveMetaStoreClient.class);
+      List<String> dbNames = new ArrayList<>(snapshot.databases.size());
+      // Walk over all databases and mock appropriate objects
+      for (HiveDb mdb : snapshot.databases) {
+        String dbName = mdb.name;
+        dbNames.add(dbName);
+        Database db = makeDb(dbName);
+        Mockito.when(client.getDatabase(dbName)).thenReturn(db);
+        List<String> tableNames = new ArrayList<>(mdb.tables.size());
+        // Walk over all tables for the database and mock appropriate objects
+        for (HiveTable table : mdb.tables) {
+          String tableName = table.name;
+          tableNames.add(tableName);
+          Table mockTable = makeTable(dbName, tableName);
+          Mockito.when(client.getTableObjectsByName(dbName,
+              Lists.newArrayList(tableName)))
+              .thenReturn(Lists.newArrayList(mockTable));
+          Mockito.when(client.listPartitionNames(dbName, tableName, (short) -1))
+              .thenReturn(table.partitions);
+          // Walk across all partitions and mock appropriate objects
+          for (String partName : table.partitions) {
+            Partition p = makePartition(dbName, tableName, partName);
+            Mockito.when(client.getPartitionsByNames(dbName, tableName,
+                Lists.<String>newArrayList(partName)))
+                .thenReturn(Lists.<Partition>newArrayList(p));
+          }
+        }
+        Mockito.when(client.getAllTables(dbName)).thenReturn(tableNames);
+      }
+      // Return all database names
+      Mockito.when(client.getAllDatabases()).thenReturn(dbNames);
+      Mockito.when(client.getCurrentNotificationEventId()).
+          thenReturn(new CurrentNotificationEventId(eventId));
+
+    }
+
+    public Long incrementNotificationEventId() {
+      eventId = eventId + 1;
+      return eventId;
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/sentry/blob/8b897d8e/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java
index b9330cc..d619623 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java
@@ -31,7 +31,7 @@ import java.sql.Statement;
 import java.util.ArrayList;
 import java.util.List;
 
-import org.apache.sentry.service.thrift.HMSFollower;
+import org.apache.sentry.service.thrift.HmsFollower;
 import org.apache.sentry.tests.e2e.hive.AbstractTestWithStaticConfiguration;
 import org.junit.BeforeClass;
 import org.junit.Before;
@@ -78,7 +78,7 @@ public class TestDbPrivilegeCleanupOnDrop extends
     to.close();
     // Check the HMS connection only when notification log is enabled.
     if (enableNotificationLog) {
-      while (!HMSFollower.isConnectedToHMS()) {
+      while (!HmsFollower.isConnectedToHms()) {
         Thread.sleep(1000);
       }
     }


Mime
View raw message