sentry-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kal...@apache.org
Subject [1/2] sentry git commit: Revert "SENTRY-2109: Fix the logic of identifying HMS out of Sync and handle gaps and out-of-sequence notifications.(Kalyan Kumar kalvagadda, reviewed-by Vadim Spector, Na Li and Arjun Mishra)" Reverting the patch as it needs som
Date Fri, 02 Feb 2018 21:14:38 GMT
Repository: sentry
Updated Branches:
  refs/heads/master afcaa4997 -> a178d7ed8


http://git-wip-us.apache.org/repos/asf/sentry/blob/a178d7ed/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestSnapshotCreationWithShorterHMSEventTtl.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestSnapshotCreationWithShorterHMSEventTtl.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestSnapshotCreationWithShorterHMSEventTtl.java
deleted file mode 100644
index f838005..0000000
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestSnapshotCreationWithShorterHMSEventTtl.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sentry.tests.e2e.dbprovider;
-
-import static org.junit.Assert.assertEquals;
-
-import java.sql.Connection;
-import java.sql.Statement;
-
-import org.apache.sentry.tests.e2e.hdfs.TestHDFSIntegrationBase;
-import org.apache.sentry.tests.e2e.hive.SlowE2ETest;
-
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-/**
- * This class covers scenario where HMSFollower takes a snapshot initially when sentry server
comes up
- * and not subsequently even when HMS event information is cleaned up as there is no out
of sync detected.
- */
-@SlowE2ETest
-public class TestSnapshotCreationWithShorterHMSEventTtl extends TestHDFSIntegrationBase {
-
-  private final static String tableName1 = "tb_1";
-  private final static String tableName2 = "tb_2";
-  private final static String tableName3 = "tb_3";
-  private final static String tableName4 = "tb_4";
-
-  protected static final String ALL_DB1 = "server=server1->db=db_1",
-          DB1 = "db_1",
-          DB2 = "db_2";
-
-  private Connection connection;
-  private Statement statement;
-
-  @BeforeClass
-  public static void setup() throws Exception {
-    shorterMetaStoreEventDbTtl = true;
-    TestHDFSIntegrationBase.setup();
-  }
-
-  @Before
-  public void initialize() throws Exception {
-    super.setUpTempDir();
-    admin = "hive";
-    connection = hiveServer2.createConnection(admin, admin);
-    statement = connection.createStatement();
-    statement.execute("create role admin_role");
-    statement.execute("grant role admin_role to group hive");
-    statement.execute("grant all on server server1 to role admin_role");
-  }
-
-  @Test
-  public void BasicSanity() throws Exception {
-    long latestSnapshotId = 0;
-    dbNames = new String[]{DB1};
-    roles = new String[]{"admin_role", "all_db1", "all_tbl1", "all_tbl2"};
-    do {
-      //Sleep for a sec allowing HMSFollower to create a snapshot
-      Thread.sleep(1000);
-      latestSnapshotId = sentryServer.get(0).getCurrentAuthzPathsSnapshotID();
-    } while (latestSnapshotId == 0);
-
-    statement.execute("CREATE DATABASE " + DB1);
-    statement.execute("CREATE DATABASE " + DB2);
-    statement.execute("create table " + DB1 + "." + tableName1
-            + " (under_col int comment 'the under column', value string)");
-    statement.execute("create table " + DB1 + "." + tableName2
-            + " (under_col int comment 'the under column', value string)");
-
-    statement.execute("create table " + DB2 + "." + tableName3
-            + " (under_col int comment 'the under column', value string)");
-
-    /*
-    With shorter TTL HMS would evict the entries in NOTIFICATION_LOG table faster.
-    Which could be value configured for "hive.metastore.event.db.listener.timetolive"
-    + 60 sec. As the cleanup happens every 60 sec. Test sleeps for 70 sec to make sure that
-    cleanup happened.
-     */
-    Thread.sleep(70000);
-    assertEquals("Another snapshot is created",
-            latestSnapshotId, sentryServer.get(0).getCurrentAuthzPathsSnapshotID());
-    statement.execute("create table " + DB2 + "." + tableName4
-            + " (under_col int comment 'the under column', value string)");
-
-    Thread.sleep(maxDelayInFetchingHMSNotifications());
-    assertEquals("Another snapshot is created",
-            latestSnapshotId, sentryServer.get(0).getCurrentAuthzPathsSnapshotID());
-  }
-}

http://git-wip-us.apache.org/repos/asf/sentry/blob/a178d7ed/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestSnapshotWithLongerHMSFollowerLongerInterval.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestSnapshotWithLongerHMSFollowerLongerInterval.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestSnapshotWithLongerHMSFollowerLongerInterval.java
deleted file mode 100644
index 6bc5d11..0000000
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestSnapshotWithLongerHMSFollowerLongerInterval.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sentry.tests.e2e.dbprovider;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-
-import java.sql.Connection;
-import java.sql.Statement;
-
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.apache.sentry.tests.e2e.hdfs.TestHDFSIntegrationBase;
-import org.apache.sentry.tests.e2e.hive.SlowE2ETest;
-
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * This class covers scenario where HMSFollower takes a snapshot initially when sentry server
comes up
- * and also subsequently when HMS event information is cleaned up and HMSFollower detects
that it is out of sync and can
- * not recover by fetching delta notifications.
- */
-@Ignore
-@SlowE2ETest
-public class TestSnapshotWithLongerHMSFollowerLongerInterval extends TestHDFSIntegrationBase
{
-
-  private static final Logger LOGGER = LoggerFactory
-          .getLogger(TestSnapshotWithLongerHMSFollowerLongerInterval.class);
-
-  protected static final String DB1 = "db_1",
-          DB2 = "db_2";
-
-  private Connection connection;
-  private Statement statement;
-
-  @BeforeClass
-  public static void setup() throws Exception {
-    // Reduces the TTL of the HMS event data so that it is cleaned up faster.
-    shorterMetaStoreEventDbTtl = true;
-    // Increases the interval between the fetches the HMSFollower does.
-    longerHMSFollowerInterval = true;
-    TestHDFSIntegrationBase.setup();
-  }
-
-  @Before
-  public void initialize() throws Exception {
-    super.setUpTempDir();
-    admin = "hive";
-    connection = hiveServer2.createConnection(admin, admin);
-    statement = connection.createStatement();
-    statement.execute("create role admin_role");
-    statement.execute("grant role admin_role to group hive");
-    statement.execute("grant all on server server1 to role admin_role");
-  }
-
-  @Test
-  public void BasicSanity() throws Exception {
-    long latestSnapshotId = 0;
-    Thread th = null;
-    dbNames = new String[]{DB1};
-    roles = new String[]{"admin_role", "all_db1", "all_tbl1", "all_tbl2"};
-    do {
-      //Sleep for a sec allowing HMSFollower to create a snapshot
-      Thread.sleep(1000);
-      latestSnapshotId = sentryServer.get(0).getCurrentAuthzPathsSnapshotID();
-    } while (latestSnapshotId == 0);
-
-    final AtomicBoolean keepRunning = new AtomicBoolean(true);
-    th = new Thread() {
-      @Override
-      public void run() {
-        try {
-          int counter = 1;
-          while (keepRunning.get()) {
-            statement.execute("CREATE DATABASE " + "db"+counter);
-            Thread.sleep(1000L);
-            counter++;
-          }
-        } catch (Exception e) {
-          LOGGER.info("Could not start Hive Server");
-        }
-      }
-    };
-    th.start();
-
-    Thread.sleep(130000);
-    assertEquals("Another snapshot is created, Snapshot ID: ", latestSnapshotId, sentryServer.get(0).getCurrentAuthzPathsSnapshotID());
-
-    Thread.sleep(130000);
-    long newSnapShot = sentryServer.get(0).getCurrentAuthzPathsSnapshotID();
-    assertFalse("Another snapshot should have been created",
-            (latestSnapshotId == newSnapShot));
-  }
-}

http://git-wip-us.apache.org/repos/asf/sentry/blob/a178d7ed/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java
index dc322b8..4cd00e6 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java
@@ -158,8 +158,6 @@ public abstract class TestHDFSIntegrationBase {
   protected static SentrySrv sentryServer;
   protected static boolean testSentryHA = false;
   protected static final long STALE_THRESHOLD = 5000;
-  protected static Boolean shorterMetaStoreEventDbTtl = false;
-  protected static Boolean longerHMSFollowerInterval = false;
 
   // It is the interval in milliseconds that hdfs uses to get acl from sentry. Default is
500, but
   // we want it to be low in our tests so that changes reflect soon
@@ -542,9 +540,6 @@ public abstract class TestHDFSIntegrationBase {
         hiveConf.set("hive.metastore.authorization.storage.checks", "true");
         hiveConf.set("hive.metastore.uris", "thrift://localhost:" + hmsPort);
         hiveConf.set("sentry.metastore.service.users", "hive");// queries made by hive user
(beeline) skip meta store check
-        if(shorterMetaStoreEventDbTtl) {
-          hiveConf.set("hive.metastore.event.db.listener.timetolive", "5s");
-        }
 
         File confDir = assertCreateDir(new File(baseDir, "etc"));
         File hiveSite = new File(confDir, "hive-site.xml");
@@ -879,14 +874,7 @@ public abstract class TestHDFSIntegrationBase {
                     "org.apache.sentry.provider.db.service.thrift.SentryPolicyStoreProcessorFactory,org.apache.sentry.hdfs.SentryHDFSServiceProcessorFactory");
             sentryProperties.put("sentry.policy.store.plugins", "org.apache.sentry.hdfs.SentryPlugin");
           }
-
-          if(longerHMSFollowerInterval) {
-            // Idea of increasing HMSFollower interval is to fetching after HMS notifications
are evicted.
-            //This configuration is dependent on hive.metastore.event.db.listener.timetolive.
-            // HMS runs cleaner every 60 sec. Below configuration should be 60 + TTL + buffer-time
-            sentryProperties.put(ServerConfig.SENTRY_HMSFOLLOWER_INTERVAL_MILLS, "120000");
-          }
-          for (Map.Entry<String, String> entry : sentryProperties.entrySet()) {
+            for (Map.Entry<String, String> entry : sentryProperties.entrySet()) {
             sentryConf.set(entry.getKey(), entry.getValue());
           }
           sentryServer = SentrySrvFactory.create(SentrySrvFactory.SentrySrvType.INTERNAL_SERVER,
@@ -902,28 +890,7 @@ public abstract class TestHDFSIntegrationBase {
     }
   }
 
-  /**
-   * Method calculates the maximun time it could take to fetch a notification that is inserted
to
-   * NOTIFICATION_LOG table of HMS database.
-   * @return maximum delay in fetching notification logged by HMS.
-   */
-  protected long maxDelayInFetchingHMSNotifications() {
-    long interval;
-    long initDelay;
-    if (sentryProperties.containsKey(ServerConfig.SENTRY_HMSFOLLOWER_INTERVAL_MILLS)) {
-      interval = Long.parseLong(sentryProperties.get(ServerConfig.SENTRY_HMSFOLLOWER_INTERVAL_MILLS));
-    } else {
-      interval = ServerConfig.SENTRY_HMSFOLLOWER_INTERVAL_MILLS_DEFAULT;
-    }
-    if (sentryProperties.containsKey(ServerConfig.SENTRY_HMSFOLLOWER_INIT_DELAY_MILLS)) {
-      initDelay = Long.parseLong(sentryProperties.get(ServerConfig.SENTRY_HMSFOLLOWER_INIT_DELAY_MILLS));
-    } else {
-      initDelay = ServerConfig.SENTRY_HMSFOLLOWER_INIT_DELAY_MILLS_DEFAULT;
-    }
-    return (interval + initDelay);
-  }
-
- @After
+  @After
   public void cleanAfterTest() throws Exception {
     //Clean up database
     Connection conn;
@@ -944,11 +911,7 @@ public abstract class TestHDFSIntegrationBase {
     stmt = conn.createStatement();
     LOGGER.info("About to clear all roles");
     for( String role:roles) {
-      try {
-        stmt.execute("drop role " + role);
-      } catch (Exception e) {
-        LOGGER.info("Exception while dropping role", e.getMessage());
-      }
+      stmt.execute("drop role " + role);
     }
     stmt.close();
     conn.close();


Mime
View raw message