sentry-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ann...@apache.org
Subject sentry git commit: SENTRY-1164: fix testCaseSensitivity test failure on a real cluster and also a minor improvements to testConcurrentClients to run locally. (Anne Yu, reviewed by Haohao). [Forced Update!]
Date Mon, 04 Apr 2016 21:07:24 GMT
Repository: sentry
Updated Branches:
  refs/heads/master 58c7c888e -> 1b0596142 (forced update)


SENTRY-1164: fix testCaseSensitivity test failure on a real cluster and also a minor improvements
to testConcurrentClients to run locally. (Anne Yu, reviewed by Haohao).


Project: http://git-wip-us.apache.org/repos/asf/sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/1b059614
Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/1b059614
Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/1b059614

Branch: refs/heads/master
Commit: 1b0596142ddc93523c2edd91c8179489e9aea759
Parents: bfb354f
Author: Anne Yu <anneyu@apache.org>
Authored: Tue Mar 29 16:29:50 2016 -0700
Committer: Anne Yu <anneyu@apache.org>
Committed: Mon Apr 4 14:37:05 2016 -0700

----------------------------------------------------------------------
 .../e2e/dbprovider/TestConcurrentClients.java   | 96 ++++++++++++++++----
 .../AbstractTestWithStaticConfiguration.java    |  4 +-
 .../sentry/tests/e2e/hive/TestOperations.java   | 20 +++-
 3 files changed, 97 insertions(+), 23 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sentry/blob/1b059614/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestConcurrentClients.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestConcurrentClients.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestConcurrentClients.java
index bf871f0..13b6e62 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestConcurrentClients.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestConcurrentClients.java
@@ -16,15 +16,24 @@
  */
 package org.apache.sentry.tests.e2e.dbprovider;
 
+import org.apache.hadoop.hive.conf.HiveConf;
+
 import org.apache.sentry.provider.db.service.thrift.SentryPolicyServiceClient;
+import org.apache.sentry.provider.db.service.thrift.TSentryPrivilege;
+import org.apache.sentry.provider.db.service.thrift.TSentryRole;
 import org.apache.sentry.provider.file.PolicyFile;
 import org.apache.sentry.tests.e2e.hive.AbstractTestWithStaticConfiguration;
 
 import org.apache.sentry.tests.e2e.hive.StaticUserGroup;
-import static org.junit.Assume.assumeTrue;
+
+import static org.hamcrest.CoreMatchers.not;
+import static org.junit.Assert.*;
+import static org.junit.matchers.JUnitMatchers.containsString;
+
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -35,11 +44,10 @@ import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.commons.lang.RandomStringUtils;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertEquals;
 
 /**
  * The test class implements concurrency tests to test:
@@ -56,14 +64,19 @@ public class TestConcurrentClients extends AbstractTestWithStaticConfiguration
{
           "sentry.e2e.concurrency.test.tables-per-db", "1"));
   private final int NUM_OF_PAR = Integer.parseInt(System.getProperty(
           "sentry.e2e.concurrency.test.partitions-per-tb", "3"));
+  // number of threads < half of number of tasks, so that there will be
+  // more than 1 threads for each task to be able to test synchronization
   private final int NUM_OF_THREADS = Integer.parseInt(System.getProperty(
-          "sentry.e2e.concurrency.test.threads", "30"));
+          "sentry.e2e.concurrency.test.threads", "2"));
+  // since test time out in 10 mins, no more than 600 / 10 = 60 tasks
   private final int NUM_OF_TASKS = Integer.parseInt(System.getProperty(
-          "sentry.e2e.concurrency.test.tasks", "100"));
+          "sentry.e2e.concurrency.test.tasks", "8"));
   private final Long HS2_CLIENT_TEST_DURATION_MS = Long.parseLong(System.getProperty(
-          "sentry.e2e.concurrency.test.hs2client.test.time.ms", "10000")); //millis
+          "sentry.e2e.concurrency.test.hs2client.test.time.ms", "4000")); //millis
   private final Long SENTRY_CLIENT_TEST_DURATION_MS = Long.parseLong(System.getProperty(
-          "sentry.e2e.concurrency.test.sentryclient.test.time.ms", "10000")); //millis
+          "sentry.e2e.concurrency.test.sentryclient.test.time.ms", "4000")); //millis
+  private final Long EXECUTOR_THREADS_MAX_WAIT_TIME = Long.parseLong(System.getProperty(
+      "sentry.e2e.concurrency.test.max.wait.time", Integer.toString(NUM_OF_TASKS * 10)));
// secs
 
   private static Map<String, String> privileges = new HashMap<String, String>();
   static {
@@ -81,14 +94,13 @@ public class TestConcurrentClients extends AbstractTestWithStaticConfiguration
{
 
   @BeforeClass
   public static void setupTestStaticConfiguration() throws Exception {
-    assumeTrue(Boolean.parseBoolean(System.getProperty("sentry.scaletest.oncluster", "false")));
     useSentryService = true;    // configure sentry client
     clientKerberos = true; // need to get client configuration from testing environments
     AbstractTestWithStaticConfiguration.setupTestStaticConfiguration();
  }
 
   static String randomString( int len ){
-    return RandomStringUtils.random(len, true, false);
+    return RandomStringUtils.random(len, true, false).toLowerCase();
   }
 
   private void execStmt(Statement stmt, String sql) throws Exception {
@@ -240,8 +252,8 @@ public class TestConcurrentClients extends AbstractTestWithStaticConfiguration
{
    * Privileges are correctly created and updated.
    * @throws Exception
    */
-  @Test
-  public void testConccurentHS2Client() throws Exception {
+  @Test(timeout=600000) // time out in 10 mins
+  public void testConcurrentHS2Client() throws Exception {
     ExecutorService executor = Executors.newFixedThreadPool(NUM_OF_THREADS);
     final TestRuntimeState state = new TestRuntimeState();
 
@@ -280,7 +292,8 @@ public class TestConcurrentClients extends AbstractTestWithStaticConfiguration
{
       });
     }
     executor.shutdown();
-    while (!executor.isTerminated()) {
+    while (!executor.awaitTermination(EXECUTOR_THREADS_MAX_WAIT_TIME, TimeUnit.SECONDS))
{
+      LOGGER.info("Awaiting completion of threads.");
       Thread.sleep(1000); //millisecond
     }
     Throwable ex = state.getFirstException();
@@ -292,14 +305,18 @@ public class TestConcurrentClients extends AbstractTestWithStaticConfiguration
{
    * Test when concurrent sentry clients talking to sentry server, threads data are synchronized
    * @throws Exception
    */
-  @Test
+  @Test(timeout=600000) // time out in 10 mins
   public void testConcurrentSentryClient() throws Exception {
     final String HIVE_KEYTAB_PATH =
-            System.getProperty("sentry.e2etest.hive.policyOwnerKeytab");
+        System.getProperty("sentry.e2etest.hive.policyOwnerKeytab");
     final SentryPolicyServiceClient client = getSentryClient("hive", HIVE_KEYTAB_PATH);
     ExecutorService executor = Executors.newFixedThreadPool(NUM_OF_THREADS);
 
     final TestRuntimeState state = new TestRuntimeState();
+    String scratchLikeDir = context.getProperty(HiveConf.ConfVars.SCRATCHDIR.varname);
+    final String uriPrefix = scratchLikeDir.contains("://") ?
+        scratchLikeDir : (fileSystem.getUri().toString() + scratchLikeDir);
+    LOGGER.info("uriPrefix = " + uriPrefix);
     for (int i = 0; i < NUM_OF_TASKS; i ++) {
       LOGGER.info("Start to test sentry client with task id [" + i + "]");
       executor.execute(new Runnable() {
@@ -312,16 +329,58 @@ public class TestConcurrentClients extends AbstractTestWithStaticConfiguration
{
           try {
             String randStr = randomString(5);
             String test_role = "test_role_" + randStr;
+            String test_uri = uriPrefix + randStr;
+
             LOGGER.info("Start to test role: " + test_role);
             Long startTime = System.currentTimeMillis();
             Long elapsedTime = 0L;
             while (Long.compare(elapsedTime, SENTRY_CLIENT_TEST_DURATION_MS) <= 0) {
               LOGGER.info("Test role " + test_role + " runs " + elapsedTime + " ms.");
               client.createRole(ADMIN1, test_role);
-              client.listRoles(ADMIN1);
+              client.grantRoleToGroup(ADMIN1, ADMINGROUP, test_role);
+
+              // validate role
+              Set<TSentryRole> sentryRoles = client.listRoles(ADMIN1);
+              String results = "";
+              for (TSentryRole role : sentryRoles) {
+                results += role.toString() + "|";
+              }
+              LOGGER.info("listRoles = " + results);
+              assertThat(results, containsString("roleName:" + test_role));
+
+              // validate privileges
+              results = "";
               client.grantServerPrivilege(ADMIN1, test_role, "server1", false);
-              client.listAllPrivilegesByRoleName(ADMIN1, test_role);
+              client.grantURIPrivilege(ADMIN1, test_role, "server1", test_uri);
+              Set<TSentryPrivilege> sPerms = client.listAllPrivilegesByRoleName(ADMIN1,
test_role);
+              for (TSentryPrivilege sp : sPerms) {
+                results += sp.toString() + "|";
+              }
+              LOGGER.info("listAllPrivilegesByRoleName = " + results);
+              assertThat(results, containsString("serverName:server1"));
+              assertThat(results, containsString("URI:" + test_uri));
+
+              client.revokeURIPrivilege(ADMIN1, test_role, "server1", test_uri);
+              client.revokeServerPrivilege(ADMIN1, test_role, "server1", false);
+
+              results = "";
+              Set<TSentryPrivilege> rPerms = client.listAllPrivilegesByRoleName(ADMIN1,
test_role);
+              for (TSentryPrivilege rp : rPerms) {
+                results += rp.toString() + "|";
+              }
+              assertThat(results, not(containsString("URI:" + test_uri)));
+              assertThat(results, not(containsString("serverName:server1")));
+
               client.dropRole(ADMIN1, test_role);
+
+              results = "";
+              Set<TSentryRole> removedRoles = client.listUserRoles(ADMIN1);
+              for (TSentryRole role : removedRoles) {
+                results += role.toString() + "|";
+              }
+              LOGGER.info("listUserRoles = " + results);
+              assertThat(results, not(containsString(test_role)));
+
               elapsedTime = System.currentTimeMillis() - startTime;
             }
             state.setNumSuccess();
@@ -333,7 +392,8 @@ public class TestConcurrentClients extends AbstractTestWithStaticConfiguration
{
       });
     }
     executor.shutdown();
-    while (!executor.isTerminated()) {
+    while (!executor.awaitTermination(EXECUTOR_THREADS_MAX_WAIT_TIME, TimeUnit.SECONDS))
{
+      LOGGER.info("Awaiting completion of threads.");
       Thread.sleep(1000); //millisecond
     }
     Throwable ex = state.getFirstException();

http://git-wip-us.apache.org/repos/asf/sentry/blob/1b059614/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java
index b961757..8515a2b 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java
@@ -151,6 +151,7 @@ public abstract class AbstractTestWithStaticConfiguration {
 
   private static LoginContext clientLoginContext;
   protected static SentryPolicyServiceClient client;
+  private static boolean startSentry = new Boolean(System.getProperty(EXTERNAL_SENTRY_SERVICE,
"false"));
 
   /**
    * Get sentry client with authenticated Subject
@@ -265,7 +266,6 @@ public abstract class AbstractTestWithStaticConfiguration {
       policyURI = policyFileLocation.getPath();
     }
 
-    boolean startSentry = Boolean.valueOf(System.getProperty(EXTERNAL_SENTRY_SERVICE, "false"));
     if ("true".equalsIgnoreCase(System.getProperty(ENABLE_SENTRY_HA, "false"))) {
       enableSentryHA = true;
     }
@@ -495,7 +495,7 @@ public abstract class AbstractTestWithStaticConfiguration {
    */
   public static SentryPolicyServiceClient getSentryClient(String clientShortName,
                                                           String clientKeyTabDir) throws
Exception {
-    if (!useSentryService) {
+    if (!startSentry) {
       LOGGER.info("Running on a minicluser env.");
       return getSentryClient();
     }

http://git-wip-us.apache.org/repos/asf/sentry/blob/1b059614/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestOperations.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestOperations.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestOperations.java
index 7d3db2b..77106d4 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestOperations.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestOperations.java
@@ -1087,9 +1087,23 @@ public class TestOperations extends AbstractTestWithStaticConfiguration
{
     try {
       createDb(ADMIN1, DB1);
       String scratchLikeDir = context.getProperty(HiveConf.ConfVars.SCRATCHDIR.varname);
-      String extParentDir = dfs.assertCreateDir(scratchLikeDir + "/ABC/hhh").toUri().toString();
-      String extTableDir = dfs.assertCreateDir(scratchLikeDir + "/abc/hhh").toUri().toString();
-      LOGGER.info("Created extParentDir = " + extParentDir + ", extTableDir = " + extTableDir);
+      LOGGER.info("scratch like dir = " + scratchLikeDir);
+      String extParentDir = scratchLikeDir + "/ABC/hhh";
+      String extTableDir = scratchLikeDir + "/abc/hhh";
+      LOGGER.info("Creating extParentDir = " + extParentDir + ", extTableDir = " + extTableDir);
+      dfs.assertCreateDir(extParentDir);
+      dfs.assertCreateDir(extTableDir);
+
+      if (! (extParentDir.toLowerCase().startsWith("hdfs://")
+          || extParentDir.toLowerCase().startsWith("s3://")
+          || extParentDir.contains("://"))) {
+        String scheme = fileSystem.getUri().toString();
+        LOGGER.info("scheme = " + scheme);
+        extParentDir = scheme + extParentDir;
+        extTableDir = scheme + extTableDir;
+        LOGGER.info("Add scheme in extParentDir = " + extParentDir + ", extTableDir = " +
extTableDir);
+      }
+
       policyFile
           .addPermissionsToRole("all_db1", privileges.get("all_db1"))
           .addPermissionsToRole("all_uri", "server=server1->uri=" + extParentDir)


Mime
View raw message