sentry-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jar...@apache.org
Subject git commit: SENTRY-176: Not able to read policy files on HDFS (Regression) (Sravya Tirukkovalur via Jarek Jarcec Cecho)
Date Thu, 08 May 2014 15:58:17 GMT
Repository: incubator-sentry
Updated Branches:
  refs/heads/master 1a72f6197 -> 4adf59a69


SENTRY-176: Not able to read policy files on HDFS (Regression) (Sravya Tirukkovalur via Jarek
Jarcec Cecho)


Project: http://git-wip-us.apache.org/repos/asf/incubator-sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-sentry/commit/4adf59a6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-sentry/tree/4adf59a6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-sentry/diff/4adf59a6

Branch: refs/heads/master
Commit: 4adf59a691e83fa625fb7b1d24b7934c323a5a02
Parents: 1a72f61
Author: Jarek Jarcec Cecho <jarcec@apache.org>
Authored: Thu May 8 08:57:05 2014 -0700
Committer: Jarek Jarcec Cecho <jarcec@apache.org>
Committed: Thu May 8 08:57:05 2014 -0700

----------------------------------------------------------------------
 .../sentry/binding/hive/conf/HiveAuthzConf.java | 16 +++--
 .../apache/sentry/provider/file/PolicyFile.java | 70 ++++----------------
 .../sentry/provider/file/PolicyFiles.java       | 19 +++---
 .../file/SimpleFileProviderBackend.java         | 55 ++++++++-------
 .../e2e/hive/AbstractTestWithHiveServer.java    | 10 ++-
 .../AbstractTestWithStaticConfiguration.java    | 32 +++++----
 .../sentry/tests/e2e/hive/TestCrossDbOps.java   | 35 +++++++---
 .../e2e/hive/TestPerDatabasePolicyFile.java     | 14 ++--
 .../tests/e2e/hive/TestServerConfiguration.java | 32 ++++-----
 .../sentry/tests/e2e/hive/fs/AbstractDFS.java   | 20 +++++-
 .../sentry/tests/e2e/hive/fs/ClusterDFS.java    |  7 +-
 .../apache/sentry/tests/e2e/hive/fs/DFS.java    |  4 ++
 .../sentry/tests/e2e/hive/fs/MiniDFS.java       | 10 +--
 .../e2e/hive/hiveserver/HiveServerFactory.java  | 34 ++++------
 14 files changed, 169 insertions(+), 189 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/4adf59a6/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/conf/HiveAuthzConf.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/conf/HiveAuthzConf.java
b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/conf/HiveAuthzConf.java
index e162bbd..7b7bf8e 100644
--- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/conf/HiveAuthzConf.java
+++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/conf/HiveAuthzConf.java
@@ -16,18 +16,18 @@
  */
 package org.apache.sentry.binding.hive.conf;
 
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Map.Entry;
-
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.mortbay.log.Log;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+
 
 public class HiveAuthzConf extends Configuration {
 
@@ -146,9 +146,11 @@ public class HiveAuthzConf extends Configuration {
   private final String hiveAuthzSiteFile;
 
   public HiveAuthzConf(URL hiveAuthzSiteURL) {
-    super(false);
+    super();
+    LOG.info("DefaultFS: " + super.get("fs.defaultFS"));
     addResource(hiveAuthzSiteURL);
     applySystemProperties();
+    LOG.info("DefaultFS: " + super.get("fs.defaultFS"));
     this.hiveAuthzSiteFile = hiveAuthzSiteURL.toString();
   }
   /**

http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/4adf59a6/sentry-provider/sentry-provider-file/src/main/java/org/apache/sentry/provider/file/PolicyFile.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-file/src/main/java/org/apache/sentry/provider/file/PolicyFile.java
b/sentry-provider/sentry-provider-file/src/main/java/org/apache/sentry/provider/file/PolicyFile.java
index 0189f85..19bf8cc 100644
--- a/sentry-provider/sentry-provider-file/src/main/java/org/apache/sentry/provider/file/PolicyFile.java
+++ b/sentry-provider/sentry-provider-file/src/main/java/org/apache/sentry/provider/file/PolicyFile.java
@@ -17,28 +17,26 @@
 
 package org.apache.sentry.provider.file;
 
-import static org.apache.sentry.provider.file.PolicyFileConstants.DATABASES;
-import static org.apache.sentry.provider.file.PolicyFileConstants.GROUPS;
-import static org.apache.sentry.provider.file.PolicyFileConstants.ROLES;
-import static org.apache.sentry.provider.file.PolicyFileConstants.USERS;
-
-import java.io.File;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Charsets;
 import com.google.common.base.Joiner;
-import com.google.common.base.Preconditions;
 import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Multimap;
 import com.google.common.io.Files;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.sentry.provider.file.PolicyFileConstants.DATABASES;
+import static org.apache.sentry.provider.file.PolicyFileConstants.GROUPS;
+import static org.apache.sentry.provider.file.PolicyFileConstants.ROLES;
+import static org.apache.sentry.provider.file.PolicyFileConstants.USERS;
 
 /**
  * PolicyFile creator. Written specifically to be used with tests. Specifically
@@ -134,50 +132,6 @@ public class PolicyFile {
     LOGGER.info("Writing policy file to " + file + ":\n" + contents);
     Files.write(contents, file, Charsets.UTF_8);
 
-    String hiveServer2 = System.getProperty("sentry.e2etest.hiveServer2Type", "InternalHiveServer2");
-
-    //Currently policyOnHDFS is only supported for UnmanagedHiveServer, and global policy
file is required to be on hdfs
-    if(hiveServer2.equals("UnmanagedHiveServer2")) {
-      String policyOnHDFS = System.getProperty("sentry.e2etest.hive.policyOnHDFS", "true");
-      if(policyOnHDFS.trim().equalsIgnoreCase("true") || file.getName().equalsIgnoreCase("sentry-provider.ini")){
-        String policyLocation = System.getProperty("sentry.e2etest.hive.policy.location",
"/user/hive/sentry");
-        String policyFileLocation = policyLocation + "/" + file.getName();
-        LOGGER.info("Moving policy file to " + policyFileLocation);
-        String userKeytab = System.getProperty("sentry.e2etest.hive.policyOwnerKeytab");
-        String userPrincipal = System.getProperty("sentry.e2etest.hive.policyOwnerPrincipal");
-        Preconditions.checkNotNull(userKeytab);
-        Preconditions.checkNotNull(userPrincipal);
-        hdfsPut(file, policyFileLocation, userKeytab, userPrincipal);
-      }
-    }
-  }
-
-  private void hdfsPut(File file, String hdfsPath, String userKeytab, String userPrincipal)
throws Exception {
-    String command, status;
-    Process p;
-
-    command = "kinit -kt " + userKeytab +  " " + userPrincipal;
-    p = Runtime.getRuntime().exec(command);
-    if(p.waitFor()!=0) {
-      throw new Exception("Setup incomplete. " + command + " FAILED");
-    }
-    else {
-      LOGGER.info("Command:" + command + " PASSED");
-    }
-
-    command = "hdfs dfs -rm " + hdfsPath;
-    p = Runtime.getRuntime().exec(command);
-    status = (p.waitFor()==0)?"PASSED":"FAILED";
-    LOGGER.warn("Command:" + command + " " + status);
-
-    command = "hdfs dfs -put " + file.getAbsolutePath() + " " + hdfsPath;
-    p = Runtime.getRuntime().exec(command);
-    if(p.waitFor()!=0) {
-      throw new Exception("Setup incomplete. " + command + " FAILED");
-    }
-    else {
-      LOGGER.info("Command:" + command + " PASSED");
-    }
   }
 
   private String getSection(String name, Map<String, String> mapping) {

http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/4adf59a6/sentry-provider/sentry-provider-file/src/main/java/org/apache/sentry/provider/file/PolicyFiles.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-file/src/main/java/org/apache/sentry/provider/file/PolicyFiles.java
b/sentry-provider/sentry-provider-file/src/main/java/org/apache/sentry/provider/file/PolicyFiles.java
index a908ec3..4f05be6 100644
--- a/sentry-provider/sentry-provider-file/src/main/java/org/apache/sentry/provider/file/PolicyFiles.java
+++ b/sentry-provider/sentry-provider-file/src/main/java/org/apache/sentry/provider/file/PolicyFiles.java
@@ -16,13 +16,8 @@
  */
 package org.apache.sentry.provider.file;
 
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-
+import com.google.common.io.ByteStreams;
+import com.google.common.io.Resources;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -30,8 +25,12 @@ import org.apache.shiro.config.Ini;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.common.io.ByteStreams;
-import com.google.common.io.Resources;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
 
 public class PolicyFiles {
 
@@ -74,6 +73,8 @@ public class PolicyFiles {
     InputStream inputStream = null;
     try {
       LOGGER.info("Opening " + path);
+      String dfsUri = fileSystem.getDefaultUri(fileSystem.getConf()).toString();
+      LOGGER.error("dfsUri " + dfsUri);
       inputStream = fileSystem.open(path);
       Ini ini = new Ini();
       ini.load(inputStream);

http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/4adf59a6/sentry-provider/sentry-provider-file/src/main/java/org/apache/sentry/provider/file/SimpleFileProviderBackend.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-file/src/main/java/org/apache/sentry/provider/file/SimpleFileProviderBackend.java
b/sentry-provider/sentry-provider-file/src/main/java/org/apache/sentry/provider/file/SimpleFileProviderBackend.java
index 2dadc47..6e8f02f 100644
--- a/sentry-provider/sentry-provider-file/src/main/java/org/apache/sentry/provider/file/SimpleFileProviderBackend.java
+++ b/sentry-provider/sentry-provider-file/src/main/java/org/apache/sentry/provider/file/SimpleFileProviderBackend.java
@@ -16,21 +16,19 @@
  */
 package org.apache.sentry.provider.file;
 
-import static org.apache.sentry.provider.file.PolicyFileConstants.DATABASES;
-import static org.apache.sentry.provider.file.PolicyFileConstants.GROUPS;
-import static org.apache.sentry.provider.file.PolicyFileConstants.ROLES;
-import static org.apache.sentry.provider.file.PolicyFileConstants.ROLE_SPLITTER;
-import static org.apache.sentry.provider.file.PolicyFileConstants.USERS;
-
-import java.io.IOException;
-import java.net.URI;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import javax.annotation.Nullable;
-
+import com.google.common.base.Splitter;
+import com.google.common.base.Strings;
+import com.google.common.collect.HashBasedTable;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Interner;
+import com.google.common.collect.Interners;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Sets;
+import com.google.common.collect.Table;
+import com.google.common.collect.Table.Cell;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -45,19 +43,19 @@ import org.apache.shiro.config.Ini;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.common.base.Splitter;
-import com.google.common.base.Strings;
-import com.google.common.collect.HashBasedTable;
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Interner;
-import com.google.common.collect.Interners;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Multimap;
-import com.google.common.collect.Sets;
-import com.google.common.collect.Table;
-import com.google.common.collect.Table.Cell;
+import javax.annotation.Nullable;
+import java.io.IOException;
+import java.net.URI;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.sentry.provider.file.PolicyFileConstants.DATABASES;
+import static org.apache.sentry.provider.file.PolicyFileConstants.GROUPS;
+import static org.apache.sentry.provider.file.PolicyFileConstants.ROLES;
+import static org.apache.sentry.provider.file.PolicyFileConstants.ROLE_SPLITTER;
+import static org.apache.sentry.provider.file.PolicyFileConstants.USERS;
 
 public class SimpleFileProviderBackend implements ProviderBackend {
 
@@ -207,6 +205,7 @@ public class SimpleFileProviderBackend implements ProviderBackend {
     Table<String, String, Set<String>> groupRolePrivilegeTableTemp = HashBasedTable.create();
     Ini ini;
     LOGGER.info("Parsing " + resourcePath);
+    LOGGER.info("Filesystem: " + fileSystem.getUri());
     try {
       try {
         ini = PolicyFiles.loadFromPath(fileSystem, resourcePath);

http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/4adf59a6/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithHiveServer.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithHiveServer.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithHiveServer.java
index 0258c2d..9b3c04a 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithHiveServer.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithHiveServer.java
@@ -16,11 +16,8 @@
  */
 package org.apache.sentry.tests.e2e.hive;
 
-import java.io.File;
-import java.util.Map;
-
+import com.google.common.io.Files;
 import junit.framework.Assert;
-
 import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
@@ -30,7 +27,8 @@ import org.junit.After;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.common.io.Files;
+import java.io.File;
+import java.util.Map;
 
 public abstract class AbstractTestWithHiveServer {
   private static final Logger LOGGER = LoggerFactory
@@ -61,7 +59,7 @@ public abstract class AbstractTestWithHiveServer {
     confDir = assertCreateDir(new File(baseDir, "etc"));
     dataDir = assertCreateDir(new File(baseDir, "data"));
     policyFile = new File(confDir, HiveServerFactory.AUTHZ_PROVIDER_FILENAME);
-    hiveServer = HiveServerFactory.create(properties, baseDir, confDir, logDir, policyFile,
fileSystem);
+    hiveServer = HiveServerFactory.create(properties, baseDir, confDir, logDir, policyFile.getPath(),
fileSystem);
     hiveServer.start();
     return new Context(hiveServer, getFileSystem(),
         baseDir, confDir, dataDir, policyFile);

http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/4adf59a6/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java
index 6444407..2b36d80 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java
@@ -16,14 +16,9 @@
  */
 package org.apache.sentry.tests.e2e.hive;
 
-import java.io.File;
-import java.sql.Connection;
-import java.sql.ResultSet;
-import java.sql.Statement;
-import java.util.Map;
-
+import com.google.common.collect.Maps;
+import com.google.common.io.Files;
 import junit.framework.Assert;
-
 import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.sentry.tests.e2e.hive.fs.DFS;
@@ -36,8 +31,11 @@ import org.junit.BeforeClass;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.common.collect.Maps;
-import com.google.common.io.Files;
+import java.io.File;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.util.Map;
 
 public abstract class AbstractTestWithStaticConfiguration {
   private static final Logger LOGGER = LoggerFactory
@@ -76,7 +74,7 @@ public abstract class AbstractTestWithStaticConfiguration {
       VIEW3 = "view_3",
       INDEX1 = "index_1",
       INDEX2 = "index_2";
-
+  protected static boolean policy_on_hdfs = false;
 
   protected static File baseDir;
   protected static File logDir;
@@ -150,9 +148,19 @@ public abstract class AbstractTestWithStaticConfiguration {
 
     String dfsType = System.getProperty(DFSFactory.FS_TYPE);
     dfs = DFSFactory.create(dfsType, baseDir);
-
     fileSystem = dfs.getFileSystem();
-    hiveServer = HiveServerFactory.create(properties, baseDir, confDir, logDir, policyFileLocation,
fileSystem);
+
+    String policyURI;
+    if (policy_on_hdfs) {
+      String dfsUri = fileSystem.getDefaultUri(fileSystem.getConf()).toString();
+      LOGGER.error("dfsUri " + dfsUri);
+      policyURI = dfsUri + System.getProperty("sentry.e2etest.hive.policy.location", "/user/hive/sentry");
+      policyURI += "/" + HiveServerFactory.AUTHZ_PROVIDER_FILENAME;
+    } else {
+      policyURI = policyFileLocation.getPath();
+    }
+
+    hiveServer = HiveServerFactory.create(properties, baseDir, confDir, logDir, policyURI,
fileSystem);
     hiveServer.start();
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/4adf59a6/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestCrossDbOps.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestCrossDbOps.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestCrossDbOps.java
index 836cb27..7ef3975 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestCrossDbOps.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestCrossDbOps.java
@@ -17,8 +17,13 @@
 
 package org.apache.sentry.tests.e2e.hive;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
+import com.google.common.io.Resources;
+import junit.framework.Assert;
+import org.apache.sentry.provider.file.PolicyFile;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
 
 import java.io.File;
 import java.io.FileOutputStream;
@@ -30,14 +35,8 @@ import java.sql.Statement;
 import java.util.ArrayList;
 import java.util.List;
 
-import junit.framework.Assert;
-
-import org.apache.sentry.provider.file.PolicyFile;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import com.google.common.io.Resources;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 
 /* Tests privileges at table scope with cross database access */
 
@@ -46,6 +45,11 @@ public class TestCrossDbOps extends AbstractTestWithStaticConfiguration
{
   private PolicyFile policyFile;
   private String loadData;
 
+  @BeforeClass
+  public static void setupTestStaticConfiguration() throws Exception{
+    policy_on_hdfs = true;
+    AbstractTestWithStaticConfiguration.setupTestStaticConfiguration();
+  }
   @Before
   public void setup() throws Exception {
     context = createContext();
@@ -84,6 +88,7 @@ public class TestCrossDbOps extends AbstractTestWithStaticConfiguration
{
         .addPermissionsToRole("insert_tab2", "server=server1->db=db2->table=tab2->action=insert")
         .setUserGroupMapping(StaticUserGroup.getStaticMapping());
     policyFile.write(context.getPolicyFile());
+    dfs.writePolicyFile(context.getPolicyFile());
 
     // admin create two databases
     Connection connection = context.createConnection(ADMIN1);
@@ -200,6 +205,7 @@ public class TestCrossDbOps extends AbstractTestWithStaticConfiguration
{
         .addPermissionsToRole("insert_tab2", "server=server1->db=db2->table=tab2->action=insert")
         .setUserGroupMapping(StaticUserGroup.getStaticMapping());
     policyFile.write(context.getPolicyFile());
+    dfs.writePolicyFile(context.getPolicyFile());
 
     // admin create two databases
     Connection connection = context.createConnection(ADMIN1);
@@ -358,6 +364,7 @@ public class TestCrossDbOps extends AbstractTestWithStaticConfiguration
{
         .addPermissionsToRole("load_data", "server=server1->URI=file://" + dataFile.getPath())
         .setUserGroupMapping(StaticUserGroup.getStaticMapping());
     policyFile.write(context.getPolicyFile());
+    dfs.writePolicyFile(context.getPolicyFile());
 
     dropDb(ADMIN1, DB1, DB2);
     createDb(ADMIN1, DB1, DB2);
@@ -388,6 +395,8 @@ public class TestCrossDbOps extends AbstractTestWithStaticConfiguration
{
     policyFile
         .setUserGroupMapping(StaticUserGroup.getStaticMapping())
         .write(context.getPolicyFile());
+    dfs.writePolicyFile(context.getPolicyFile());
+
     dropDb(ADMIN1, DB1);
     createDb(ADMIN1, DB1);
     Connection adminCon = context.createConnection(ADMIN1);
@@ -417,6 +426,7 @@ public class TestCrossDbOps extends AbstractTestWithStaticConfiguration
{
         .addPermissionsToRole("db1_tab1_insert", "server=server1->db=db1->table=table_1->action=insert")
         .setUserGroupMapping(StaticUserGroup.getStaticMapping());
     policyFile.write(context.getPolicyFile());
+    dfs.writePolicyFile(context.getPolicyFile());
 
     Connection adminCon = context.createConnection(ADMIN1);
     Statement adminStmt = context.createStatement(adminCon);
@@ -447,6 +457,7 @@ public class TestCrossDbOps extends AbstractTestWithStaticConfiguration
{
         .addRolesToGroup(USERGROUP1, "db1_tab2_all")
         .setUserGroupMapping(StaticUserGroup.getStaticMapping());
     policyFile.write(context.getPolicyFile());
+    dfs.writePolicyFile(context.getPolicyFile());
 
     dropDb(ADMIN1, DB1);
     createDb(ADMIN1, DB1);
@@ -493,6 +504,7 @@ public class TestCrossDbOps extends AbstractTestWithStaticConfiguration
{
         .addPermissionsToRole("db1_tab1_select", "server=server1->db=db1->table=table_1->action=select")
         .setUserGroupMapping(StaticUserGroup.getStaticMapping())
         .write(context.getPolicyFile());
+    dfs.writePolicyFile(context.getPolicyFile());
 
     // create dbs
     Connection adminCon = context.createConnection(ADMIN1);
@@ -570,6 +582,7 @@ public class TestCrossDbOps extends AbstractTestWithStaticConfiguration
{
         .addRolesToGroup(USERGROUP1, GROUP1_ROLE)
         .setUserGroupMapping(StaticUserGroup.getStaticMapping())
         .write(context.getPolicyFile());
+    dfs.writePolicyFile(context.getPolicyFile());
 
     dropDb(ADMIN1, DB1, DB2);
     createDb(ADMIN1, DB1, DB2);
@@ -610,6 +623,7 @@ public class TestCrossDbOps extends AbstractTestWithStaticConfiguration
{
 
     policyFile.removePermissionsFromRole(GROUP1_ROLE, ALL_DB2);
     policyFile.write(context.getPolicyFile());
+    dfs.writePolicyFile(context.getPolicyFile());
 
     // create db1.view1 as select from db2.tbl2
     statement.execute("DROP VIEW IF EXISTS " + DB1 + "." + VIEW2);
@@ -648,6 +662,7 @@ public class TestCrossDbOps extends AbstractTestWithStaticConfiguration
{
         .addPermissionsToRole("load_data", "server=server1->URI=file://" + dataFile.getPath())
         .setUserGroupMapping(StaticUserGroup.getStaticMapping())
         .write(context.getPolicyFile());
+    dfs.writePolicyFile(context.getPolicyFile());
 
     // admin create two databases
     dropDb(ADMIN1, DB1, DB2);

http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/4adf59a6/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPerDatabasePolicyFile.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPerDatabasePolicyFile.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPerDatabasePolicyFile.java
index 805d60f..5c7c7d1 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPerDatabasePolicyFile.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPerDatabasePolicyFile.java
@@ -17,19 +17,16 @@
 
 package org.apache.sentry.tests.e2e.hive;
 
-import static org.junit.Assert.assertTrue;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.sql.Connection;
-import java.sql.Statement;
-
+import com.google.common.io.Resources;
 import org.apache.sentry.provider.file.PolicyFile;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
-import com.google.common.io.Resources;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.sql.Connection;
+import java.sql.Statement;
 
 public class TestPerDatabasePolicyFile extends AbstractTestWithStaticConfiguration {
   private static final String SINGLE_TYPE_DATA_FILE_NAME = "kv1.dat";
@@ -45,7 +42,6 @@ public class TestPerDatabasePolicyFile extends AbstractTestWithStaticConfigurati
     context = createContext();
     globalPolicyFile = context.getPolicyFile();
     dataDir = context.getDataDir();
-    assertTrue("Could not delete " + globalPolicyFile, context.deletePolicyFile());
     dataFile = new File(dataDir, SINGLE_TYPE_DATA_FILE_NAME);
     FileOutputStream to = new FileOutputStream(dataFile);
     Resources.copy(Resources.getResource(SINGLE_TYPE_DATA_FILE_NAME), to);

http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/4adf59a6/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestServerConfiguration.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestServerConfiguration.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestServerConfiguration.java
index 6bb39bc..c05bb4f 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestServerConfiguration.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestServerConfiguration.java
@@ -17,18 +17,9 @@
 
 package org.apache.sentry.tests.e2e.hive;
 
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.sql.Connection;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.Map;
-
+import com.google.common.base.Charsets;
+import com.google.common.collect.Maps;
+import com.google.common.io.Files;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.sentry.binding.hive.HiveAuthzBindingSessionHook;
@@ -40,9 +31,17 @@ import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
-import com.google.common.base.Charsets;
-import com.google.common.collect.Maps;
-import com.google.common.io.Files;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Map;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
 
 public class TestServerConfiguration extends AbstractTestWithHiveServer {
 
@@ -113,8 +112,6 @@ public class TestServerConfiguration extends AbstractTestWithHiveServer
{
   @Test
   public void testRemovalOfPolicyFile() throws Exception {
     context = createContext(properties);
-    File policyFile = context.getPolicyFile();
-    assertTrue("Could not delete " + policyFile, policyFile.delete());
     Connection connection = context.createConnection(ADMIN1);
     Statement statement = context.createStatement(connection);
     try {
@@ -132,7 +129,6 @@ public class TestServerConfiguration extends AbstractTestWithHiveServer
{
   public void testCorruptionOfPolicyFile() throws Exception {
     context = createContext(properties);
     File policyFile = context.getPolicyFile();
-    assertTrue("Could not delete " + policyFile, policyFile.delete());
     FileOutputStream out = new FileOutputStream(policyFile);
     out.write("this is not valid".getBytes(Charsets.UTF_8));
     out.close();

http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/4adf59a6/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/fs/AbstractDFS.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/fs/AbstractDFS.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/fs/AbstractDFS.java
index 145584d..8b1345d 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/fs/AbstractDFS.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/fs/AbstractDFS.java
@@ -16,16 +16,22 @@
  */
 package org.apache.sentry.tests.e2e.hive.fs;
 
-import java.io.IOException;
-
 import junit.framework.Assert;
-
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
 
 public abstract class AbstractDFS implements DFS{
+  private static final Logger LOGGER = LoggerFactory
+      .getLogger(AbstractDFS.class);
   protected static FileSystem fileSystem;
   protected static Path dfsBaseDir;
+  public Path sentryDir;
+
 
   @Override
   public FileSystem getFileSystem(){
@@ -53,6 +59,14 @@ public abstract class AbstractDFS implements DFS{
     return dfsBaseDir;
   }
 
+  @Override
+  public void writePolicyFile(File srcFile) throws IOException {
+    String policyFileName = srcFile.getName();
+    Path destPath = new Path(sentryDir, policyFileName);
+    fileSystem.copyFromLocalFile(true, true, new Path(srcFile.getAbsolutePath()), destPath);
+    LOGGER.info("Copied file to HDFS: " + destPath.toString());
+  }
+
   protected void cleanBaseDir() throws Exception {
    cleanDir(dfsBaseDir);
   }

http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/4adf59a6/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/fs/ClusterDFS.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/fs/ClusterDFS.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/fs/ClusterDFS.java
index d5db811..a16bb38 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/fs/ClusterDFS.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/fs/ClusterDFS.java
@@ -16,9 +16,6 @@
  */
 package org.apache.sentry.tests.e2e.hive.fs;
 
-import java.security.PrivilegedExceptionAction;
-import java.util.Random;
-
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -26,13 +23,15 @@ import org.apache.hadoop.security.UserGroupInformation;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.security.PrivilegedExceptionAction;
+import java.util.Random;
+
 public class ClusterDFS extends AbstractDFS{
   private static final Logger LOGGER = LoggerFactory
       .getLogger(ClusterDFS.class);
   public static final String TEST_USER = "sentry.e2etest.hive.test.user";
   private static final String testUser = System.getProperty(TEST_USER, "hive");
   private static final String KEYTAB_LOCATION = System.getProperty("sentry.e2e.hive.keytabs.location");
-  private Path sentryDir;
   private UserGroupInformation ugi;
 
   ClusterDFS() throws Exception{

http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/4adf59a6/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/fs/DFS.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/fs/DFS.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/fs/DFS.java
index 9e9bb27..872a084 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/fs/DFS.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/fs/DFS.java
@@ -19,10 +19,14 @@ package org.apache.sentry.tests.e2e.hive.fs;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 
+import java.io.File;
+import java.io.IOException;
+
 public interface DFS {
   public FileSystem getFileSystem();
   public void tearDown() throws Exception;
   public Path assertCreateDir(String dir) throws Exception;
   public Path getBaseDir();
   public void createBaseDir() throws Exception;
+  public void writePolicyFile(File srcFile) throws IOException;
 }

http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/4adf59a6/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/fs/MiniDFS.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/fs/MiniDFS.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/fs/MiniDFS.java
index de684a9..184c066 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/fs/MiniDFS.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/fs/MiniDFS.java
@@ -16,14 +16,14 @@
  */
 package org.apache.sentry.tests.e2e.hive.fs;
 
-import java.io.File;
-
 import junit.framework.Assert;
-
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 
+import java.io.File;
+import java.util.concurrent.TimeoutException;
+
 public class MiniDFS extends AbstractDFS {
   private static MiniDFSCluster dfsCluster;
 
@@ -33,6 +33,8 @@ public class MiniDFS extends AbstractDFS {
     conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, dfsDir.getPath());
     dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
     fileSystem = dfsCluster.getFileSystem();
+    String policyDir = System.getProperty("sentry.e2etest.hive.policy.location", "/user/hive/sentry");
+    sentryDir = super.assertCreateDfsDir(new Path(fileSystem.getUri() + policyDir));
     dfsBaseDir = assertCreateDfsDir(new Path(new Path(fileSystem.getUri()), "/base"));
   }
 
@@ -51,4 +53,4 @@ public class MiniDFS extends AbstractDFS {
     }
     return dir;
   }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/4adf59a6/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/HiveServerFactory.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/HiveServerFactory.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/HiveServerFactory.java
index f00efdb..8437845 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/HiveServerFactory.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/HiveServerFactory.java
@@ -16,14 +16,8 @@
  */
 package org.apache.sentry.tests.e2e.hive.hiveserver;
 
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.net.ServerSocket;
-import java.net.URL;
-import java.util.Map;
-
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.io.Resources;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hive.conf.HiveConf;
@@ -35,8 +29,13 @@ import org.junit.Assert;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.io.Resources;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.ServerSocket;
+import java.net.URL;
+import java.util.Map;
 
 public class HiveServerFactory {
   private static final Logger LOGGER = LoggerFactory
@@ -70,7 +69,7 @@ public class HiveServerFactory {
   }
 
   public static HiveServer create(Map<String, String> properties,
-      File baseDir, File confDir, File logDir, File policyFile,
+      File baseDir, File confDir, File logDir, String policyFile,
       FileSystem fileSystem)
           throws Exception {
     String type = properties.get(HIVESERVER2_TYPE);
@@ -85,16 +84,8 @@ public class HiveServerFactory {
   }
   private static HiveServer create(HiveServer2Type type,
       Map<String, String> properties, File baseDir, File confDir,
-      File logDir, File policyFile, FileSystem fileSystem) throws Exception {
+      File logDir, String policyFile, FileSystem fileSystem) throws Exception {
 
-    if(policyFile.exists()) {
-      LOGGER.info("Policy file " + policyFile + " exists");
-    } else {
-      LOGGER.info("Creating policy file " + policyFile);
-      FileOutputStream to = new FileOutputStream(policyFile);
-      Resources.copy(Resources.getResource(AUTHZ_PROVIDER_FILENAME), to);
-      to.close();
-    }
     if(type.equals(HiveServer2Type.UnmanagedHiveServer2)){
       LOGGER.info("Creating UnmanagedHiveServer");
       return new UnmanagedHiveServer();
@@ -119,7 +110,8 @@ public class HiveServerFactory {
       properties.put(ACCESS_TESTING_MODE, "true");
     }
     if(!properties.containsKey(AUTHZ_PROVIDER_RESOURCE)) {
-      properties.put(AUTHZ_PROVIDER_RESOURCE, policyFile.getPath());
+      LOGGER.info("Policy File location: " + policyFile);
+      properties.put(AUTHZ_PROVIDER_RESOURCE, policyFile);
     }
     if(!properties.containsKey(AUTHZ_PROVIDER)) {
       properties.put(AUTHZ_PROVIDER, LocalGroupResourceAuthorizationProvider.class.getName());


Mime
View raw message