sentry-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cohei...@apache.org
Subject [7/9] sentry git commit: SENTRY-2048: Bump Hive version to 2.3.2 (Sergio Pena, reviewed by kalyan kumar kalvagadda, Na Li)
Date Mon, 20 Nov 2017 14:47:28 GMT
SENTRY-2048: Bump Hive version to 2.3.2 (Sergio Pena, reviewed by kalyan kumar kalvagadda,
Na Li)


Project: http://git-wip-us.apache.org/repos/asf/sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/5f64fe9f
Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/5f64fe9f
Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/5f64fe9f

Branch: refs/heads/akolb-cli
Commit: 5f64fe9f34fb857ccca5a5b09b9ff46300c98674
Parents: a355979
Author: Sergio Pena <sergio.pena@cloudera.com>
Authored: Sun Nov 19 10:47:09 2017 -0600
Committer: Sergio Pena <sergio.pena@cloudera.com>
Committed: Sun Nov 19 10:47:09 2017 -0600

----------------------------------------------------------------------
 pom.xml                                         |  2 +-
 .../json/SentryJSONMessageDeserializer.java     | 79 +++++++++++++++++++-
 .../json/SentryJSONMessageFactory.java          | 48 ++++++++++++
 .../hive/ql/exec/SentryFilterDDLTask.java       |  9 ++-
 .../hive/ql/exec/SentryGrantRevokeTask.java     |  7 +-
 .../binding/hive/HiveAuthzBindingHook.java      |  2 +-
 .../hive/authz/DefaultSentryValidator.java      | 17 +++++
 .../hive/authz/HiveAuthzBindingHookBase.java    | 10 ---
 .../hive/authz/SentryHiveAuthorizerImpl.java    | 13 ++++
 .../metastore/SentryHiveMetaStoreClient.java    |  2 +-
 .../TestSentryHiveAuthorizationTaskFactory.java | 16 ++--
 .../e2e/hdfs/TestHDFSIntegrationAdvanced.java   |  9 ++-
 .../tests/e2e/hdfs/TestHDFSIntegrationBase.java |  4 +
 .../e2e/hive/TestPrivilegesAtColumnScope.java   | 31 ++++----
 .../e2e/hive/hiveserver/HiveServerFactory.java  | 10 +++
 ...actMetastoreTestWithStaticConfiguration.java |  3 +-
 .../e2e/metastore/TestMetastoreEndToEnd.java    |  6 +-
 sentry-tests/sentry-tests-solr/pom.xml          |  8 ++
 18 files changed, 230 insertions(+), 46 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 6487a26..d863627 100644
--- a/pom.xml
+++ b/pom.xml
@@ -72,7 +72,7 @@ limitations under the License.
     <guava.version>14.0.1</guava.version>
     <hadoop.version>2.7.2</hadoop.version>
     <hamcrest.version>1.3</hamcrest.version>
-    <hive.version>2.0.0</hive.version>
+    <hive.version>2.3.2</hive.version>
     <jackson.version>1.8.8</jackson.version>
     <jdo-api.version>3.0.1</jdo-api.version>
     <jetty.version>9.3.21.v20170918</jetty.version>

http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java
b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java
index d11b261..929ac8c 100644
--- a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java
+++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java
@@ -19,6 +19,11 @@
 package org.apache.sentry.binding.metastore.messaging.json;
 
 import org.apache.hive.hcatalog.messaging.*;
+import org.apache.hive.hcatalog.messaging.json.JSONAlterIndexMessage;
+import org.apache.hive.hcatalog.messaging.json.JSONCreateFunctionMessage;
+import org.apache.hive.hcatalog.messaging.json.JSONCreateIndexMessage;
+import org.apache.hive.hcatalog.messaging.json.JSONDropFunctionMessage;
+import org.apache.hive.hcatalog.messaging.json.JSONDropIndexMessage;
 import org.apache.hive.hcatalog.messaging.json.JSONInsertMessage;
 import org.codehaus.jackson.map.DeserializationConfig;
 import org.codehaus.jackson.map.ObjectMapper;
@@ -130,13 +135,81 @@ public class SentryJSONMessageDeserializer extends MessageDeserializer
{
   }
 
   /**
+   * Method to de-serialize CreateFunctionMessage instance.
+   */
+  @Override
+  public CreateFunctionMessage getCreateFunctionMessage(String messageBody) {
+    // Sentry does not need this message, but it needs to be implemented so that Hive can
+    // complete the notification log for such event.
+    try {
+      return mapper.readValue(messageBody, JSONCreateFunctionMessage.class);
+    } catch (Exception e) {
+      throw new IllegalArgumentException("Could not construct JSONCreateFunctionMessage:
", e);
+    }
+  }
+
+  /**
+   * Method to de-serialize DropFunctionMessage instance.
+   */
+  @Override
+  public DropFunctionMessage getDropFunctionMessage(String messageBody) {
+    // Sentry does not need this message, but it needs to be implemented so that Hive can
+    // complete the notification log for such event.
+    try {
+      return mapper.readValue(messageBody, JSONDropFunctionMessage.class);
+    } catch (Exception e) {
+      throw new IllegalArgumentException("Could not construct JSONDropDatabaseMessage: ",
e);
+    }
+  }
+
+  /**
+   * Method to de-serialize CreateIndexMessage instance.                                
                                                                                         
                        +   */
+  @Override
+  public CreateIndexMessage getCreateIndexMessage(String messageBody) {
+    // Sentry does not need this message, but it needs to be implemented so that Hive can
+    // complete the notification log for such event.
+    try {
+      return mapper.readValue(messageBody, JSONCreateIndexMessage.class);
+    } catch (Exception e) {
+      throw new IllegalArgumentException("Could not construct JSONCreateIndexMessage: ",
e);
+    }
+  }
+
+  /**
+   * Method to de-serialize DropIndexMessage instance.
+   */
+  @Override
+  public DropIndexMessage getDropIndexMessage(String messageBody) {
+    // Sentry does not need this message, but it needs to be implemented so that Hive can
+    // complete the notification log for such event.
+    try {
+      return mapper.readValue(messageBody, JSONDropIndexMessage.class);
+    } catch (Exception e) {
+      throw new IllegalArgumentException("Could not construct JSONDropIndexMessage: ", e);
+    }
+  }
+
+  /**
+   * Method to de-serialize AlterIndexMessage instance.
+   */
+  @Override
+  public AlterIndexMessage getAlterIndexMessage(String messageBody) {
+    // Sentry does not need this message, but it needs to be implemented so that Hive can
+    // complete the notification log for such event.
+    try {
+      return mapper.readValue(messageBody, JSONAlterIndexMessage.class);
+    } catch (Exception e) {
+      throw new IllegalArgumentException("Could not construct JSONAlterIndexMessage: ", e);
+    }
+  }
+
+  /**
    * Method to de-serialize JSONInsertMessage instance.
    */
   @Override
   public InsertMessage getInsertMessage(String messageBody) {
-    // Sentry would be not be interested in InsertMessage as these are generated when is
data is
-    // added inserted. This method is implemented for completeness. This is reason why, new
sentry
-    // JSON class is not defined for InsertMessage.
+    // Sentry does not need this message, but it needs to be implemented so that Hive can
+    // complete the notification log for such event.
     try {
       return mapper.readValue(messageBody, JSONInsertMessage.class);
     } catch (Exception e) {

http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java
b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java
index b531976..0af02d1 100644
--- a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java
+++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java
@@ -22,9 +22,16 @@ import com.google.common.collect.Lists;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.Function;
+import org.apache.hadoop.hive.metastore.api.Index;
 import org.apache.hadoop.hive.metastore.api.Partition;
 import org.apache.hadoop.hive.metastore.api.Table;
 import org.apache.hive.hcatalog.messaging.*;
+import org.apache.hive.hcatalog.messaging.json.JSONAlterIndexMessage;
+import org.apache.hive.hcatalog.messaging.json.JSONCreateFunctionMessage;
+import org.apache.hive.hcatalog.messaging.json.JSONCreateIndexMessage;
+import org.apache.hive.hcatalog.messaging.json.JSONDropFunctionMessage;
+import org.apache.hive.hcatalog.messaging.json.JSONDropIndexMessage;
 import org.apache.hive.hcatalog.messaging.json.JSONInsertMessage;
 
 import java.util.*;
@@ -108,6 +115,47 @@ public class SentryJSONMessageFactory extends MessageFactory {
   }
 
   @Override
+  public CreateFunctionMessage buildCreateFunctionMessage(Function function) {
+    // Sentry would be not be interested in CreateFunctionMessage as these are generated
when is data is
+    // added inserted. This method is implemented for completeness. This is reason why, new
sentry
+    // JSON class is not defined for CreateFunctionMessage
+    return new JSONCreateFunctionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, function,
now());
+  }
+
+  @Override
+  public DropFunctionMessage buildDropFunctionMessage(Function function) {
+    // Sentry would be not be interested in DropFunctionMessage as these are generated when
is data is
+    // added inserted. This method is implemented for completeness. This is reason why, new
sentry
+    // JSON class is not defined for DropFunctionMessage
+    return new JSONDropFunctionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, function,
now());
+
+  }
+
+  @Override
+  public CreateIndexMessage buildCreateIndexMessage(Index index) {
+    // Sentry would be not be interested in CreateIndexMessage as these are generated when
is data is
+    // added inserted. This method is implemented for completeness. This is reason why, new
sentry
+    // JSON class is not defined for CreateIndexMessage
+    return new JSONCreateIndexMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, index, now());
+  }
+
+  @Override
+  public DropIndexMessage buildDropIndexMessage(Index index) {
+    // Sentry would be not be interested in DropIndexMessage as these are generated when
is data is
+    // added inserted. This method is implemented for completeness. This is reason why, new
sentry
+    // JSON class is not defined for DropIndexMessage
+    return new JSONDropIndexMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, index, now());
+  }
+
+  @Override
+  public AlterIndexMessage buildAlterIndexMessage(Index before, Index after) {
+    // Sentry would be not be interested in AlterIndexMessage as these are generated when
is data is
+    // added inserted. This method is implemented for completeness. This is reason why, new
sentry
+    // JSON class is not defined for AlterIndexMessage
+    return new JSONAlterIndexMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, before, after,
now());
+  }
+
+  @Override
   public InsertMessage buildInsertMessage(String db, String table, Map<String,String>
partKeyVals,
       List<String> files) {
     // Sentry would be not be interested in InsertMessage as these are generated when is
data is

http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-binding/sentry-binding-hive/src/main/java/org/apache/hadoop/hive/ql/exec/SentryFilterDDLTask.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/hadoop/hive/ql/exec/SentryFilterDDLTask.java
b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/hadoop/hive/ql/exec/SentryFilterDDLTask.java
index 672acb6..efe113e 100644
--- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/hadoop/hive/ql/exec/SentryFilterDDLTask.java
+++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/hadoop/hive/ql/exec/SentryFilterDDLTask.java
@@ -143,9 +143,6 @@ public class SentryFilterDDLTask extends DDLTask {
     parentTasks = ddlTask.getParentTasks();
     backupTask = ddlTask.getBackupTask();
     backupChildrenTasks = ddlTask.getBackupChildrenTasks();
-    started = ddlTask.started();
-    isdone = ddlTask.done();
-    queued = ddlTask.getQueued();
     id = ddlTask.getId();
     taskCounters = ddlTask.getCounters();
     feedSubscribers = ddlTask.getFeedSubscribers();
@@ -157,5 +154,11 @@ public class SentryFilterDDLTask extends DDLTask {
     setException(ddlTask.getException());
     console = ddlTask.console;
     setFetchSource(ddlTask.isFetchSource());
+    taskHandle = ddlTask.getTaskHandle();
+    conf = ddlTask.conf;
+    queryState = ddlTask.queryState;
+    driverContext = ddlTask.getDriverContext();
+    clonedConf = ddlTask.clonedConf;
+    queryDisplay = ddlTask.queryDisplay;
   }
 }

http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-binding/sentry-binding-hive/src/main/java/org/apache/hadoop/hive/ql/exec/SentryGrantRevokeTask.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/hadoop/hive/ql/exec/SentryGrantRevokeTask.java
b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/hadoop/hive/ql/exec/SentryGrantRevokeTask.java
index 61052cf..21a6abf 100644
--- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/hadoop/hive/ql/exec/SentryGrantRevokeTask.java
+++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/hadoop/hive/ql/exec/SentryGrantRevokeTask.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.hive.metastore.api.PrincipalType;
 import org.apache.hadoop.hive.ql.CompilationOpContext;
 import org.apache.hadoop.hive.ql.DriverContext;
 import org.apache.hadoop.hive.ql.QueryPlan;
+import org.apache.hadoop.hive.ql.QueryState;
 import org.apache.hadoop.hive.ql.hooks.ReadEntity;
 import org.apache.hadoop.hive.ql.hooks.WriteEntity;
 import org.apache.hadoop.hive.ql.metadata.AuthorizationException;
@@ -109,11 +110,11 @@ public class SentryGrantRevokeTask extends Task<DDLWork> implements
Serializable
   private HiveOperation stmtOperation;
 
   @Override
-  public void initialize(HiveConf conf, QueryPlan queryPlan, DriverContext ctx,
+  public void initialize(QueryState queryState, QueryPlan queryPlan, DriverContext ctx,
       CompilationOpContext opContext) {
     // CompilationOpContext is an unused parameter on the initialize() method.
-    super.initialize(conf, queryPlan, driverContext, null);
-    this.conf = conf;
+    super.initialize(queryState, queryPlan, driverContext, null);
+    this.conf = queryState.getConf();
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHook.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHook.java
b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHook.java
index 802bf9c..e4620ea 100644
--- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHook.java
+++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHook.java
@@ -291,7 +291,7 @@ public class HiveAuthzBindingHook extends HiveAuthzBindingHookBase {
   @Override
   public void postAnalyze(HiveSemanticAnalyzerHookContext context,
       List<Task<? extends Serializable>> rootTasks) throws SemanticException
{
-    HiveOperation stmtOperation = getCurrentHiveStmtOp();
+    HiveOperation stmtOperation = context.getHiveOperation();
     HiveAuthzPrivileges stmtAuthObject;
 
     stmtAuthObject = HiveAuthzPrivilegesMap.getHiveAuthzPrivileges(stmtOperation);

http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/DefaultSentryValidator.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/DefaultSentryValidator.java
b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/DefaultSentryValidator.java
index d1f071e..319a1be 100644
--- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/DefaultSentryValidator.java
+++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/DefaultSentryValidator.java
@@ -27,6 +27,7 @@ import java.util.List;
 import java.util.Set;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.metadata.AuthorizationException;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
 import org.apache.hadoop.hive.ql.plan.HiveOperation;
 import org.apache.hadoop.hive.ql.security.HiveAuthenticationProvider;
 import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAccessControlException;
@@ -375,6 +376,22 @@ public class DefaultSentryValidator extends SentryHiveAuthorizationValidator
{
     return listObjs;
   }
 
+  @Override
+  public List<HivePrivilegeObject> applyRowFilterAndColumnMasking(HiveAuthzContext
hiveAuthzContext,
+      List<HivePrivilegeObject> list) throws SemanticException {
+    // Sentry does not support this feature yet. Returning null is enough to let Hive
+    // that no row filtering nor column masking will be applied.
+    return null;
+  }
+
+  @Override
+  public boolean needTransform() {
+    // Hive uses this value to know whether a Hive query must be transformed if row filtering
+    // or column masking is applied. Sentry does not support such feature yet, so returning
+    // false is enough to let Hive know that the query is not required to be transformed.
+    return false;
+  }
+
   private List<HivePrivilegeObject> filterShowTables(List<HivePrivilegeObject>
listObjs,
       String userName, HiveAuthzBinding hiveAuthzBinding) {
     List<HivePrivilegeObject> filteredResult = new ArrayList<HivePrivilegeObject>();

http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/HiveAuthzBindingHookBase.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/HiveAuthzBindingHookBase.java
b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/HiveAuthzBindingHookBase.java
index 2e299a9..9c60c22 100644
--- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/HiveAuthzBindingHookBase.java
+++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/HiveAuthzBindingHookBase.java
@@ -886,18 +886,8 @@ public abstract class HiveAuthzBindingHookBase extends AbstractSemanticAnalyzerH
     }
   }
 
-  protected HiveOperation getCurrentHiveStmtOp() {
-    SessionState sessState = SessionState.get();
-    if (sessState == null) {
-      // TODO: Warn
-      return null;
-    }
-    return sessState.getHiveOperation();
-  }
-
   protected Subject getCurrentSubject(HiveSemanticAnalyzerHookContext context) {
     // Extract the username from the hook context
     return new Subject(context.getUserName());
   }
-
 }

http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/SentryHiveAuthorizerImpl.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/SentryHiveAuthorizerImpl.java
b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/SentryHiveAuthorizerImpl.java
index 1596bce..86ff0cc 100644
--- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/SentryHiveAuthorizerImpl.java
+++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/SentryHiveAuthorizerImpl.java
@@ -20,11 +20,13 @@ import java.util.List;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.exec.SentryHivePrivilegeObjectDesc;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
 import org.apache.hadoop.hive.ql.plan.PrivilegeObjectDesc;
 import org.apache.hadoop.hive.ql.security.authorization.DefaultHiveAuthorizationTranslator;
 import org.apache.hadoop.hive.ql.security.authorization.plugin.AbstractHiveAuthorizer;
 import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAccessControlException;
 import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthorizationTranslator;
+import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthorizer.VERSION;
 import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzContext;
 import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzPluginException;
 import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveOperationType;
@@ -165,6 +167,17 @@ public class SentryHiveAuthorizerImpl extends AbstractHiveAuthorizer
{
     return hiveTranslator;
   }
 
+  @Override
+  public List<HivePrivilegeObject> applyRowFilterAndColumnMasking(HiveAuthzContext
hiveAuthzContext,
+      List<HivePrivilegeObject> list) throws SemanticException {
+    return null;
+  }
+
+  @Override
+  public boolean needTransform() {
+    return false;
+  }
+
   protected static HivePrivilegeObjectType getPrivObjectType(
       SentryHivePrivilegeObjectDesc privSubjectDesc) {
     if (privSubjectDesc.getObject() == null) {

http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryHiveMetaStoreClient.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryHiveMetaStoreClient.java
b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryHiveMetaStoreClient.java
index cfc5c04..e30a860 100644
--- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryHiveMetaStoreClient.java
+++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryHiveMetaStoreClient.java
@@ -49,7 +49,7 @@ public class SentryHiveMetaStoreClient extends HiveMetaStoreClient implements
 
   public SentryHiveMetaStoreClient(HiveConf conf, HiveMetaHookLoader hookLoader)
       throws MetaException {
-    super(conf, hookLoader);
+    super(conf, hookLoader, true);
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-binding/sentry-binding-hive/src/test/java/org/apache/sentry/binding/hive/TestSentryHiveAuthorizationTaskFactory.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive/src/test/java/org/apache/sentry/binding/hive/TestSentryHiveAuthorizationTaskFactory.java
b/sentry-binding/sentry-binding-hive/src/test/java/org/apache/sentry/binding/hive/TestSentryHiveAuthorizationTaskFactory.java
index de073ed..c75f57d 100644
--- a/sentry-binding/sentry-binding-hive/src/test/java/org/apache/sentry/binding/hive/TestSentryHiveAuthorizationTaskFactory.java
+++ b/sentry-binding/sentry-binding-hive/src/test/java/org/apache/sentry/binding/hive/TestSentryHiveAuthorizationTaskFactory.java
@@ -22,6 +22,7 @@ import java.io.Serializable;
 import java.util.HashMap;
 import java.util.List;
 
+import org.apache.hadoop.hive.ql.QueryState;
 import org.junit.Assert;
 
 import org.apache.commons.io.FileUtils;
@@ -37,7 +38,6 @@ import org.apache.hadoop.hive.ql.metadata.Partition;
 import org.apache.hadoop.hive.ql.metadata.Table;
 import org.apache.hadoop.hive.ql.parse.ASTNode;
 import org.apache.hadoop.hive.ql.parse.DDLSemanticAnalyzer;
-import org.apache.hadoop.hive.ql.parse.ParseDriver;
 import org.apache.hadoop.hive.ql.parse.ParseUtils;
 import org.apache.hadoop.hive.ql.parse.SemanticException;
 import org.apache.hadoop.hive.ql.plan.DDLWork;
@@ -70,7 +70,6 @@ public class TestSentryHiveAuthorizationTaskFactory {
   private static final String SERVER = "server1";
 
 
-  private ParseDriver parseDriver;
   private DDLSemanticAnalyzer analyzer;
   private HiveConf conf;
   private Context context;
@@ -91,12 +90,19 @@ public class TestSentryHiveAuthorizationTaskFactory {
     conf.setVar(ConfVars.HIVE_AUTHORIZATION_TASK_FACTORY,
         SentryHiveAuthorizationTaskFactoryImpl.class.getName());
 
+    // This configuration avoids starting the HS2 WebUI which was causes test failures when
+    // HS2 is configured for concurrency
+    conf.setBoolVar(HiveConf.ConfVars.HIVE_IN_TEST, true);
+
+    // This configuration avoids that the HMS fails if the Metastore schema has not version
+    // information. For some reason, HMS does not set a version initially on our tests.
+    conf.setBoolVar(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION, false);
+
     db = Mockito.mock(Hive.class);
     table = new Table(DB, TABLE);
     partition = new Partition(table);
     context = new Context(conf);
-    parseDriver = new ParseDriver();
-    analyzer = new DDLSemanticAnalyzer(conf, db);
+    analyzer = new DDLSemanticAnalyzer(new QueryState(conf), db);
     SessionState.start(conf);
     Mockito.when(db.getTable(TABLE, false)).thenReturn(table);
     Mockito.when(db.getPartition(table, new HashMap<String, String>(), false))
@@ -487,7 +493,7 @@ public class TestSentryHiveAuthorizationTaskFactory {
   }
 
   private ASTNode parse(String command) throws Exception {
-    return ParseUtils.findRootNonNullToken(parseDriver.parse(command));
+    return ParseUtils.parse(command);
   }
 
   private DDLWork analyze(ASTNode ast) throws Exception {

http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationAdvanced.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationAdvanced.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationAdvanced.java
index 95bbaeb..33ace57 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationAdvanced.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationAdvanced.java
@@ -791,7 +791,14 @@ public class TestHDFSIntegrationAdvanced extends TestHDFSIntegrationBase
{
      // Alter table tab1 to be tbCopy which is at scheme-less location.
      // And the corresponding path will be updated to sentry server.
      hmsClient.alter_table(dbName, "tab1", tbCopy);
-     Assert.assertEquals(hmsClient.getTable(dbName, tblName).getSd().getLocation(), "/tmp/external");
+
+     // Remove the checking for the location of the table. The HMS will never return scheme-less
+     // URI locations anymore. However, if any NPE being triggered in future because of any
changes,
+     // the test case will cover it and capture it.
+     // i.e. hdfs://<localhost>/tmp/external (location with scheme)
+     //      /tmp/external                   (location without scheme)
+     // Assert.assertEquals("/tmp/external", hmsClient.getTable(dbName, tblName).getSd().getLocation());
+
      verifyOnPath("/tmp/external", FsAction.ALL, StaticUserGroup.HIVE, true);
 
      stmt.close();

http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java
index 7495fc7..25a678b 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java
@@ -575,6 +575,10 @@ public abstract class TestHDFSIntegrationBase {
         String hadoopTempDir = System.getProperty("java.io.tmpdir") + File.separator + "hadoop-tmp";
         hiveConf.set("hadoop.tmp.dir", hadoopTempDir);
 
+        // This configuration will avoid that the HMS fails if the metastore schema has not
version
+        // information. For some reason, HMS does not set a version initially on our tests.
+        hiveConf.set(ConfVars.METASTORE_SCHEMA_VERIFICATION.varname, "false");
+
         // Sets hive.metastore.authorization.storage.checks to true, so that
         // disallow the operations such as drop-partition if the user in question
         // doesn't have permissions to delete the corresponding directory

http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPrivilegesAtColumnScope.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPrivilegesAtColumnScope.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPrivilegesAtColumnScope.java
index c2fee2a..9d820a8 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPrivilegesAtColumnScope.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPrivilegesAtColumnScope.java
@@ -205,12 +205,13 @@ public class TestPrivilegesAtColumnScope extends AbstractTestWithStaticConfigura
       context.verifyAuthzException(e);
     }
 
-    // negative test: test user can't query column of view
+    // positive test: test user can query column of view
+    // Hive 2 adds a new column view authorization as part of the Column Pruning feature
+    // See Hive ColumnPrunerSelectProc.process() on how view authorization is handled
     try {
       statement.execute("SELECT COUNT(A) FROM VIEW_1");
-      Assert.fail("Expected SQL exception");
     } catch (SQLException e) {
-      context.verifyAuthzException(e);
+      Assert.fail("Exception not expected.");
     }
     // negative test: test user can't query column of view
     try {
@@ -255,18 +256,18 @@ public class TestPrivilegesAtColumnScope extends AbstractTestWithStaticConfigura
       context.verifyAuthzException(e);
     }
 
-    // negative test: test user can't query view
+    // positive test: test user can query column of view
+    // Hive 2 adds a new column view authorization as part of the Column Pruning feature
+    // See Hive ColumnPrunerSelectProc.process() on how view authorization is handled
     try {
       statement.execute("SELECT COUNT(A) FROM VIEW_1");
-      Assert.fail("Expected SQL exception");
     } catch (SQLException e) {
-      context.verifyAuthzException(e);
+      Assert.fail("Exception not expected.");
     }
     try {
       statement.execute("SELECT COUNT(B) FROM VIEW_1");
-      Assert.fail("Expected SQL exception");
     } catch (SQLException e) {
-      context.verifyAuthzException(e);
+      Assert.fail("Exception not expected.");
     }
 
     // negative test: test user can't create a new view
@@ -378,12 +379,13 @@ public class TestPrivilegesAtColumnScope extends AbstractTestWithStaticConfigura
     Statement statement = context.createStatement(connection);
     statement.execute("USE DB_1");
 
-    // test user can't execute query VIEW_1 JOIN VIEW_2
+    // test user can execute query VIEW_1 JOIN VIEW_2
+    // Hive 2 adds a new column view authorization as part of the Column Pruning feature
+    // See Hive ColumnPrunerSelectProc.process() on how view authorization is handled
     try {
       statement.execute("SELECT COUNT(*) FROM VIEW_1 V1 JOIN VIEW_2 V2 ON (V1.B = V2.B)");
-      Assert.fail("Expected SQL Exception");
     } catch (SQLException e) {
-      context.verifyAuthzException(e);
+      Assert.fail("Exception not expected");
     }
 
     // test user can't execute query VIEW_1 JOIN TAB_2
@@ -438,12 +440,13 @@ public class TestPrivilegesAtColumnScope extends AbstractTestWithStaticConfigura
       context.verifyAuthzException(e);
     }
 
-    // test user can't execute query VIEW_1 JOIN VIEW_2
+    // test user can execute query VIEW_1 JOIN VIEW_2
+    // Hive 2 adds a new column view authorization as part of the Column Pruning feature
+    // See Hive ColumnPrunerSelectProc.process() on how view authorization is handled
     try {
       statement.execute("SELECT COUNT(*) FROM VIEW_1 V1 JOIN VIEW_2 V2 ON (V1.B = V2.B)");
-      Assert.fail("Expected SQL Exception");
     } catch (SQLException e) {
-      context.verifyAuthzException(e);
+      Assert.fail("Exception not expected");
     }
 
     // test user can't execute query TAB_1 JOIN TAB_2

http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/HiveServerFactory.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/HiveServerFactory.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/HiveServerFactory.java
index d5eb137..7d41348 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/HiveServerFactory.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/HiveServerFactory.java
@@ -206,6 +206,16 @@ public class HiveServerFactory {
     String hadoopTempDir = System.getProperty("java.io.tmpdir") + File.separator + "hadoop-tmp";
     properties.put("hadoop.tmp.dir", hadoopTempDir);
 
+    // This configuration will avoid that the HMS fails if the metastore schema has not version
+    // information. For some reason, HMS does not set a version initially on our tests.
+    properties.put(ConfVars.METASTORE_SCHEMA_VERIFICATION.varname, "false");
+
+    // Disable join cartesian checks to allow Sentry tests to pass
+    properties.put(ConfVars.HIVE_STRICT_CHECKS_CARTESIAN.varname, "false");
+
+    // Disable capability checks (these checks do not work when Hive is in testing mode)
+    properties.put(ConfVars.METASTORE_CAPABILITY_CHECK.varname, "false");
+
     if (!properties.containsKey(METASTORE_BYPASS)) {
       properties.put(METASTORE_BYPASS, "hive,impala," + System.getProperty("user.name", ""));
     } else {

http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/AbstractMetastoreTestWithStaticConfiguration.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/AbstractMetastoreTestWithStaticConfiguration.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/AbstractMetastoreTestWithStaticConfiguration.java
index d0139ad..f14cbb6 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/AbstractMetastoreTestWithStaticConfiguration.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/AbstractMetastoreTestWithStaticConfiguration.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.hive.cli.CliSessionState;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
 import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.metastore.api.Order;
 import org.apache.hadoop.hive.metastore.api.Partition;
@@ -138,7 +139,7 @@ public abstract class AbstractMetastoreTestWithStaticConfiguration extends
 
   public void alterPartitionWithLocation(HiveMetaStoreClient client, Partition partition,
String location) throws Exception {
     partition.getSd().setLocation(location);
-    client.alter_partition(partition.getDbName(), partition.getTableName(), partition);
+    client.alter_partition(partition.getDbName(), partition.getTableName(), partition, new
EnvironmentContext());
   }
 
   public void renamePartition(HiveMetaStoreClient client, Partition partition, Partition
newPartition) throws Exception {

http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestMetastoreEndToEnd.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestMetastoreEndToEnd.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestMetastoreEndToEnd.java
index 7f53527..f8f304f 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestMetastoreEndToEnd.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestMetastoreEndToEnd.java
@@ -507,14 +507,14 @@ public class  TestMetastoreEndToEnd extends
     client = context.getMetaStoreClient(USER1_1);
     Partition newPartition = client.getPartition(dbName, tabName1, partVals1);
     newPartition.getSd().setLocation(tabDir1);
-    client.alter_partition(dbName, tabName1, newPartition);
+    client.alter_partition(dbName, tabName1, newPartition, null);
     client.close();
 
     // user with Table and URI privileges should be able to alter partition set location
     client = context.getMetaStoreClient(USER2_1);
     newPartition = client.getPartition(dbName, tabName1, partVals2);
     newPartition.getSd().setLocation(tabDir1);
-    client.alter_partition(dbName, tabName1, newPartition);
+    client.alter_partition(dbName, tabName1, newPartition, null);
     client.close();
 
     policyFile.addRolesToGroup(USERGROUP3, db_all_role);
@@ -524,7 +524,7 @@ public class  TestMetastoreEndToEnd extends
     newPartition = client.getPartition(dbName, tabName1, partVals2);
     newPartition.getSd().setLocation(tabDir1);
     try {
-      client.alter_partition(dbName, tabName1, newPartition);
+      client.alter_partition(dbName, tabName1, newPartition, null);
       fail("alter partition with location should have failed");
     } catch (MetaException e) {
       Context.verifyMetastoreAuthException(e);

http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-tests/sentry-tests-solr/pom.xml
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-solr/pom.xml b/sentry-tests/sentry-tests-solr/pom.xml
index 723fa9d..9d400b2 100644
--- a/sentry-tests/sentry-tests-solr/pom.xml
+++ b/sentry-tests/sentry-tests-solr/pom.xml
@@ -110,6 +110,10 @@ limitations under the License.
           <groupId>org.apache.hbase</groupId>
           <artifactId>hbase-client</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>org.apache.hive</groupId>
+          <artifactId>hive-shims</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
     <dependency>
@@ -195,6 +199,10 @@ limitations under the License.
           <groupId>org.eclipse.jetty.aggregate</groupId>
           <artifactId>jetty-all</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>org.apache.hive</groupId>
+          <artifactId>hive-llap-server</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
     <dependency>


Mime
View raw message