sentry-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cohei...@apache.org
Subject [5/9] sentry git commit: SENTRY-2207 Refactor out Sentry CLI from sentry-provider-db into own module. Steve Moist, reviewed by Colm O hEigeartaigh.
Date Mon, 30 Apr 2018 16:35:41 GMT
http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-provider/sentry-provider-db/src/test/resources/indexer_config_import_tool.ini
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/test/resources/indexer_config_import_tool.ini b/sentry-provider/sentry-provider-db/src/test/resources/indexer_config_import_tool.ini
deleted file mode 100644
index c1bfe4b..0000000
--- a/sentry-provider/sentry-provider-db/src/test/resources/indexer_config_import_tool.ini
+++ /dev/null
@@ -1,29 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#  http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-[groups]
-corporal = corporal_role
-sergeant = corporal_role, sergeant_role
-general = corporal_role, sergeant_role, general_role
-commander_in_chief = corporal_role, sergeant_role, general_role, commander_in_chief_role
-
-[roles]
-corporal_role = indexer=info->action=read, \
-  indexer=info->action=write
-sergeant_role = indexer=info->action=write
-general_role = indexer=info->action=*
-commander_in_chief_role = indexer=*

http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-provider/sentry-provider-db/src/test/resources/indexer_invalid.ini
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/test/resources/indexer_invalid.ini b/sentry-provider/sentry-provider-db/src/test/resources/indexer_invalid.ini
deleted file mode 100644
index 03083a7..0000000
--- a/sentry-provider/sentry-provider-db/src/test/resources/indexer_invalid.ini
+++ /dev/null
@@ -1,21 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#  http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-[groups]
-
-[roles]
-

http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-provider/sentry-provider-db/src/test/resources/solr_case.ini
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/test/resources/solr_case.ini b/sentry-provider/sentry-provider-db/src/test/resources/solr_case.ini
deleted file mode 100644
index fbbebfc..0000000
--- a/sentry-provider/sentry-provider-db/src/test/resources/solr_case.ini
+++ /dev/null
@@ -1,26 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#  http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-[groups]
-groupa = RoLe1
-groupb = rOlE1
-groupc = ROLE2
-
-[roles]
-RoLe1 = collection=*
-rOlE1 = collection=*
-ROLE2 = collection=*

http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-provider/sentry-provider-db/src/test/resources/solr_config_import_tool.ini
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/test/resources/solr_config_import_tool.ini b/sentry-provider/sentry-provider-db/src/test/resources/solr_config_import_tool.ini
deleted file mode 100644
index da7df4c..0000000
--- a/sentry-provider/sentry-provider-db/src/test/resources/solr_config_import_tool.ini
+++ /dev/null
@@ -1,29 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#  http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-[groups]
-corporal = corporal_role
-sergeant = corporal_role, sergeant_role
-general = corporal_role, sergeant_role, general_role
-commander_in_chief = corporal_role, sergeant_role, general_role, commander_in_chief_role
-
-[roles]
-corporal_role = collection=info->action=query, \
-  collection=info->action=update
-sergeant_role = collection=info->action=update
-general_role = collection=info->action=*
-commander_in_chief_role = collection=*

http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-provider/sentry-provider-db/src/test/resources/solr_invalid.ini
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/test/resources/solr_invalid.ini b/sentry-provider/sentry-provider-db/src/test/resources/solr_invalid.ini
deleted file mode 100644
index 03083a7..0000000
--- a/sentry-provider/sentry-provider-db/src/test/resources/solr_invalid.ini
+++ /dev/null
@@ -1,21 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#  http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-[groups]
-
-[roles]
-

http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-tools/pom.xml
----------------------------------------------------------------------
diff --git a/sentry-tools/pom.xml b/sentry-tools/pom.xml
index 4d8fc89..b882c6f 100644
--- a/sentry-tools/pom.xml
+++ b/sentry-tools/pom.xml
@@ -26,6 +26,7 @@ limitations under the License.
     <modelVersion>4.0.0</modelVersion>
 
     <artifactId>sentry-tools</artifactId>
+    <version>2.1.0-SNAPSHOT</version>
 
     <dependencies>
         <dependency>
@@ -33,27 +34,56 @@ limitations under the License.
             <artifactId>sentry-binding-hive</artifactId>
         </dependency>
         <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-log4j12</artifactId>
+            <groupId>org.apache.sentry</groupId>
+            <artifactId>sentry-provider-db</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.sentry</groupId>
+            <artifactId>sentry-provider-db</artifactId>
+            <version>${project.version}</version>
+            <type>test-jar</type>
+            <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>commons-cli</groupId>
             <artifactId>commons-cli</artifactId>
         </dependency>
         <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-log4j12</artifactId>
+        </dependency>
+        <dependency>
             <groupId>com.budhash.cliche</groupId>
             <artifactId>cliche-shell</artifactId>
             <version>0.9.3</version>
         </dependency>
         <dependency>
-            <groupId>org.apache.sentry</groupId>
-            <artifactId>sentry-provider-db</artifactId>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-minikdc</artifactId>
+            <scope>test</scope>
         </dependency>
     </dependencies>
 
     <build>
-        <sourceDirectory>${basedir}/src/main/java</sourceDirectory>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <configuration>
+                    <reuseForks>false</reuseForks>
+                </configuration>
+            </plugin>
+        </plugins>
+        <testResources>
+            <testResource>
+                <directory>src/test/resources</directory>
+            </testResource>
+        </testResources>
     </build>
-
-
 </project>

http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-tools/src/main/java/org/apache/sentry/SentryMain.java
----------------------------------------------------------------------
diff --git a/sentry-tools/src/main/java/org/apache/sentry/SentryMain.java b/sentry-tools/src/main/java/org/apache/sentry/SentryMain.java
index e92155c..495d4cd 100644
--- a/sentry-tools/src/main/java/org/apache/sentry/SentryMain.java
+++ b/sentry-tools/src/main/java/org/apache/sentry/SentryMain.java
@@ -24,7 +24,7 @@ import org.apache.commons.cli.HelpFormatter;
 import org.apache.commons.cli.Options;
 import org.apache.log4j.PropertyConfigurator;
 import org.apache.sentry.binding.hive.authz.SentryConfigTool;
-import org.apache.sentry.provider.db.tools.SentrySchemaTool;
+import org.apache.sentry.cli.tools.SentrySchemaTool;
 import org.apache.sentry.service.thrift.SentryService;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-tools/src/main/java/org/apache/sentry/cli/tools/PermissionsMigrationToolCommon.java
----------------------------------------------------------------------
diff --git a/sentry-tools/src/main/java/org/apache/sentry/cli/tools/PermissionsMigrationToolCommon.java b/sentry-tools/src/main/java/org/apache/sentry/cli/tools/PermissionsMigrationToolCommon.java
new file mode 100644
index 0000000..8b5130c
--- /dev/null
+++ b/sentry-tools/src/main/java/org/apache/sentry/cli/tools/PermissionsMigrationToolCommon.java
@@ -0,0 +1,349 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sentry.cli.tools;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Optional;
+import java.util.Set;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.cli.Parser;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.sentry.core.common.ActiveRoleSet;
+import org.apache.sentry.core.common.utils.PolicyFileConstants;
+import org.apache.sentry.core.common.utils.PolicyFiles;
+import org.apache.sentry.core.common.utils.Version;
+import org.apache.sentry.policy.common.PrivilegeUtils;
+import org.apache.sentry.provider.common.ProviderBackendContext;
+import org.apache.sentry.provider.db.generic.service.thrift.SentryGenericServiceClient;
+import org.apache.sentry.provider.db.generic.service.thrift.SentryGenericServiceClientFactory;
+import org.apache.sentry.provider.db.generic.service.thrift.TSentryPrivilege;
+import org.apache.sentry.provider.db.generic.service.thrift.TSentryRole;
+import org.apache.sentry.provider.db.generic.tools.GenericPrivilegeConverter;
+import org.apache.sentry.provider.file.SimpleFileProviderBackend;
+import org.apache.shiro.config.Ini;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Sets;
+import com.google.common.collect.Table;
+
+/**
+ * This class provides basic framework required to migrate permissions between different Sentry
+ * versions. Individual components (e.g. SOLR, KAFKA) needs to override the this class
+ * to provide component specific migration functionality.
+ */
+public abstract class PermissionsMigrationToolCommon {
+  private static final Logger LOGGER = LoggerFactory.getLogger(PermissionsMigrationToolCommon.class);
+  public static final String SOLR_SERVICE_NAME = "sentry.service.client.solr.service.name";
+
+  private Version sourceVersion;
+  private Optional<String> confPath = Optional.empty();
+  private Optional<String> policyFile = Optional.empty();
+  private Optional<String> outputFile = Optional.empty();
+  private boolean dryRun = false;
+
+  /**
+   * @return version of Sentry for which the privileges need to be migrated.
+   */
+  public final Version getSourceVersion() {
+    return sourceVersion;
+  }
+
+  /**
+   * This method returns the name of the component for the migration purpose.
+   * @param conf The Sentry configuration
+   * @return the name of the component
+   */
+  protected abstract String getComponent(Configuration conf);
+
+
+  /**
+   * This method returns the name of the service name for the migration purpose.
+   *
+   * @param conf The Sentry configuration
+   * @return the name of the service
+   */
+  protected abstract String getServiceName(Configuration conf);
+
+  /**
+   * Migrate the privileges specified via <code>privileges</code>.
+   *
+   * @param privileges A collection of privileges to be migrated.
+   * @return A collection of migrated privileges
+   *         An empty collection if migration is not necessary for the specified privileges.
+   */
+  protected abstract Collection<String> transformPrivileges (Collection<String> privileges);
+
+  /**
+   *  parse arguments
+   * <pre>
+   *   -s,--source                   Sentry source version
+   *   -c,--sentry_conf <filepath>   sentry config file path
+   *   -p --policy_file <filepath>   sentry (source) policy file path
+   *   -o --output      <filepath>   sentry (target) policy file path
+   *   -d --dry_run                  provides the output the migration for inspection without
+   *                                 making any configuration changes.
+   *   -h,--help                     print usage
+   * </pre>
+   * @param args
+   */
+  protected boolean parseArgs(String [] args) {
+    Options options = new Options();
+
+    Option sourceVersionOpt = new Option("s", "source", true, "Source Sentry version");
+    sourceVersionOpt.setRequired(true);
+    options.addOption(sourceVersionOpt);
+
+    Option sentryConfPathOpt = new Option("c", "sentry_conf", true,
+        "sentry-site.xml file path (only required in case of Sentry service)");
+    sentryConfPathOpt.setRequired(false);
+    options.addOption(sentryConfPathOpt);
+
+    Option sentryPolicyFileOpt = new Option("p", "policy_file", true,
+        "sentry (source) policy file path (only in case of file based Sentry configuration)");
+    sentryPolicyFileOpt.setRequired(false);
+    options.addOption(sentryPolicyFileOpt);
+
+    Option sentryOutputFileOpt = new Option("o", "output", true,
+        "sentry (target) policy file path (only in case of file based Sentry configuration)");
+    sentryOutputFileOpt.setRequired(false);
+    options.addOption(sentryOutputFileOpt);
+
+    Option dryRunOpt = new Option("d", "dry_run", false,
+        "provides the output the migration for inspection without making actual configuration changes");
+    dryRunOpt.setRequired(false);
+    options.addOption(dryRunOpt);
+
+    // help option
+    Option helpOpt = new Option("h", "help", false, "Shell usage");
+    helpOpt.setRequired(false);
+    options.addOption(helpOpt);
+
+    // this Option is parsed first for help option
+    Options helpOptions = new Options();
+    helpOptions.addOption(helpOpt);
+
+    try {
+      Parser parser = new GnuParser();
+
+      // parse help option first
+      CommandLine cmd = parser.parse(helpOptions, args, true);
+      for (Option opt : cmd.getOptions()) {
+        if (opt.getOpt().equals("h")) {
+          // get the help option, print the usage and exit
+          usage(options);
+          return false;
+        }
+      }
+
+      // without help option
+      cmd = parser.parse(options, args);
+
+      String sourceVersionStr = null;
+
+      for (Option opt : cmd.getOptions()) {
+        if (opt.getOpt().equals("s")) {
+          sourceVersionStr = opt.getValue();
+        } else if (opt.getOpt().equals("c")) {
+          confPath = Optional.of(opt.getValue());
+        } else if (opt.getOpt().equals("p")) {
+          policyFile = Optional.of(opt.getValue());
+        }  else if (opt.getOpt().equals("o")) {
+          outputFile = Optional.of(opt.getValue());
+        }  else if (opt.getOpt().equals("d")) {
+          dryRun = true;
+        }
+      }
+
+      sourceVersion = Version.parse(sourceVersionStr);
+
+      if (!(confPath.isPresent() || policyFile.isPresent())) {
+        System.out.println("Please select either file-based Sentry configuration (-p and -o flags)"
+            + " or Sentry service (-c flag) for migration.");
+        usage(options);
+        return false;
+      }
+
+      if (confPath.isPresent() && (policyFile.isPresent() || outputFile.isPresent())) {
+        System.out.println("In order to migrate service based Sentry configuration,"
+            + " do not specify either -p or -o parameters");
+        usage(options);
+        return false;
+      }
+
+      if (!confPath.isPresent() && (policyFile.isPresent() ^ outputFile.isPresent())) {
+        System.out.println("In order to migrate file based Sentry configuration,"
+            + " please make sure to specify both -p and -o parameters.");
+        usage(options);
+        return false;
+      }
+
+    } catch (ParseException | java.text.ParseException pe) {
+      System.out.println(pe.getMessage());
+      usage(options);
+      return false;
+    }
+    return true;
+  }
+
+  // print usage
+  private void usage(Options sentryOptions) {
+    HelpFormatter formatter = new HelpFormatter();
+    formatter.printHelp("sentryMigrationTool", sentryOptions);
+  }
+
+  public void run() throws Exception {
+    if (policyFile.isPresent()) {
+      migratePolicyFile();
+    } else {
+      migrateSentryServiceConfig();
+    }
+  }
+
+  private void migrateSentryServiceConfig() throws Exception {
+    Configuration conf = getSentryConf();
+    String component = getComponent(conf);
+    String serviceName = getServiceName(conf);
+    GenericPrivilegeConverter converter = new GenericPrivilegeConverter(component, serviceName, false);
+
+    // instantiate a client for sentry service.  This sets the ugi, so must
+    // be done before getting the ugi below.
+    try(SentryGenericServiceClient client =
+                SentryGenericServiceClientFactory.create(getSentryConf())) {
+      UserGroupInformation ugi = UserGroupInformation.getLoginUser();
+      String requestorName = ugi.getShortUserName();
+
+      for (TSentryRole r : client.listAllRoles(requestorName, component)) {
+        for (TSentryPrivilege p : client.listAllPrivilegesByRoleName(requestorName,
+            r.getRoleName(), component, serviceName)) {
+
+          String privilegeStr = converter.toString(p);
+          Collection<String> privileges = Collections.singleton(privilegeStr);
+          Collection<String> migrated = transformPrivileges(privileges);
+          if (!migrated.isEmpty()) {
+            LOGGER.info("{} For role {} migrating privileges from {} to {}", getDryRunMessage(), r.getRoleName(),
+                privileges, migrated);
+
+            /*
+             * Note that it is not possible to provide transactional (all-or-nothing) behavior for these configuration
+             * changes since the Sentry client/server protocol does not support. e.g. under certain failure conditions
+             * like crash of Sentry server or network disconnect between client/server, it is possible that the migration
+             * can not complete but can also not be rolled back. Hence this migration tool relies on the fact that privilege
+             * grant/revoke operations are idempotent and hence re-execution of the migration tool will fix any inconsistency
+             * due to such failures.
+             **/
+            boolean originalPermPresent = false;
+            for (String perm : migrated) {
+              if (perm.equalsIgnoreCase(privilegeStr)) {
+                originalPermPresent = true;
+                continue;
+              }
+              TSentryPrivilege x = converter.fromString(perm);
+              LOGGER.info("{} GRANT permission {}", getDryRunMessage(), perm);
+              if (!dryRun) {
+                client.grantPrivilege(requestorName, r.getRoleName(), component, x);
+              }
+            }
+
+            // Revoke old permission (only if not part of migrated permissions)
+            if (!originalPermPresent) {
+              LOGGER.info("{} REVOKE permission {}", getDryRunMessage(), privilegeStr);
+              if (!dryRun) {
+                client.revokePrivilege(requestorName, r.getRoleName(), component, p);
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+
+  private void migratePolicyFile () throws Exception {
+    Configuration conf = getSentryConf();
+    Path sourceFile = new Path (policyFile.get());
+    SimpleFileProviderBackend policyFileBackend = new SimpleFileProviderBackend(conf, sourceFile);
+    ProviderBackendContext ctx = new ProviderBackendContext();
+    policyFileBackend.initialize(ctx);
+
+    Set<String> roles = Sets.newHashSet();
+    Table<String, String, Set<String>> groupRolePrivilegeTable =
+        policyFileBackend.getGroupRolePrivilegeTable();
+
+    Ini output = PolicyFiles.loadFromPath(sourceFile.getFileSystem(conf), sourceFile);
+    Ini.Section rolesSection = output.get(PolicyFileConstants.ROLES);
+
+    for (String groupName : groupRolePrivilegeTable.rowKeySet()) {
+      for (String roleName : policyFileBackend.getRoles(Collections.singleton(groupName), ActiveRoleSet.ALL)) {
+        if (!roles.contains(roleName)) {
+          // Do the actual migration
+          Set<String> privileges = groupRolePrivilegeTable.get(groupName, roleName);
+          Collection<String> migrated = transformPrivileges(privileges);
+
+          if (!migrated.isEmpty()) {
+            LOGGER.info("{} For role {} migrating privileges from {} to {}", getDryRunMessage(),
+                roleName, privileges, migrated);
+            if (!dryRun) {
+              rolesSection.put(roleName, PrivilegeUtils.fromPrivilegeStrings(migrated));
+            }
+          }
+
+          roles.add(roleName);
+        }
+      }
+    }
+
+    if (!dryRun) {
+      Path targetFile = new Path (outputFile.get());
+      PolicyFiles.writeToPath(output, targetFile.getFileSystem(conf), targetFile);
+      LOGGER.info("Successfully saved migrated Sentry policy file at {}", outputFile.get());
+    }
+  }
+
+  private String getDryRunMessage() {
+    return dryRun ? "[Dry Run]" : "";
+  }
+
+  private Configuration getSentryConf() {
+    Configuration conf = new Configuration();
+    if (confPath.isPresent()) {
+      conf.addResource(new Path(confPath.get()), true);
+    }
+    return conf;
+  }
+
+  @VisibleForTesting
+  public boolean executeConfigTool(String [] args) throws Exception {
+    boolean result = true;
+    if (parseArgs(args)) {
+      run();
+    } else {
+      result = false;
+    }
+    return result;
+  }
+}

http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-tools/src/main/java/org/apache/sentry/cli/tools/PermissionsMigrationToolSolr.java
----------------------------------------------------------------------
diff --git a/sentry-tools/src/main/java/org/apache/sentry/cli/tools/PermissionsMigrationToolSolr.java b/sentry-tools/src/main/java/org/apache/sentry/cli/tools/PermissionsMigrationToolSolr.java
new file mode 100644
index 0000000..c24ae93
--- /dev/null
+++ b/sentry-tools/src/main/java/org/apache/sentry/cli/tools/PermissionsMigrationToolSolr.java
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sentry.cli.tools;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.sentry.core.common.utils.SentryConstants;
+import org.apache.sentry.core.model.solr.validator.SolrPrivilegeValidator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class provides SOLR specific functionality required for migrating Sentry privileges.
+ */
+public class PermissionsMigrationToolSolr extends PermissionsMigrationToolCommon {
+  private static final Logger LOGGER = LoggerFactory.getLogger(PermissionsMigrationToolSolr.class);
+
+
+  @Override
+  protected String getComponent(Configuration conf) {
+    return "SOLR";
+  }
+
+  @Override
+  protected String getServiceName(Configuration conf) {
+    return conf.get(SOLR_SERVICE_NAME, "service1");
+  }
+
+  @Override
+  protected Collection<String> transformPrivileges(Collection<String> privileges) {
+    List<String> result = new ArrayList<>();
+    boolean migrated = false;
+
+    if (getSourceVersion().major == 1) { // Migrate only Sentry 1.x permissions
+      for (String p : privileges) {
+        SolrPrivilegeValidator v = new SolrPrivilegeValidator();
+        v.validate(p, false);
+
+        if ("collection".equalsIgnoreCase(v.getEntityType()) && "admin".equalsIgnoreCase(v.getEntityName())) {
+          result.add(getPermissionStr("admin", "collections", v.getActionName()));
+          result.add(getPermissionStr("admin", "cores", v.getActionName()));
+          migrated = true;
+        } else if ("collection".equalsIgnoreCase(v.getEntityType()) && "*".equals(v.getEntityName())) {
+          result.add(getPermissionStr("admin", "collections", v.getActionName()));
+          result.add(getPermissionStr("admin", "cores", v.getActionName()));
+          result.add(p);
+          migrated = true;
+        } else {
+          result.add(p);
+        }
+      }
+    }
+
+    return migrated ? result : Collections.emptyList();
+  }
+
+  private String getPermissionStr (String entityType, String entityName, String action) {
+    StringBuilder builder = new StringBuilder();
+    builder.append(entityType);
+    builder.append(SentryConstants.KV_SEPARATOR);
+    builder.append(entityName);
+    if (action != null) {
+      builder.append(SentryConstants.AUTHORIZABLE_SEPARATOR);
+      builder.append(SentryConstants.PRIVILEGE_NAME);
+      builder.append(SentryConstants.KV_SEPARATOR);
+      builder.append(action);
+    }
+    return builder.toString();
+  }
+
+  public static void main(String[] args) throws Exception {
+    PermissionsMigrationToolSolr solrTool = new PermissionsMigrationToolSolr();
+    try {
+      solrTool.executeConfigTool(args);
+    } catch (Exception e) {
+      LOGGER.error(e.getMessage(), e);
+      Throwable current = e;
+      // find the first printable message;
+      while (current != null && current.getMessage() == null) {
+        current = current.getCause();
+      }
+      String error = "";
+      if (current != null && current.getMessage() != null) {
+        error = "Message: " + current.getMessage();
+      }
+      System.out.println("The operation failed. " + error);
+      System.exit(1);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-tools/src/main/java/org/apache/sentry/cli/tools/SentryConfigToolCommon.java
----------------------------------------------------------------------
diff --git a/sentry-tools/src/main/java/org/apache/sentry/cli/tools/SentryConfigToolCommon.java b/sentry-tools/src/main/java/org/apache/sentry/cli/tools/SentryConfigToolCommon.java
new file mode 100644
index 0000000..2d4f973
--- /dev/null
+++ b/sentry-tools/src/main/java/org/apache/sentry/cli/tools/SentryConfigToolCommon.java
@@ -0,0 +1,152 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.cli.tools;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.cli.Parser;
+
+abstract public class SentryConfigToolCommon {
+  private String policyFile;
+  private boolean validate;
+  private boolean importPolicy;
+  private boolean checkCompat;
+  private String confPath;
+
+ /**
+   *  parse arguments
+   * <pre>
+   *   -conf,--sentry_conf <filepath>     sentry config file path
+   *   -p,--policy_ini     <arg>          policy file path
+   *   -v,--validate                      validate policy file
+   *   -c,--checkcompat                   check compatibility with service
+   *   -i,--import                        import policy file
+   *   -h,--help                          print usage
+   * </pre>
+   * @param args
+   */
+  protected boolean parseArgs(String [] args) {
+    Options options = new Options();
+
+    Option globalPolicyPath = new Option("p", "policy_ini", true,
+        "Policy file path");
+    globalPolicyPath.setRequired(true);
+    options.addOption(globalPolicyPath);
+
+    Option validateOpt = new Option("v", "validate", false,
+        "Validate policy file");
+    validateOpt.setRequired(false);
+    options.addOption(validateOpt);
+
+    Option checkCompatOpt = new Option("c","checkcompat",false,
+        "Check compatibility with Sentry Service");
+    checkCompatOpt.setRequired(false);
+    options.addOption(checkCompatOpt);
+
+    Option importOpt = new Option("i", "import", false,
+        "Import policy file");
+    importOpt.setRequired(false);
+    options.addOption(importOpt);
+
+    // file path of sentry-site
+    Option sentrySitePathOpt = new Option("conf", "sentry_conf", true, "sentry-site file path");
+    sentrySitePathOpt.setRequired(true);
+    options.addOption(sentrySitePathOpt);
+
+    // help option
+    Option helpOpt = new Option("h", "help", false, "Shell usage");
+    helpOpt.setRequired(false);
+    options.addOption(helpOpt);
+
+    // this Options is parsed first for help option
+    Options helpOptions = new Options();
+    helpOptions.addOption(helpOpt);
+
+    try {
+      Parser parser = new GnuParser();
+
+      // parse help option first
+      CommandLine cmd = parser.parse(helpOptions, args, true);
+      for (Option opt : cmd.getOptions()) {
+        if (opt.getOpt().equals("h")) {
+          // get the help option, print the usage and exit
+          usage(options);
+          return false;
+        }
+      }
+
+      // without help option
+      cmd = parser.parse(options, args);
+
+      for (Option opt : cmd.getOptions()) {
+        if (opt.getOpt().equals("p")) {
+          policyFile = opt.getValue();
+        } else if (opt.getOpt().equals("v")) {
+          validate = true;
+        } else if (opt.getOpt().equals("i")) {
+          importPolicy = true;
+        } else if (opt.getOpt().equals("c")) {
+          checkCompat = true;
+        } else if (opt.getOpt().equals("conf")) {
+          confPath = opt.getValue();
+        }
+      }
+
+      if (!validate && !importPolicy) {
+        throw new IllegalArgumentException("No action specified; at least one of action or import must be specified");
+      }
+    } catch (ParseException pe) {
+      System.out.println(pe.getMessage());
+      usage(options);
+      return false;
+    }
+    return true;
+  }
+
+  // print usage
+  private void usage(Options sentryOptions) {
+    HelpFormatter formatter = new HelpFormatter();
+    formatter.printHelp("sentryConfigTool", sentryOptions);
+  }
+
+  public abstract void run() throws Exception;
+
+  @VisibleForTesting
+  public boolean executeConfigTool(String [] args) throws Exception {
+    boolean result = true;
+    if (parseArgs(args)) {
+      run();
+    } else {
+      result = false;
+    }
+    return result;
+  }
+
+  public String getPolicyFile() { return policyFile; }
+  public boolean getValidate() { return validate; }
+  public boolean getImportPolicy() { return importPolicy; }
+  public boolean getCheckCompat() { return checkCompat; }
+  public String getConfPath() { return confPath; }
+}

http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-tools/src/main/java/org/apache/sentry/cli/tools/SentryConfigToolIndexer.java
----------------------------------------------------------------------
diff --git a/sentry-tools/src/main/java/org/apache/sentry/cli/tools/SentryConfigToolIndexer.java b/sentry-tools/src/main/java/org/apache/sentry/cli/tools/SentryConfigToolIndexer.java
new file mode 100644
index 0000000..712d9ed
--- /dev/null
+++ b/sentry-tools/src/main/java/org/apache/sentry/cli/tools/SentryConfigToolIndexer.java
@@ -0,0 +1,341 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.cli.tools;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import com.google.common.collect.Table;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.sentry.core.common.Action;
+import org.apache.sentry.core.common.exception.SentryConfigurationException;
+import org.apache.sentry.core.common.utils.KeyValue;
+import org.apache.sentry.core.model.indexer.IndexerPrivilegeModel;
+import org.apache.sentry.provider.common.ProviderBackend;
+import org.apache.sentry.provider.common.ProviderBackendContext;
+import org.apache.sentry.provider.db.generic.service.thrift.SentryGenericServiceClient;
+import org.apache.sentry.provider.db.generic.service.thrift.SentryGenericServiceClientFactory;
+import org.apache.sentry.provider.db.generic.tools.GenericPrivilegeConverter;
+import org.apache.sentry.provider.file.SimpleFileProviderBackend;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.sentry.core.common.utils.SentryConstants.AUTHORIZABLE_SPLITTER;
+import static org.apache.sentry.provider.common.AuthorizationComponent.HBASE_INDEXER;
+import static org.apache.sentry.service.thrift.ServiceConstants.ClientConfig.SERVICE_NAME;
+
+/**
+ * SentryConfigToolIndexer is an administrative tool used to parse a HBase Indexer policy file
+ * and add the role, group mappings, and privileges therein to the Sentry service.
+ */
+public class SentryConfigToolIndexer {
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(SentryConfigToolIndexer.class);
+
+  private String policyFile;
+  private boolean validate;
+  private boolean importPolicy;
+  private boolean checkCompat;
+  private String confPath;
+
+  private String serviceName;
+
+
+
+  public String getPolicyFile() { return policyFile; }
+
+  public boolean getValidate() { return validate; }
+  public boolean getImportPolicy() { return importPolicy; }
+  public boolean getCheckCompat() { return checkCompat; }
+  public String getConfPath() { return confPath; }
+  public String getServiceName() {
+    return serviceName;
+  }
+
+  /**
+   * Adds command line options for the tool to the passed Options object. Used to extend existing options.
+   * @param options
+   */
+  public void setupOptions(Options options) {
+    Option globalPolicyPath = new Option("f", "policy_ini", true,
+            "Policy file path");
+    globalPolicyPath.setRequired(false);
+    options.addOption(globalPolicyPath);
+
+    Option validateOpt = new Option("v", "validate", false,
+            "Validate policy file");
+    validateOpt.setRequired(false);
+    options.addOption(validateOpt);
+
+    Option checkCompatOpt = new Option("c","checkcompat",false,
+            "Check compatibility with Sentry Service");
+    checkCompatOpt.setRequired(false);
+    options.addOption(checkCompatOpt);
+
+    Option importOpt = new Option("i", "import", false,
+            "Import policy file");
+    importOpt.setRequired(false);
+    options.addOption(importOpt);
+
+  }
+
+  /**
+   * Parses and processes the arguments from the given command line object.
+   * @param cmd
+   */
+  public void parseOptions(CommandLine cmd) {
+    boolean isToolActive = false;
+    for (Option opt : cmd.getOptions()) {
+      if (opt.getOpt().equals("mgr")) {
+        isToolActive = true;
+      }
+    }
+    if (!isToolActive) {
+      return;
+    }
+    for (Option opt : cmd.getOptions()) {
+      if (opt.getOpt().equals("f")) {
+        policyFile = opt.getValue();
+      } else if (opt.getOpt().equals("v")) {
+        validate = true;
+      } else if (opt.getOpt().equals("i")) {
+        importPolicy = true;
+      } else if (opt.getOpt().equals("c")) {
+        checkCompat = true;
+      } else if (opt.getOpt().equals("conf")) {
+        confPath = opt.getValue();
+      } else if (opt.getOpt().equals("s")) {
+        serviceName = opt.getValue();
+      }
+    }
+    if (policyFile == null) {
+      throw new IllegalArgumentException("Missing required option: f");
+    }
+    if (!validate && !importPolicy) {
+      throw new IllegalArgumentException("No action specified; at least one of action or import must be specified");
+    }
+  }
+
+
+  /**
+   * Processes the necessary command based on the arguments parsed earlier.
+   * @throws Exception
+   */
+  public void run() throws Exception {
+    String component = HBASE_INDEXER;
+    Configuration conf = getSentryConf();
+
+    String service = conf.get(SERVICE_NAME, getServiceName());
+
+    if (service == null) {
+      throw new IllegalArgumentException("Service was not defined. Please, use -s command option, or sentry.provider.backend.generic.service-name configuration entry.");
+    }
+
+    LOGGER.info(String.format("Context: component=%s, service=%s", component, service));
+    // instantiate a solr client for sentry service.  This sets the ugi, so must
+    // be done before getting the ugi below.
+    try(SentryGenericServiceClient client =
+                SentryGenericServiceClientFactory.create(conf)) {
+      UserGroupInformation ugi = UserGroupInformation.getLoginUser();
+      String requestorName = ugi.getShortUserName();
+
+      convertINIToSentryServiceCmds(component, service, requestorName, conf, client,
+              getPolicyFile(), getValidate(), getImportPolicy(), getCheckCompat());
+    }
+  }
+
+  private Configuration getSentryConf() {
+    Configuration conf = new Configuration();
+    conf.addResource(new Path(getConfPath()), true);
+    return conf;
+  }
+
+  private void convertINIToSentryServiceCmds(String component,
+      String service, String requestorName,
+      Configuration conf, SentryGenericServiceClient client,
+      String policyFile, boolean validate, boolean importPolicy,
+      boolean checkCompat) throws Exception {
+
+    //instantiate a file providerBackend for parsing
+    LOGGER.info("Reading policy file at: " + policyFile);
+    SimpleFileProviderBackend policyFileBackend =
+        new SimpleFileProviderBackend(conf, policyFile);
+    ProviderBackendContext context = new ProviderBackendContext();
+    context.setValidators(IndexerPrivilegeModel.getInstance().getPrivilegeValidators());
+    policyFileBackend.initialize(context);
+    if (validate) {
+      validatePolicy(policyFileBackend);
+    }
+
+    if (checkCompat) {
+      checkCompat(policyFileBackend);
+    }
+
+    //import the relations about group,role and privilege into the DB store
+    Set<String> roles = Sets.newHashSet();
+    Table<String, String, Set<String>> groupRolePrivilegeTable =
+        policyFileBackend.getGroupRolePrivilegeTable();
+    GenericPrivilegeConverter converter = new GenericPrivilegeConverter(component, service, false);
+
+    for (String groupName : groupRolePrivilegeTable.rowKeySet()) {
+      for (String roleName : groupRolePrivilegeTable.columnKeySet()) {
+        if (!roles.contains(roleName)) {
+          LOGGER.info(dryRunMessage(importPolicy) + "Creating role: " + roleName.toLowerCase(Locale.US));
+          if (importPolicy) {
+            client.createRoleIfNotExist(requestorName, roleName, component);
+          }
+          roles.add(roleName);
+        }
+
+        Set<String> privileges = groupRolePrivilegeTable.get(groupName, roleName);
+        if (privileges == null) {
+          continue;
+        }
+        LOGGER.info(dryRunMessage(importPolicy) + "Adding role: " + roleName.toLowerCase(Locale.US) + " to group: " + groupName);
+        if (importPolicy) {
+          client.grantRoleToGroups(requestorName, roleName, component, Sets.newHashSet(groupName));
+        }
+
+        for (String permission : privileges) {
+          String action = null;
+
+          for (String authorizable : AUTHORIZABLE_SPLITTER.
+              trimResults().split(permission)) {
+            KeyValue kv = new KeyValue(authorizable);
+            String key = kv.getKey();
+            String value = kv.getValue();
+            if ("action".equalsIgnoreCase(key)) {
+              action = value;
+            }
+          }
+
+          // Service doesn't support not specifying action
+          if (action == null) {
+            permission += "->action=" + Action.ALL;
+          }
+          LOGGER.info(dryRunMessage(importPolicy) + "Adding permission: " + permission + " to role: " + roleName.toLowerCase(Locale.US));
+          if (importPolicy) {
+            client.grantPrivilege(requestorName, roleName, component, converter.fromString(permission));
+          }
+        }
+      }
+    }
+  }
+
+  private void validatePolicy(ProviderBackend backend) throws Exception {
+    try {
+      backend.validatePolicy(true);
+    } catch (SentryConfigurationException e) {
+      printConfigErrorsWarnings(e);
+      throw e;
+    }
+  }
+
+  private void printConfigErrorsWarnings(SentryConfigurationException configException) {
+    System.out.println(" *** Found configuration problems *** ");
+    for (String errMsg : configException.getConfigErrors()) {
+      System.out.println("ERROR: " + errMsg);
+    }
+    for (String warnMsg : configException.getConfigWarnings()) {
+      System.out.println("Warning: " + warnMsg);
+    }
+  }
+
+  private void checkCompat(SimpleFileProviderBackend backend) throws Exception {
+    Map<String, Set<String>> rolesCaseMapping = new HashMap<String, Set<String>>();
+    Table<String, String, Set<String>> groupRolePrivilegeTable =
+      backend.getGroupRolePrivilegeTable();
+
+    for (String roleName : groupRolePrivilegeTable.columnKeySet()) {
+      String roleNameLower = roleName.toLowerCase(Locale.US);
+      if (!roleName.equals(roleNameLower)) {
+        if (!rolesCaseMapping.containsKey(roleNameLower)) {
+          rolesCaseMapping.put(roleNameLower, Sets.newHashSet(roleName));
+        } else {
+          rolesCaseMapping.get(roleNameLower).add(roleName);
+        }
+      }
+    }
+
+    List<String> errors = new LinkedList<String>();
+    StringBuilder warningString = new StringBuilder();
+    if (!rolesCaseMapping.isEmpty()) {
+      warningString.append("The following roles names will be lower cased when added to the Sentry Service.\n");
+      warningString.append("This will cause document-level security to fail to match the role tokens.\n");
+      warningString.append("Role names: ");
+    }
+    boolean firstWarning = true;
+
+    for (Map.Entry<String, Set<String>> entry : rolesCaseMapping.entrySet()) {
+      Set<String> caseMapping = entry.getValue();
+      if (caseMapping.size() > 1) {
+        StringBuilder errorString = new StringBuilder();
+        errorString.append("The following (cased) roles map to the same role in the sentry service: ");
+        boolean first = true;
+        for (String casedRole : caseMapping) {
+          errorString.append(first ? "" : ", ");
+          errorString.append(casedRole);
+          first = false;
+        }
+        errorString.append(".  Role in service: ").append(entry.getKey());
+        errors.add(errorString.toString());
+      }
+
+      for (String casedRole : caseMapping) {
+        warningString.append(firstWarning? "" : ", ");
+        warningString.append(casedRole);
+        firstWarning = false;
+      }
+    }
+
+    for (String error : errors) {
+      System.out.println("ERROR: " + error);
+    }
+    System.out.println("\n");
+
+    System.out.println("Warning: " + warningString.toString());
+    if (errors.size() > 0) {
+      SentryConfigurationException ex =
+          new SentryConfigurationException("Compatibility check failure");
+      ex.setConfigErrors(errors);
+      ex.setConfigWarnings(Lists.<String>asList(warningString.toString(), new String[0]));
+      throw ex;
+    }
+  }
+
+  private String dryRunMessage(boolean importPolicy) {
+    if (importPolicy) {
+      return "";
+    } else {
+      return "[Dry Run] ";
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-tools/src/main/java/org/apache/sentry/cli/tools/SentryConfigToolSolr.java
----------------------------------------------------------------------
diff --git a/sentry-tools/src/main/java/org/apache/sentry/cli/tools/SentryConfigToolSolr.java b/sentry-tools/src/main/java/org/apache/sentry/cli/tools/SentryConfigToolSolr.java
new file mode 100644
index 0000000..dfa981d
--- /dev/null
+++ b/sentry-tools/src/main/java/org/apache/sentry/cli/tools/SentryConfigToolSolr.java
@@ -0,0 +1,265 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.cli.tools;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import com.google.common.collect.Table;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.sentry.core.common.Action;
+import org.apache.sentry.core.common.exception.SentryConfigurationException;
+import org.apache.sentry.core.common.utils.KeyValue;
+import org.apache.sentry.core.common.utils.SentryConstants;
+import org.apache.sentry.core.model.solr.SolrPrivilegeModel;
+import org.apache.sentry.provider.common.ProviderBackend;
+import org.apache.sentry.provider.common.ProviderBackendContext;
+import org.apache.sentry.provider.db.generic.service.thrift.SentryGenericServiceClient;
+import org.apache.sentry.provider.db.generic.service.thrift.SentryGenericServiceClientFactory;
+import org.apache.sentry.provider.db.generic.tools.GenericPrivilegeConverter;
+import org.apache.sentry.provider.file.SimpleFileProviderBackend;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * SentryConfigToolSolr is an administrative tool used to parse a Solr policy file
+ * and add the role, group mappings, and privileges therein to the Sentry service.
+ */
+public class SentryConfigToolSolr extends SentryConfigToolCommon {
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(SentryConfigToolSolr.class);
+  public static final String SOLR_SERVICE_NAME = "sentry.service.client.solr.service.name";
+
+  @Override
+  public void run() throws Exception {
+    String component = "SOLR";
+    Configuration conf = getSentryConf();
+
+    String service = conf.get(SOLR_SERVICE_NAME, "service1");
+    // instantiate a solr client for sentry service.  This sets the ugi, so must
+    // be done before getting the ugi below.
+    try(SentryGenericServiceClient client =
+                SentryGenericServiceClientFactory.create(conf)) {
+      UserGroupInformation ugi = UserGroupInformation.getLoginUser();
+      String requestorName = ugi.getShortUserName();
+
+      convertINIToSentryServiceCmds(component, service, requestorName, conf, client,
+              getPolicyFile(), getValidate(), getImportPolicy(), getCheckCompat());
+    }
+  }
+
+  private Configuration getSentryConf() {
+    Configuration conf = new Configuration();
+    conf.addResource(new Path(getConfPath()), true);
+    return conf;
+  }
+
+   /**
+    * Convert policy file to solrctl commands -- based on SENTRY-480
+    */
+  private void convertINIToSentryServiceCmds(String component,
+      String service, String requestorName,
+      Configuration conf, SentryGenericServiceClient client,
+      String policyFile, boolean validate, boolean importPolicy,
+      boolean checkCompat) throws Exception {
+
+    //instantiate a file providerBackend for parsing
+    LOGGER.info("Reading policy file at: " + policyFile);
+    SimpleFileProviderBackend policyFileBackend =
+        new SimpleFileProviderBackend(conf, policyFile);
+    ProviderBackendContext context = new ProviderBackendContext();
+    context.setValidators(SolrPrivilegeModel.getInstance().getPrivilegeValidators());
+    policyFileBackend.initialize(context);
+    if (validate) {
+      validatePolicy(policyFileBackend);
+    }
+
+    if (checkCompat) {
+      checkCompat(policyFileBackend);
+    }
+
+    //import the relations about group,role and privilege into the DB store
+    Set<String> roles = Sets.newHashSet();
+    Table<String, String, Set<String>> groupRolePrivilegeTable =
+        policyFileBackend.getGroupRolePrivilegeTable();
+    GenericPrivilegeConverter converter = new GenericPrivilegeConverter(component, service, false);
+
+    for (String groupName : groupRolePrivilegeTable.rowKeySet()) {
+      for (String roleName : groupRolePrivilegeTable.columnKeySet()) {
+        if (!roles.contains(roleName)) {
+          LOGGER.info(dryRunMessage(importPolicy) + "Creating role: " + roleName.toLowerCase(Locale.US));
+          if (importPolicy) {
+            client.createRoleIfNotExist(requestorName, roleName, component);
+          }
+          roles.add(roleName);
+        }
+
+        Set<String> privileges = groupRolePrivilegeTable.get(groupName, roleName);
+        if (privileges == null) {
+          continue;
+        }
+        LOGGER.info(dryRunMessage(importPolicy) + "Adding role: " + roleName.toLowerCase(Locale.US) + " to group: " + groupName);
+        if (importPolicy) {
+          client.grantRoleToGroups(requestorName, roleName, component, Sets.newHashSet(groupName));
+        }
+
+        for (String permission : privileges) {
+          String action = null;
+
+          for (String authorizable : SentryConstants.AUTHORIZABLE_SPLITTER.
+              trimResults().split(permission)) {
+            KeyValue kv = new KeyValue(authorizable);
+            String key = kv.getKey();
+            String value = kv.getValue();
+            if ("action".equalsIgnoreCase(key)) {
+              action = value;
+            }
+          }
+
+          // Service doesn't support not specifying action
+          if (action == null) {
+            permission += "->action=" + Action.ALL;
+          }
+          LOGGER.info(dryRunMessage(importPolicy) + "Adding permission: " + permission + " to role: " + roleName.toLowerCase(Locale.US));
+          if (importPolicy) {
+            client.grantPrivilege(requestorName, roleName, component, converter.fromString(permission));
+          }
+        }
+      }
+    }
+  }
+
+  private void validatePolicy(ProviderBackend backend) throws Exception {
+    try {
+      backend.validatePolicy(true);
+    } catch (SentryConfigurationException e) {
+      printConfigErrorsWarnings(e);
+      throw e;
+    }
+  }
+
+  private void printConfigErrorsWarnings(SentryConfigurationException configException) {
+    System.out.println(" *** Found configuration problems *** ");
+    for (String errMsg : configException.getConfigErrors()) {
+      System.out.println("ERROR: " + errMsg);
+    }
+    for (String warnMsg : configException.getConfigWarnings()) {
+      System.out.println("Warning: " + warnMsg);
+    }
+  }
+
+  private void checkCompat(SimpleFileProviderBackend backend) throws Exception {
+    Map<String, Set<String>> rolesCaseMapping = new HashMap<String, Set<String>>();
+    Table<String, String, Set<String>> groupRolePrivilegeTable =
+      backend.getGroupRolePrivilegeTable();
+
+    for (String roleName : groupRolePrivilegeTable.columnKeySet()) {
+      String roleNameLower = roleName.toLowerCase(Locale.US);
+      if (!roleName.equals(roleNameLower)) {
+        if (!rolesCaseMapping.containsKey(roleNameLower)) {
+          rolesCaseMapping.put(roleNameLower, Sets.newHashSet(roleName));
+        } else {
+          rolesCaseMapping.get(roleNameLower).add(roleName);
+        }
+      }
+    }
+
+    List<String> errors = new LinkedList<String>();
+    StringBuilder warningString = new StringBuilder();
+    if (!rolesCaseMapping.isEmpty()) {
+      warningString.append("The following roles names will be lower cased when added to the Sentry Service.\n");
+      warningString.append("This will cause document-level security to fail to match the role tokens.\n");
+      warningString.append("Role names: ");
+    }
+    boolean firstWarning = true;
+
+    for (Map.Entry<String, Set<String>> entry : rolesCaseMapping.entrySet()) {
+      Set<String> caseMapping = entry.getValue();
+      if (caseMapping.size() > 1) {
+        StringBuilder errorString = new StringBuilder();
+        errorString.append("The following (cased) roles map to the same role in the sentry service: ");
+        boolean first = true;
+        for (String casedRole : caseMapping) {
+          errorString.append(first ? "" : ", ");
+          errorString.append(casedRole);
+          first = false;
+        }
+        errorString.append(".  Role in service: ").append(entry.getKey());
+        errors.add(errorString.toString());
+      }
+
+      for (String casedRole : caseMapping) {
+        warningString.append(firstWarning? "" : ", ");
+        warningString.append(casedRole);
+        firstWarning = false;
+      }
+    }
+
+    for (String error : errors) {
+      System.out.println("ERROR: " + error);
+    }
+    System.out.println("\n");
+
+    System.out.println("Warning: " + warningString.toString());
+    if (errors.size() > 0) {
+      SentryConfigurationException ex =
+          new SentryConfigurationException("Compatibility check failure");
+      ex.setConfigErrors(errors);
+      ex.setConfigWarnings(Lists.<String>asList(warningString.toString(), new String[0]));
+      throw ex;
+    }
+  }
+
+  private String dryRunMessage(boolean importPolicy) {
+    if (importPolicy) {
+      return "";
+    } else {
+      return "[Dry Run] ";
+    }
+  }
+
+  public static void main(String[] args) throws Exception {
+    SentryConfigToolSolr solrTool = new SentryConfigToolSolr();
+    try {
+      solrTool.executeConfigTool(args);
+    } catch (Exception e) {
+      LOGGER.error(e.getMessage(), e);
+      Throwable current = e;
+      // find the first printable message;
+      while (current != null && current.getMessage() == null) {
+        current = current.getCause();
+      }
+      String error = "";
+      if (current != null && current.getMessage() != null) {
+        error = "Message: " + current.getMessage();
+      }
+      System.out.println("The operation failed. " + error);
+      System.exit(1);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-tools/src/main/java/org/apache/sentry/cli/tools/SentrySchemaHelper.java
----------------------------------------------------------------------
diff --git a/sentry-tools/src/main/java/org/apache/sentry/cli/tools/SentrySchemaHelper.java b/sentry-tools/src/main/java/org/apache/sentry/cli/tools/SentrySchemaHelper.java
new file mode 100644
index 0000000..3723fd9
--- /dev/null
+++ b/sentry-tools/src/main/java/org/apache/sentry/cli/tools/SentrySchemaHelper.java
@@ -0,0 +1,315 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sentry.cli.tools;
+
+import java.util.IllegalFormatException;
+
+public final class SentrySchemaHelper {
+  public static final String DB_DERBY = "derby";
+  public static final String DB_MYSQL = "mysql";
+  public static final String DB_POSTGRACE = "postgres";
+  public static final String DB_ORACLE = "oracle";
+  public static final String DB_DB2 = "db2";
+
+  public interface NestedScriptParser {
+
+    public enum CommandType {
+      PARTIAL_STATEMENT,
+      TERMINATED_STATEMENT,
+      COMMENT
+    }
+
+    String DEFAUTL_DELIMITER = ";";
+    /***
+     * Find the type of given command
+     * @param dbCommand
+     * @return
+     */
+    boolean isPartialCommand(String dbCommand) throws IllegalArgumentException;
+
+    /** Parse the DB specific nesting format and extract the inner script name if any
+     * @param dbCommand command from parent script
+     * @return
+     * @throws IllegalFormatException
+     */
+    String getScriptName(String dbCommand) throws IllegalArgumentException;
+
+    /***
+     * Find if the given command is a nested script execution
+     * @param dbCommand
+     * @return
+     */
+    boolean isNestedScript(String dbCommand);
+
+    /***
+     * Find if the given command is should be passed to DB
+     * @param dbCommand
+     * @return
+     */
+    boolean isNonExecCommand(String dbCommand);
+
+    /***
+     * Get the SQL statement delimiter
+     * @return
+     */
+    String getDelimiter();
+
+    /***
+     * Clear any client specific tags
+     * @return
+     */
+    String cleanseCommand(String dbCommand);
+
+    /***
+     * Does the DB required table/column names quoted
+     * @return
+     */
+    boolean needsQuotedIdentifier();
+
+    /***
+     * Set DB specific options if any
+     * @param dbOps
+     */
+    void setDbOpts(String dbOps);
+  }
+
+
+  /***
+   * Base implemenation of NestedScriptParser
+   * abstractCommandParser.
+   *
+   */
+  private static abstract class AbstractCommandParser implements NestedScriptParser {
+    private String dbOpts = null;
+
+    @Override
+    public boolean isPartialCommand(String dbCommand) throws IllegalArgumentException{
+      if (dbCommand == null || dbCommand.isEmpty()) {
+        throw new IllegalArgumentException("invalid command line " + dbCommand);
+      }
+      String trimmedDbCommand = dbCommand.trim();
+      return !(trimmedDbCommand.endsWith(getDelimiter()) || isNonExecCommand(trimmedDbCommand));
+    }
+
+    @Override
+    public boolean isNonExecCommand(String dbCommand) {
+      return dbCommand.startsWith("--") || dbCommand.startsWith("#");
+    }
+
+    @Override
+    public String getDelimiter() {
+      return DEFAUTL_DELIMITER;
+    }
+
+    @Override
+    public String cleanseCommand(String dbCommand) {
+      // strip off the delimiter
+      if (dbCommand.endsWith(getDelimiter())) {
+        dbCommand = dbCommand.substring(0,
+            dbCommand.length() - getDelimiter().length());
+      }
+      return dbCommand;
+    }
+
+    @Override
+    public boolean needsQuotedIdentifier() {
+      return false;
+    }
+
+    @Override
+    public void setDbOpts(String dbOpts) {
+      this.dbOpts = dbOpts;
+    }
+
+    protected String getDbOpts() {
+      return dbOpts;
+    }
+  }
+
+
+  // Derby commandline parser
+  public static class DerbyCommandParser extends AbstractCommandParser {
+    private static final String DERBY_NESTING_TOKEN = "RUN";
+
+    @Override
+    public String getScriptName(String dbCommand) throws IllegalArgumentException {
+
+      if (!isNestedScript(dbCommand)) {
+        throw new IllegalArgumentException("Not a script format " + dbCommand);
+      }
+      String[] tokens = dbCommand.split(" ");
+      if (tokens.length != 2) {
+        throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
+      }
+      return tokens[1].replace(";", "").replaceAll("'", "");
+    }
+
+    @Override
+    public boolean isNestedScript(String dbCommand) {
+      // Derby script format is RUN '<file>'
+     return dbCommand.startsWith(DERBY_NESTING_TOKEN);
+    }
+  }
+
+
+  // MySQL parser
+  public static class MySqlCommandParser extends AbstractCommandParser {
+    private static final String MYSQL_NESTING_TOKEN = "SOURCE";
+    private static final String DELIMITER_TOKEN = "DELIMITER";
+    private String delimiter = DEFAUTL_DELIMITER;
+
+    @Override
+    public boolean isPartialCommand(String dbCommand) throws IllegalArgumentException{
+      boolean isPartial = super.isPartialCommand(dbCommand);
+      // if this is a delimiter directive, reset our delimiter
+      if (dbCommand.startsWith(DELIMITER_TOKEN)) {
+        String[] tokens = dbCommand.split(" ");
+        if (tokens.length != 2) {
+          throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
+        }
+        delimiter = tokens[1];
+      }
+      return isPartial;
+    }
+
+    @Override
+    public String getScriptName(String dbCommand) throws IllegalArgumentException {
+      String[] tokens = dbCommand.split(" ");
+      if (tokens.length != 2) {
+        throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
+      }
+      // remove ending ';'
+      return tokens[1].replace(";", "");
+    }
+
+    @Override
+    public boolean isNestedScript(String dbCommand) {
+      return dbCommand.startsWith(MYSQL_NESTING_TOKEN);
+    }
+
+    @Override
+    public String getDelimiter() {
+      return delimiter;
+    }
+
+    @Override
+    public boolean isNonExecCommand(String dbCommand) {
+      return super.isNonExecCommand(dbCommand) ||
+          dbCommand.startsWith("/*") && dbCommand.endsWith("*/") ||
+          dbCommand.startsWith(DELIMITER_TOKEN);
+    }
+
+    @Override
+    public String cleanseCommand(String dbCommand) {
+      return super.cleanseCommand(dbCommand).replaceAll("/\\*.*?\\*/[^;]", "");
+    }
+
+  }
+
+  // Postgres specific parser
+  public static class PostgresCommandParser extends AbstractCommandParser {
+    public static final String POSTGRES_STRING_COMMAND_FILTER = "SET standard_conforming_strings";
+    public static final String POSTGRES_STRING_CLIENT_ENCODING = "SET client_encoding";
+    public static final String POSTGRES_SKIP_STANDARD_STRING = "postgres.filter.81";
+    private static final String POSTGRES_NESTING_TOKEN = "\\i";
+
+    @Override
+    public String getScriptName(String dbCommand) throws IllegalArgumentException {
+      String[] tokens = dbCommand.split(" ");
+      if (tokens.length != 2) {
+        throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
+      }
+      // remove ending ';'
+      return tokens[1].replace(";", "");
+    }
+
+    @Override
+    public boolean isNestedScript(String dbCommand) {
+      return dbCommand.startsWith(POSTGRES_NESTING_TOKEN);
+    }
+
+    @Override
+    public boolean needsQuotedIdentifier() {
+      return true;
+    }
+
+    @Override
+    public boolean isNonExecCommand(String dbCommand) {
+      // Skip "standard_conforming_strings" command which is not supported in older postgres
+      if (POSTGRES_SKIP_STANDARD_STRING.equalsIgnoreCase(getDbOpts()) 
+        && (dbCommand.startsWith(POSTGRES_STRING_COMMAND_FILTER) || dbCommand.startsWith(POSTGRES_STRING_CLIENT_ENCODING))) {
+        return true;
+      }
+      return super.isNonExecCommand(dbCommand);
+    }
+  }
+
+  //Oracle specific parser
+  public static class OracleCommandParser extends AbstractCommandParser {
+    private static final String ORACLE_NESTING_TOKEN = "@";
+    @Override
+    public String getScriptName(String dbCommand) throws IllegalArgumentException {
+      if (!isNestedScript(dbCommand)) {
+        throw new IllegalArgumentException("Not a nested script format " + dbCommand);
+      }
+      // remove ending ';' and starting '@'
+      return dbCommand.replace(";", "").replace(ORACLE_NESTING_TOKEN, "");
+    }
+
+    @Override
+    public boolean isNestedScript(String dbCommand) {
+      return dbCommand.startsWith(ORACLE_NESTING_TOKEN);
+    }
+  }
+
+  // DB2 commandline parser
+  public static class DB2CommandParser extends AbstractCommandParser {
+
+    @Override
+    public String getScriptName(String dbCommand) throws IllegalArgumentException {
+        //DB2 does not support nesting script
+        throw new IllegalArgumentException("DB2 does not support nesting script " + dbCommand);
+    }
+
+    @Override
+    public boolean isNestedScript(String dbCommand) {
+        //DB2 does not support nesting script
+     return false;
+    }
+  }
+
+  public static NestedScriptParser getDbCommandParser(String dbName) {
+    if (dbName.equalsIgnoreCase(DB_DERBY)) {
+      return new DerbyCommandParser();
+    } else if (dbName.equalsIgnoreCase(DB_MYSQL)) {
+      return new MySqlCommandParser();
+    } else if (dbName.equalsIgnoreCase(DB_POSTGRACE)) {
+      return new PostgresCommandParser();
+    } else if (dbName.equalsIgnoreCase(DB_ORACLE)) {
+        return new OracleCommandParser();
+    } else if (dbName.equalsIgnoreCase(DB_DB2)) {
+      return new DB2CommandParser();
+    } else {
+      throw new IllegalArgumentException("Unknown dbType " + dbName);
+    }
+  }
+  
+  private SentrySchemaHelper() {
+    // Make constructor private to avoid instantiation
+  }
+}


Mime
View raw message