sentry-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gqs...@apache.org
Subject [1/2] incubator-sentry git commit: SENTRY-647: Add e2e tests for Sqoop Sentry integration (Guoquan Shen, reviewed by Dapeng Sun)
Date Thu, 09 Jul 2015 05:08:13 GMT
Repository: incubator-sentry
Updated Branches:
  refs/heads/master c8d5fcef9 -> 987618115


http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/98761811/sentry-tests/sentry-tests-sqoop/src/test/java/org/apache/sentry/tests/e2e/sqoop/TestServerScopeEndToEnd.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-sqoop/src/test/java/org/apache/sentry/tests/e2e/sqoop/TestServerScopeEndToEnd.java
b/sentry-tests/sentry-tests-sqoop/src/test/java/org/apache/sentry/tests/e2e/sqoop/TestServerScopeEndToEnd.java
new file mode 100644
index 0000000..85bae92
--- /dev/null
+++ b/sentry-tests/sentry-tests-sqoop/src/test/java/org/apache/sentry/tests/e2e/sqoop/TestServerScopeEndToEnd.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sentry.tests.e2e.sqoop;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import org.apache.sentry.core.model.sqoop.SqoopActionConstant;
+import org.apache.sqoop.client.SqoopClient;
+import org.apache.sqoop.model.MJob;
+import org.apache.sqoop.model.MLink;
+import org.apache.sqoop.model.MPrincipal;
+import org.apache.sqoop.model.MPrivilege;
+import org.apache.sqoop.model.MResource;
+import org.apache.sqoop.model.MRole;
+import org.apache.sqoop.security.SecurityError;
+import org.junit.Test;
+
+import com.google.common.collect.Lists;
+
+public class TestServerScopeEndToEnd extends AbstractSqoopSentryTestBase {
+
+  @Test
+  public void testServerScopePrivilege() throws Exception {
+    /**
+     * ADMIN_USER create two links and one job
+     */
+    SqoopClient client = sqoopServerRunner.getSqoopClient(ADMIN_USER);
+    MLink rdbmsLink = client.createLink("generic-jdbc-connector");
+    sqoopServerRunner.fillRdbmsLinkConfig(rdbmsLink);
+    sqoopServerRunner.saveLink(client, rdbmsLink);
+
+    MLink hdfsLink = client.createLink("hdfs-connector");
+    sqoopServerRunner.fillHdfsLink(hdfsLink);
+    sqoopServerRunner.saveLink(client, hdfsLink);
+
+    MJob job1 = client.createJob(hdfsLink.getPersistenceId(), rdbmsLink.getPersistenceId());
+    // set HDFS "FROM" config for the job, since the connector test case base class only
has utilities for HDFS!
+    sqoopServerRunner.fillHdfsFromConfig(job1);
+    // set the RDBM "TO" config here
+    sqoopServerRunner.fillRdbmsToConfig(job1);
+    // create job
+    sqoopServerRunner.saveJob(client, job1);
+
+
+    MResource  sqoopServer1 = new MResource(SQOOP_SERVER_NAME, MResource.TYPE.SERVER);
+    /**
+     * ADMIN_USER grant read privilege on server SQOOP_SERVER_NAME to role1
+     */
+    MRole role1 = new MRole(ROLE1);
+    MPrincipal group1 = new MPrincipal(GROUP1, MPrincipal.TYPE.GROUP);
+    MPrivilege readPrivilege = new MPrivilege(sqoopServer1,SqoopActionConstant.READ, false);
+    client.createRole(role1);
+    client.grantRole(Lists.newArrayList(role1), Lists.newArrayList(group1));
+    client.grantPrivilege(Lists.newArrayList(new MPrincipal(role1.getName(), MPrincipal.TYPE.ROLE)),
+        Lists.newArrayList(readPrivilege));
+
+    /**
+     * ADMIN_USER grant write privilege on server SQOOP_SERVER_NAME to role2
+     * ADMIN_USER grant read privilege on connector all to role2 (for update link required)
+     * ADMIN_USER grant read privilege on link all to role2 (for update job required)
+     */
+    MRole role2 = new MRole(ROLE2);
+    MPrincipal group2 = new MPrincipal(GROUP2, MPrincipal.TYPE.GROUP);
+    MPrivilege writePrivilege = new MPrivilege(sqoopServer1,SqoopActionConstant.WRITE, false);
+    client.createRole(role2);
+
+    MResource allConnector = new MResource(SqoopActionConstant.ALL, MResource.TYPE.CONNECTOR);
+    MResource allLink = new MResource(SqoopActionConstant.ALL, MResource.TYPE.LINK);
+    MPrivilege readAllConPriv = new MPrivilege(allConnector,SqoopActionConstant.READ, false);
+    MPrivilege readAllLinkPriv = new MPrivilege(allLink,SqoopActionConstant.READ, false);
+
+    client.grantRole(Lists.newArrayList(role2), Lists.newArrayList(group2));
+    client.grantPrivilege(Lists.newArrayList(new MPrincipal(role2.getName(), MPrincipal.TYPE.ROLE)),
+        Lists.newArrayList(writePrivilege, readAllConPriv, readAllLinkPriv));
+
+    /**
+     * ADMIN_USER grant all privilege on server SQOOP_SERVER_NAME to role3
+     */
+    MRole role3 = new MRole(ROLE3);
+    MPrincipal group3 = new MPrincipal(GROUP3, MPrincipal.TYPE.GROUP);
+    MPrivilege allPrivilege = new MPrivilege(sqoopServer1,SqoopActionConstant.ALL_NAME, false);
+    client.createRole(role3);
+    client.grantRole(Lists.newArrayList(role3), Lists.newArrayList(group3));
+    client.grantPrivilege(Lists.newArrayList(new MPrincipal(role3.getName(), MPrincipal.TYPE.ROLE)),
+        Lists.newArrayList(allPrivilege));
+
+    /**
+     * user1 has only the read privilege on server SQOOP_SERVER_NAME to role1,
+     * so user1 can show connector, link and jobs. The user1 can't update the link and
+     * job
+     */
+    client = sqoopServerRunner.getSqoopClient(USER1);
+    try {
+      // show connector
+      assertTrue(client.getConnector("generic-jdbc-connector") != null);
+      assertTrue(client.getConnector("hdfs-connector") != null);
+      assertTrue(client.getConnectors().size() > 0);
+      // show link
+      assertTrue(client.getLink(hdfsLink.getPersistenceId()) != null);
+      assertTrue(client.getLink(rdbmsLink.getPersistenceId()) != null);
+      assertTrue(client.getLinks().size() == 2);
+      // show job
+      assertTrue(client.getJob(job1.getPersistenceId()) != null);
+      assertTrue(client.getJobs().size() == 1);
+    } catch (Exception e) {
+      fail("unexpected Authorization exception happend");
+    }
+    // user1 can't update link and job
+    try {
+      hdfsLink.setName("hdfs1_update_user1");
+      client.updateLink(hdfsLink);
+      fail("expected Authorization exception happend");
+    } catch (Exception e) {
+      assertCausedMessage(e, SecurityError.AUTH_0014.getMessage());
+    }
+
+    try {
+      job1.setName("job1_update_user1");
+      client.updateJob(job1);
+      fail("expected Authorization exception happend");
+    } catch (Exception e) {
+      assertCausedMessage(e, SecurityError.AUTH_0014.getMessage());
+    }
+
+    /**
+     * user2 has the write privilege on server SQOOP_SERVER_NAME to role2. In order to update
link and job,
+     * user2 also has the read privilege on connector all and link all
+     * user2 can update link and jobs. The user2 can't show job
+     */
+    client = sqoopServerRunner.getSqoopClient(USER2);
+    try {
+      // update link and job
+      hdfsLink.setName("hdfs1_update_user2");
+      client.updateLink(hdfsLink);
+      job1.setName("job1_update_user2");
+      client.updateJob(job1);
+    } catch (Exception e) {
+      fail("unexpected Authorization exception happend");
+    }
+    // user2 can't show job
+    assertTrue(client.getJobs().size() == 0);
+
+    /**
+     * user3 has the all privilege on server SQOOP_SERVER_NAME to role3.
+     * user3 can do any operation on any sqoop resource
+     */
+    client = sqoopServerRunner.getSqoopClient(USER3);
+    try {
+      // show connector
+      assertTrue(client.getConnector("generic-jdbc-connector") != null);
+      assertTrue(client.getConnector("hdfs-connector") != null);
+      assertTrue(client.getConnectors().size() > 0);
+      // show link
+      assertTrue(client.getLink(hdfsLink.getPersistenceId()) != null);
+      assertTrue(client.getLink(rdbmsLink.getPersistenceId()) != null);
+      assertTrue(client.getLinks().size() == 2);
+      // show job
+      assertTrue(client.getJob(job1.getPersistenceId()) != null);
+      assertTrue(client.getJobs().size() == 1);
+      // update link
+      hdfsLink.setName("hdfs1_update_user3");
+      client.updateLink(hdfsLink);
+      // update job
+      job1.setName("job1_update_user3");
+      client.updateJob(job1);
+    } catch (Exception e) {
+      fail("unexpected Authorization exception happend");
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/98761811/sentry-tests/sentry-tests-sqoop/src/test/java/org/apache/sentry/tests/e2e/sqoop/TestShowPrivilege.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-sqoop/src/test/java/org/apache/sentry/tests/e2e/sqoop/TestShowPrivilege.java
b/sentry-tests/sentry-tests-sqoop/src/test/java/org/apache/sentry/tests/e2e/sqoop/TestShowPrivilege.java
new file mode 100644
index 0000000..609239f
--- /dev/null
+++ b/sentry-tests/sentry-tests-sqoop/src/test/java/org/apache/sentry/tests/e2e/sqoop/TestShowPrivilege.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sentry.tests.e2e.sqoop;
+
+import org.apache.sentry.core.model.sqoop.SqoopActionConstant;
+import org.apache.sentry.sqoop.SentrySqoopError;
+import org.apache.sqoop.client.SqoopClient;
+import org.apache.sqoop.model.MPrincipal;
+import org.apache.sqoop.model.MPrivilege;
+import org.apache.sqoop.model.MResource;
+import org.apache.sqoop.model.MRole;
+import org.junit.Test;
+
+import com.google.common.collect.Lists;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+public class TestShowPrivilege extends AbstractSqoopSentryTestBase {
+
+  @Test
+  public void testNotSupportShowOnUser() throws Exception {
+    SqoopClient client = sqoopServerRunner.getSqoopClient(ADMIN_USER);
+    MPrincipal user1 = new MPrincipal("not_support_user1", MPrincipal.TYPE.USER);
+    MResource resource1 = new MResource("all", MResource.TYPE.CONNECTOR);
+    try {
+      client.getPrivilegesByPrincipal(user1, resource1);
+      fail("expected not support exception happend");
+    } catch (Exception e) {
+      assertCausedMessage(e, SentrySqoopError.SHOW_PRIVILEGE_NOT_SUPPORTED_FOR_PRINCIPAL);
+    }
+  }
+
+  @Test
+  public void testNotSupportShowOnGroup() throws Exception {
+    SqoopClient client = sqoopServerRunner.getSqoopClient(ADMIN_USER);
+    MPrincipal group1 = new MPrincipal("not_support_group1", MPrincipal.TYPE.GROUP);
+    MResource resource1 = new MResource("all", MResource.TYPE.CONNECTOR);
+    try {
+      client.getPrivilegesByPrincipal(group1, resource1);
+      fail("expected not support exception happend");
+    } catch (Exception e) {
+      assertCausedMessage(e, SentrySqoopError.SHOW_PRIVILEGE_NOT_SUPPORTED_FOR_PRINCIPAL);
+    }
+  }
+
+  @Test
+  public void testShowPrivileges() throws Exception {
+    /**
+     * user1 belongs to group group1
+     * admin user grant role role1 to group group1
+     * admin user grant read privilege on connector all to role role1
+     */
+    SqoopClient client = sqoopServerRunner.getSqoopClient(ADMIN_USER);
+    MRole role1 = new MRole(ROLE1);
+    MPrincipal group1Princ = new MPrincipal(GROUP1, MPrincipal.TYPE.GROUP);
+    MPrincipal role1Princ = new MPrincipal(ROLE1, MPrincipal.TYPE.ROLE);
+    MResource allConnector = new MResource(SqoopActionConstant.ALL, MResource.TYPE.CONNECTOR);
+    MPrivilege readPriv = new MPrivilege(allConnector, SqoopActionConstant.READ, false);
+    client.createRole(role1);
+    client.grantRole(Lists.newArrayList(role1), Lists.newArrayList(group1Princ));
+    client.grantPrivilege(Lists.newArrayList(role1Princ), Lists.newArrayList(readPriv));
+
+    // user1 show privilege on role1
+    client = sqoopServerRunner.getSqoopClient(USER1);
+    assertTrue(client.getPrivilegesByPrincipal(role1Princ, allConnector).size() == 1);
+
+    // user2 can't show privilege on role1, because user2 doesn't belong to role1
+    client = sqoopServerRunner.getSqoopClient(USER2);
+    try {
+      client.getPrivilegesByPrincipal(role1Princ, allConnector);
+      fail("expected SentryAccessDeniedException happend");
+    } catch (Exception e) {
+      assertCausedMessage(e, "SentryAccessDeniedException");
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/98761811/sentry-tests/sentry-tests-sqoop/src/test/java/org/apache/sentry/tests/e2e/sqoop/TomcatSqoopRunner.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-sqoop/src/test/java/org/apache/sentry/tests/e2e/sqoop/TomcatSqoopRunner.java
b/sentry-tests/sentry-tests-sqoop/src/test/java/org/apache/sentry/tests/e2e/sqoop/TomcatSqoopRunner.java
new file mode 100644
index 0000000..0d50574
--- /dev/null
+++ b/sentry-tests/sentry-tests-sqoop/src/test/java/org/apache/sentry/tests/e2e/sqoop/TomcatSqoopRunner.java
@@ -0,0 +1,320 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sentry.tests.e2e.sqoop;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotSame;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.log4j.Logger;
+import org.apache.sqoop.client.SqoopClient;
+import org.apache.sqoop.common.test.db.DatabaseProvider;
+import org.apache.sqoop.common.test.db.DatabaseProviderFactory;
+import org.apache.sqoop.common.test.db.TableName;
+import org.apache.sqoop.common.test.utils.NetworkUtils;
+import org.apache.sqoop.model.MConfigList;
+import org.apache.sqoop.model.MJob;
+import org.apache.sqoop.model.MLink;
+import org.apache.sqoop.model.MPersistableEntity;
+import org.apache.sqoop.test.minicluster.SqoopMiniCluster;
+import org.apache.sqoop.validation.Status;
+import org.codehaus.cargo.container.ContainerType;
+import org.codehaus.cargo.container.InstalledLocalContainer;
+import org.codehaus.cargo.container.configuration.ConfigurationType;
+import org.codehaus.cargo.container.configuration.LocalConfiguration;
+import org.codehaus.cargo.container.deployable.WAR;
+import org.codehaus.cargo.container.installer.Installer;
+import org.codehaus.cargo.container.installer.ZipURLInstaller;
+import org.codehaus.cargo.container.property.GeneralPropertySet;
+import org.codehaus.cargo.container.property.ServletPropertySet;
+import org.codehaus.cargo.container.tomcat.TomcatPropertySet;
+import org.codehaus.cargo.generic.DefaultContainerFactory;
+import org.codehaus.cargo.generic.configuration.DefaultConfigurationFactory;
+
+import com.google.common.base.Joiner;
+
+public class TomcatSqoopRunner {
+  private static final Logger LOG = Logger.getLogger(TomcatSqoopRunner.class);
+  private SqoopServerEnableSentry server;
+  private DatabaseProvider provider;
+  private String temporaryPath;
+
+  public TomcatSqoopRunner(String temporaryPath, String serverName, String sentrySite)
+      throws Exception {
+    this.temporaryPath = temporaryPath;
+    this.server = new SqoopServerEnableSentry(temporaryPath, serverName, sentrySite);
+    this.provider = DatabaseProviderFactory.getProvider(System.getProperties());
+  }
+
+  public void start() throws Exception {
+    server.start();
+    provider.start();
+  }
+
+  public void stop() throws Exception {
+    server.stop();
+    provider.stop();
+  }
+
+
+  /**
+   * create link.
+   *
+   * With asserts to make sure that it was created correctly.
+   * @param sqoopClient
+   * @param link
+   */
+  public void saveLink(SqoopClient client, MLink link) {
+    assertEquals(Status.OK, client.saveLink(link));
+    assertNotSame(MPersistableEntity.PERSISTANCE_ID_DEFAULT, link.getPersistenceId());
+  }
+
+  /**
+   * create link.
+   *
+   * With asserts to make sure that it was created correctly.
+   * @param sqoopClient
+   * @param link
+   */
+  public void updateLink(SqoopClient client, MLink link) {
+    assertEquals(Status.OK, client.updateLink(link));
+    assertNotSame(MPersistableEntity.PERSISTANCE_ID_DEFAULT, link.getPersistenceId());
+  }
+
+  /**
+   * Create job.
+   *
+   * With asserts to make sure that it was created correctly.
+   *
+   * @param job
+   */
+  public void saveJob(SqoopClient client, MJob job) {
+    assertEquals(Status.OK, client.saveJob(job));
+    assertNotSame(MPersistableEntity.PERSISTANCE_ID_DEFAULT, job.getPersistenceId());
+  }
+
+  /**
+   * fill link.
+   *
+   * With asserts to make sure that it was filled correctly.
+   *
+   * @param link
+   */
+  public void fillHdfsLink(MLink link) {
+    MConfigList configs = link.getConnectorLinkConfig();
+    configs.getStringInput("linkConfig.confDir").setValue(server.getConfigurationPath());
+  }
+
+  /**
+   * Fill link config based on currently active provider.
+   *
+   * @param link MLink object to fill
+   */
+  public void fillRdbmsLinkConfig(MLink link) {
+    MConfigList configs = link.getConnectorLinkConfig();
+    configs.getStringInput("linkConfig.jdbcDriver").setValue(provider.getJdbcDriver());
+    configs.getStringInput("linkConfig.connectionString").setValue(provider.getConnectionUrl());
+    configs.getStringInput("linkConfig.username").setValue(provider.getConnectionUsername());
+    configs.getStringInput("linkConfig.password").setValue(provider.getConnectionPassword());
+  }
+
+  public void fillHdfsFromConfig(MJob job) {
+    MConfigList fromConfig = job.getFromJobConfig();
+    fromConfig.getStringInput("fromJobConfig.inputDirectory").setValue(temporaryPath + "/output");
+  }
+
+  public void fillRdbmsToConfig(MJob job) {
+    MConfigList toConfig = job.getToJobConfig();
+    toConfig.getStringInput("toJobConfig.tableName").setValue(provider.
+        escapeTableName(new TableName(getClass().getSimpleName()).getTableName()));
+  }
+
+  /**
+   * get a sqoopClient for specific user
+   * @param user
+   */
+  public SqoopClient getSqoopClient(String user) {
+    setAuthenticationUser(user);
+    return new SqoopClient(server.getServerUrl());
+  }
+
+  /**
+   * Set the mock user in the Sqoop simple authentication
+   * @param user
+   */
+  private void setAuthenticationUser(String user) {
+    System.setProperty("user.name", user);
+  }
+
+  private static class SqoopServerEnableSentry extends SqoopMiniCluster {
+    private static final String WAR_PATH = "thirdparty/sqoop.war";
+    private static final String TOMCAT_PATH = "thirdparty/apache-tomcat-6.0.36.zip";
+
+    private InstalledLocalContainer container = null;
+    private Integer port;
+    private Integer ajpPort;
+    private String sentrySite;
+    private String serverName;
+
+    SqoopServerEnableSentry(String temporaryPath, String serverName, String sentrySite)
+        throws Exception {
+      super(temporaryPath);
+      this.serverName = serverName;
+      this.sentrySite = sentrySite;
+      // Random port
+      this.port = NetworkUtils.findAvailablePort();
+      this.ajpPort = NetworkUtils.findAvailablePort();
+    }
+
+    @Override
+    public Map<String, String> getSecurityConfiguration() {
+      Map<String, String> properties = new HashMap<String, String>();
+      configureAuthentication(properties);
+      configureSentryAuthorization(properties);
+      return properties;
+    }
+
+    private void configureAuthentication(Map<String, String> properties) {
+      /** Simple Authentication */
+      properties.put("org.apache.sqoop.authentication.type", "SIMPLE");
+      properties.put("org.apache.sqoop.authentication.handler",
+          "org.apache.sqoop.security.SimpleAuthenticationHandler");
+    }
+
+    private void configureSentryAuthorization(Map<String, String> properties) {
+      properties.put("org.apache.sqoop.security.authorization.handler",
+          "org.apache.sentry.sqoop.authz.SentryAuthorizationHander");
+      properties.put("org.apache.sqoop.security.authorization.access_controller",
+          "org.apache.sentry.sqoop.authz.SentryAccessController");
+      properties.put("org.apache.sqoop.security.authorization.validator",
+          "org.apache.sentry.sqoop.authz.SentryAuthorizationValidator");
+      properties.put("org.apache.sqoop.security.authorization.server_name", serverName);
+      properties.put("sentry.sqoop.site.url", sentrySite);
+      /** set Sentry related jars into classpath */
+      List<String> extraClassPath = new LinkedList<String>();
+      for (String jar : System.getProperty("java.class.path").split(":")) {
+        if ((jar.contains("sentry") || jar.contains("shiro-core") || jar.contains("libthrift"))
+            && jar.endsWith("jar")) {
+          extraClassPath.add(jar);
+        }
+      }
+      properties.put("org.apache.sqoop.classpath.extra",Joiner.on(":").join(extraClassPath));
+    }
+
+    @Override
+    public void start() throws Exception {
+      // Container has already been started
+      if (container != null) {
+        return;
+      }
+      prepareTemporaryPath();
+
+      // Source: http://cargo.codehaus.org/Functional+testing
+      String tomcatPath = getTemporaryPath() + "/tomcat";
+      String extractPath = tomcatPath + "/extract";
+      String confPath = tomcatPath + "/conf";
+
+      Installer installer = new ZipURLInstaller(new File(TOMCAT_PATH).toURI().toURL(), null,
extractPath);
+      installer.install();
+
+      LocalConfiguration configuration = (LocalConfiguration) new DefaultConfigurationFactory()
+          .createConfiguration("tomcat6x", ContainerType.INSTALLED, ConfigurationType.STANDALONE,
+              confPath);
+      container = (InstalledLocalContainer) new DefaultContainerFactory().createContainer("tomcat6x",
+          ContainerType.INSTALLED, configuration);
+
+      // Set home to our installed tomcat instance
+      container.setHome(installer.getHome());
+
+      // Store tomcat logs into file as they are quite handy for debugging
+      container.setOutput(getTemporaryPath() + "/log/tomcat.log");
+
+      // Propagate system properties to the container
+      Map<String, String> map = new HashMap<String, String>((Map) System.getProperties());
+      container.setSystemProperties(map);
+
+      // Propagate Hadoop jars to the container classpath
+      // In real world, they would be installed manually by user
+      List<String> extraClassPath = new LinkedList<String>();
+      String[] classpath = System.getProperty("java.class.path").split(":");
+      for (String jar : classpath) {
+        if (jar.contains("hadoop-") || // Hadoop jars
+            jar.contains("hive-") || // Hive jars
+            jar.contains("commons-") || // Apache Commons libraries
+            jar.contains("httpcore-") || // Apache Http Core libraries
+            jar.contains("httpclient-") || // Apache Http Client libraries
+            jar.contains("htrace-") || // htrace-core libraries, new added in
+                                       // Hadoop 2.6.0
+            jar.contains("zookeeper-") || // zookeeper libraries, new added in
+                                          // Hadoop 2.6.0
+            jar.contains("curator-") || // curator libraries, new added in Hadoop
+                                        // 2.6.0
+            jar.contains("log4j-") || // Log4j
+            jar.contains("slf4j-") || // Slf4j
+            jar.contains("jackson-") || // Jackson
+            jar.contains("derby") || // Derby drivers
+            jar.contains("avro-") || // Avro
+            jar.contains("parquet-") || // Parquet
+            jar.contains("mysql") || // MySQL JDBC driver
+            jar.contains("postgre") || // PostgreSQL JDBC driver
+            jar.contains("oracle") || // Oracle driver
+            jar.contains("terajdbc") || // Teradata driver
+            jar.contains("tdgs") || // Teradata driver
+            jar.contains("nzjdbc") || // Netezza driver
+            jar.contains("sqljdbc") || // Microsoft SQL Server driver
+            jar.contains("libfb303") || // Facebook thrift lib
+            jar.contains("datanucleus-") || // Data nucleus libs
+            jar.contains("google") // Google libraries (guava, ...)
+        ) {
+          extraClassPath.add(jar);
+        }
+      }
+      container.setExtraClasspath(extraClassPath.toArray(new String[extraClassPath.size()]));
+
+      // Finally deploy Sqoop server war file
+      configuration.addDeployable(new WAR(WAR_PATH));
+      configuration.setProperty(ServletPropertySet.PORT, port.toString());
+      configuration.setProperty(TomcatPropertySet.AJP_PORT, ajpPort.toString());
+      //configuration.setProperty(GeneralPropertySet.JVMARGS, "\"-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=8006\"");
+      LOG.info("Tomcat extract path: " + extractPath);
+      LOG.info("Tomcat home path: " + installer.getHome());
+      LOG.info("Tomcat config home path: " + confPath);
+      LOG.info("Starting tomcat server on port " + port);
+      container.start();
+    }
+
+    @Override
+    public void stop() throws Exception {
+      if (container != null) {
+        container.stop();
+      }
+    }
+
+    /**
+     * Return server URL.
+     */
+    public String getServerUrl() {
+      // We're not doing any changes, so return default URL
+      return "http://localhost:" + port + "/sqoop/";
+    }
+  }
+}


Mime
View raw message