sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a..@apache.org
Subject sqoop git commit: SQOOP-1821: Sqoop2: External connector loading
Date Tue, 20 Jan 2015 20:50:44 GMT
Repository: sqoop
Updated Branches:
  refs/heads/sqoop2 49f104bb5 -> 913521e1a


SQOOP-1821: Sqoop2: External connector loading

(Veena Basavaraj via Abraham Elmahrek)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/913521e1
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/913521e1
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/913521e1

Branch: refs/heads/sqoop2
Commit: 913521e1a93b2571d2c83b10c51609b54744eee9
Parents: 49f104b
Author: Abraham Elmahrek <abe@apache.org>
Authored: Tue Jan 20 12:50:15 2015 -0800
Committer: Abraham Elmahrek <abe@apache.org>
Committed: Tue Jan 20 12:50:15 2015 -0800

----------------------------------------------------------------------
 .../java/org/apache/sqoop/utils/ClassUtils.java |   2 +
 .../org/apache/sqoop/utils/TestClassUtils.java  |   2 +-
 .../sqoop/connector/ConnectorHandler.java       |   5 +-
 .../sqoop/connector/ConnectorManager.java       |   6 ++
 .../sqoop/connector/ConnectorManagerUtils.java  |  87 +++++++++++++++++--
 .../sqoop/core/ConfigurationConstants.java      |   7 ++
 .../connector/TestConnectorManagerUtils.java    |  64 ++++++++++++++
 core/src/test/resources/test-connector.jar      | Bin 0 -> 55938 bytes
 core/src/test/resources/test-non-connector.jar  | Bin 0 -> 19827 bytes
 dist/src/main/server/conf/sqoop.properties      |   8 +-
 .../test/minicluster/SqoopMiniCluster.java      |  13 +--
 11 files changed, 179 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/913521e1/common/src/main/java/org/apache/sqoop/utils/ClassUtils.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/utils/ClassUtils.java b/common/src/main/java/org/apache/sqoop/utils/ClassUtils.java
index 0be4d41..d6a8254 100644
--- a/common/src/main/java/org/apache/sqoop/utils/ClassUtils.java
+++ b/common/src/main/java/org/apache/sqoop/utils/ClassUtils.java
@@ -57,6 +57,8 @@ public final class ClassUtils {
           klass = ctxLoader.loadClass(className);
         } catch (ClassNotFoundException ex) {
           LOG.debug("Exception while load class: " + className, ex);
+          // wrapping it in runtime, to avoid chainging the signature of methods currently
invoking this method
+          throw new RuntimeException(ex);
         }
       }
     }

http://git-wip-us.apache.org/repos/asf/sqoop/blob/913521e1/common/src/test/java/org/apache/sqoop/utils/TestClassUtils.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/utils/TestClassUtils.java b/common/src/test/java/org/apache/sqoop/utils/TestClassUtils.java
index 161a1fa..58a27ba 100644
--- a/common/src/test/java/org/apache/sqoop/utils/TestClassUtils.java
+++ b/common/src/test/java/org/apache/sqoop/utils/TestClassUtils.java
@@ -29,7 +29,7 @@ import static org.testng.Assert.assertNull;
  */
 public class TestClassUtils {
 
-  @Test
+  @Test(expectedExceptions = Exception.class)
   public void testLoadClass() {
     assertNull(ClassUtils.loadClass("A"));
     assertEquals(A.class, ClassUtils.loadClass(A.class.getName()));

http://git-wip-us.apache.org/repos/asf/sqoop/blob/913521e1/core/src/main/java/org/apache/sqoop/connector/ConnectorHandler.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/connector/ConnectorHandler.java b/core/src/main/java/org/apache/sqoop/connector/ConnectorHandler.java
index 1919b4b..bfdb7b3 100644
--- a/core/src/main/java/org/apache/sqoop/connector/ConnectorHandler.java
+++ b/core/src/main/java/org/apache/sqoop/connector/ConnectorHandler.java
@@ -31,6 +31,7 @@ import org.apache.sqoop.model.MConnector;
 import org.apache.sqoop.model.MFromConfig;
 import org.apache.sqoop.model.MLinkConfig;
 import org.apache.sqoop.model.MToConfig;
+import org.apache.sqoop.utils.ClassUtils;
 
 public final class ConnectorHandler {
 
@@ -71,8 +72,8 @@ public final class ConnectorHandler {
 
     Class<?> connectorClass = null;
     try {
-      connectorClass = Class.forName(connectorClassName);
-    } catch (ClassNotFoundException ex) {
+      connectorClass = ClassUtils.loadClass(connectorClassName);
+    } catch (Exception ex) {
       throw new SqoopException(ConnectorError.CONN_0005,
               connectorClassName, ex);
     }

http://git-wip-us.apache.org/repos/asf/sqoop/blob/913521e1/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java b/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java
index b9d4d60..0907efb 100644
--- a/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java
+++ b/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java
@@ -27,6 +27,7 @@ import java.util.Map;
 import java.util.ResourceBundle;
 import java.util.Set;
 
+import org.apache.commons.lang.StringUtils;
 import org.apache.log4j.Logger;
 import org.apache.sqoop.common.SqoopException;
 import org.apache.sqoop.connector.spi.SqoopConnector;
@@ -158,6 +159,11 @@ public class ConnectorManager implements Reconfigurable {
       LOG.trace("Begin connector manager initialization");
     }
 
+    // add external connectors into the class path
+    // NOTE: class loading happens later in the ConnectorHandler
+    ConnectorManagerUtils.addExternalConnectorsJarsToClasspath(SqoopConfiguration.getInstance().getContext()
+        .getString(ConfigurationConstants.EXTERNAL_CONNECTOR_LOAD_PATH, StringUtils.EMPTY));
+
     List<URL> connectorConfigs = ConnectorManagerUtils.getConnectorConfigs();
 
     LOG.info("Connector config urls: " + connectorConfigs);

http://git-wip-us.apache.org/repos/asf/sqoop/blob/913521e1/core/src/main/java/org/apache/sqoop/connector/ConnectorManagerUtils.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/connector/ConnectorManagerUtils.java b/core/src/main/java/org/apache/sqoop/connector/ConnectorManagerUtils.java
index c7193ee..bd85e99 100644
--- a/core/src/main/java/org/apache/sqoop/connector/ConnectorManagerUtils.java
+++ b/core/src/main/java/org/apache/sqoop/connector/ConnectorManagerUtils.java
@@ -17,14 +17,22 @@
  */
 package org.apache.sqoop.connector;
 
+import org.apache.commons.lang.StringUtils;
 import org.apache.sqoop.common.SqoopException;
 import org.apache.sqoop.core.ConfigurationConstants;
 
+import java.io.File;
 import java.io.IOException;
+import java.net.MalformedURLException;
 import java.net.URL;
+import java.net.URLClassLoader;
 import java.util.ArrayList;
 import java.util.Enumeration;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
 
 /**
  * Utilities for ConnectorManager.
@@ -33,7 +41,7 @@ public class ConnectorManagerUtils {
 
   /**
    * Get a list of URLs of connectors that are installed.
-   * Check
+   *
    * @return List of URLs.
    */
   public static List<URL> getConnectorConfigs() {
@@ -41,9 +49,8 @@ public class ConnectorManagerUtils {
 
     try {
       // Check ConnectorManager classloader.
-      Enumeration<URL> appPathConfigs =
-          ConnectorManager.class.getClassLoader().getResources(
-              ConfigurationConstants.FILENAME_CONNECTOR_PROPERTIES);
+      Enumeration<URL> appPathConfigs = ConnectorManager.class.getClassLoader().getResources(
+          ConfigurationConstants.FILENAME_CONNECTOR_PROPERTIES);
       while (appPathConfigs.hasMoreElements()) {
         connectorConfigs.add(appPathConfigs.nextElement());
       }
@@ -51,8 +58,7 @@ public class ConnectorManagerUtils {
       // Check thread context classloader.
       ClassLoader ctxLoader = Thread.currentThread().getContextClassLoader();
       if (ctxLoader != null) {
-        Enumeration<URL> ctxPathConfigs = ctxLoader.getResources(
-            ConfigurationConstants.FILENAME_CONNECTOR_PROPERTIES);
+        Enumeration<URL> ctxPathConfigs = ctxLoader.getResources(ConfigurationConstants.FILENAME_CONNECTOR_PROPERTIES);
 
         while (ctxPathConfigs.hasMoreElements()) {
           URL configUrl = ctxPathConfigs.nextElement();
@@ -67,4 +73,73 @@ public class ConnectorManagerUtils {
 
     return connectorConfigs;
   }
+
+  public static Set<File> getConnectorJars(String path) {
+    if (StringUtils.isEmpty(path)) {
+      return null;
+    }
+    Set<File> jarFiles = new HashSet<File>();
+    File folder = new File(path);
+    if (folder.exists()) {
+      for (File file : folder.listFiles()) {
+        if (file.isDirectory()) {
+          jarFiles.addAll(getConnectorJars(file.getPath()));
+        }
+        if (file.getName().endsWith(".jar") && isConnectorJar(file)) {
+          jarFiles.add(file);
+        }
+      }
+    }
+    return jarFiles;
+  }
+
+  static boolean isConnectorJar(File file) {
+    try {
+      @SuppressWarnings("resource")
+      JarEntry entry = new JarFile(file).getJarEntry(ConfigurationConstants.FILENAME_CONNECTOR_PROPERTIES);
+      return entry != null;
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  public static void addExternalConnectorsJarsToClasspath(String path) {
+    if (StringUtils.isEmpty(path)) {
+      return;
+    }
+
+    ClassLoader currentThreadClassLoader = Thread.currentThread().getContextClassLoader();
+    if (currentThreadClassLoader != null) {
+
+      // Add the 'org.apache.sqoop.connector.external.loadpath' to the classpath
+      // Chain the current thread classloader
+      ExternalConnectorJarFileLoader connectorUrlClassLoader = new ExternalConnectorJarFileLoader(new
URL[] {},
+          currentThreadClassLoader);
+      // the property always holds a path to the folder containing the jars
+      Set<File> connectorJars = getConnectorJars(path);
+      if (connectorJars != null && !connectorJars.isEmpty()) {
+        for (File jar : connectorJars) {
+          connectorUrlClassLoader.addJarFile(jar.getPath());
+        }
+        // Replace the thread classloader- assuming there is permission to do so
+        Thread.currentThread().setContextClassLoader(connectorUrlClassLoader);
+      }
+    }
+  }
+
+  public static class ExternalConnectorJarFileLoader extends URLClassLoader {
+    public ExternalConnectorJarFileLoader(URL[] urls, ClassLoader parent) {
+      super(urls, parent);
+    }
+
+    public void addJarFile(String path) {
+      String urlPath = "jar:file://" + path + "!/";
+      try {
+        addURL(new URL(urlPath));
+      } catch (MalformedURLException e) {
+        throw new RuntimeException(e);
+      }
+    }
+
+  }
 }

http://git-wip-us.apache.org/repos/asf/sqoop/blob/913521e1/core/src/main/java/org/apache/sqoop/core/ConfigurationConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/core/ConfigurationConstants.java b/core/src/main/java/org/apache/sqoop/core/ConfigurationConstants.java
index f341108..49e5d6f 100644
--- a/core/src/main/java/org/apache/sqoop/core/ConfigurationConstants.java
+++ b/core/src/main/java/org/apache/sqoop/core/ConfigurationConstants.java
@@ -77,6 +77,13 @@ public final class ConfigurationConstants {
       "org.apache.sqoop.driver.autoupgrade";
 
   /**
+   # Support loading external connector jars only
+   # The loader will look for sqoopconnector.properties file in the jar before loading
+   # "/path/to/external/connectors/": Add all the connector JARs in the specified folder
+   */
+  public static final String EXTERNAL_CONNECTOR_LOAD_PATH = "org.apache.sqoop.connector.external.loadpath";
+
+  /**
    * Enable Sqoop App to kill Tomcat in case that it will fail to load.
    */
   public static final String KILL_TOMCAT_ON_FAILURE = "sqoop.kill_tomcat_on_load_failure";

http://git-wip-us.apache.org/repos/asf/sqoop/blob/913521e1/core/src/test/java/org/apache/sqoop/connector/TestConnectorManagerUtils.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/sqoop/connector/TestConnectorManagerUtils.java
b/core/src/test/java/org/apache/sqoop/connector/TestConnectorManagerUtils.java
new file mode 100644
index 0000000..18f8edb
--- /dev/null
+++ b/core/src/test/java/org/apache/sqoop/connector/TestConnectorManagerUtils.java
@@ -0,0 +1,64 @@
+package org.apache.sqoop.connector;
+
+import static org.testng.AssertJUnit.assertFalse;
+import static org.testng.AssertJUnit.assertTrue;
+import static org.testng.AssertJUnit.assertNull;
+import static org.testng.AssertJUnit.assertEquals;
+import org.apache.sqoop.utils.ClassUtils;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+import org.testng.AssertJUnit;
+
+import java.io.File;
+import java.net.URL;
+import java.util.List;
+import java.util.Set;
+
+public class TestConnectorManagerUtils {
+
+  private String workingDir;
+
+  @BeforeMethod(alwaysRun = true)
+  public void setUp() {
+    workingDir = System.getProperty("user.dir");
+  }
+
+  @Test
+  public void testGetConnectorJarsNullPath() {
+    Set<File> files = ConnectorManagerUtils.getConnectorJars(null);
+    assertNull(files);
+  }
+
+  @Test
+  public void testGetConnectorJarsNonNullPath() {
+    String path = workingDir + "/src/test/resources";
+    Set<File> files = ConnectorManagerUtils.getConnectorJars(path);
+    assertEquals(1, files.size());
+  }
+
+  @Test
+  public void testIsConnectorJar() {
+    String path = workingDir + "/src/test/resources/test-connector.jar";
+    File connectorJar = new File(path);
+    assertTrue(connectorJar.exists());
+    assertTrue(ConnectorManagerUtils.isConnectorJar(connectorJar));
+  }
+
+  @Test
+  public void testIsNotConnectorJar() {
+    String path = workingDir + "/src/test/resources/test-non-connector.jar";
+    File file = new File(path);
+    assertTrue(file.exists());
+    assertFalse(ConnectorManagerUtils.isConnectorJar(file));
+  }
+
+  @Test
+  public void testAddExternalConnectorJarToClasspath() {
+    String path = workingDir + "/src/test/resources";
+    ConnectorManagerUtils.addExternalConnectorsJarsToClasspath(path);
+    List<URL> urls = ConnectorManagerUtils.getConnectorConfigs();
+    assertEquals(1, urls.size());
+    ClassUtils.loadClass("org.apache.sqoop.connector.jdbc.GenericJdbcConnector");
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/913521e1/core/src/test/resources/test-connector.jar
----------------------------------------------------------------------
diff --git a/core/src/test/resources/test-connector.jar b/core/src/test/resources/test-connector.jar
new file mode 100644
index 0000000..fd3b116
Binary files /dev/null and b/core/src/test/resources/test-connector.jar differ

http://git-wip-us.apache.org/repos/asf/sqoop/blob/913521e1/core/src/test/resources/test-non-connector.jar
----------------------------------------------------------------------
diff --git a/core/src/test/resources/test-non-connector.jar b/core/src/test/resources/test-non-connector.jar
new file mode 100644
index 0000000..01f82ca
Binary files /dev/null and b/core/src/test/resources/test-non-connector.jar differ

http://git-wip-us.apache.org/repos/asf/sqoop/blob/913521e1/dist/src/main/server/conf/sqoop.properties
----------------------------------------------------------------------
diff --git a/dist/src/main/server/conf/sqoop.properties b/dist/src/main/server/conf/sqoop.properties
index 824c5eb..fbcb1fa 100755
--- a/dist/src/main/server/conf/sqoop.properties
+++ b/dist/src/main/server/conf/sqoop.properties
@@ -163,4 +163,10 @@ org.apache.sqoop.execution.engine=org.apache.sqoop.execution.mapreduce.Mapreduce
 #
 #org.apache.sqoop.security.authorization.handler=org.apache.sqoop.security.Authorization.DefaultAuthorizationHandler
 #org.apache.sqoop.security.authorization.access_controller=org.apache.sqoop.security.Authorization.DefaultAuthorizationAccessController
-#org.apache.sqoop.security.authorization.validator=org.apache.sqoop.security.Authorization.DefaultAuthorizationValidator
\ No newline at end of file
+#org.apache.sqoop.security.authorization.validator=org.apache.sqoop.security.Authorization.DefaultAuthorizationValidator
+
+
+# External connectors load path
+# "/path/to/external/connectors/": Add all the connector JARs in the specified folder
+#
+org.apache.sqoop.connector.external.loadpath=

http://git-wip-us.apache.org/repos/asf/sqoop/blob/913521e1/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java b/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java
index e4eecbf..ad45189 100644
--- a/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java
+++ b/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java
@@ -132,6 +132,7 @@ public abstract class SqoopMiniCluster {
     mapToProperties(sqoopProperties, getSecurityConfiguration());
     mapToProperties(sqoopProperties, getConnectorManagerConfiguration());
     mapToProperties(sqoopProperties, getDriverManagerConfiguration());
+    mapToProperties(sqoopProperties, getExternalConnectorLoadPathConfiguration());
 
     FileUtils.writeLines(f, sqoopProperties);
 
@@ -212,17 +213,19 @@ public abstract class SqoopMiniCluster {
 
   protected Map<String, String> getConnectorManagerConfiguration() {
     Map<String, String> properties = new HashMap<String, String>();
-
-    properties.put("org.apache.sqoop.connector.autoupgrade", "true");
-
+    properties.put(ConfigurationConstants.CONNECTOR_AUTO_UPGRADE, "true");
     return properties;
   }
 
   protected Map<String, String> getDriverManagerConfiguration() {
     Map<String, String> properties = new HashMap<String, String>();
+    properties.put(ConfigurationConstants.DRIVER_AUTO_UPGRADE, "true");
+    return properties;
+  }
 
-    properties.put("org.apache.sqoop.driver.autoupgrade", "true");
-
+  protected Map<String, String> getExternalConnectorLoadPathConfiguration() {
+    Map<String, String> properties = new HashMap<String, String>();
+    properties.put(ConfigurationConstants.EXTERNAL_CONNECTOR_LOAD_PATH, "");
     return properties;
   }
 }


Mime
View raw message