sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jar...@apache.org
Subject sqoop git commit: SQOOP-1731: Sqoop2: Generic JDBC upgrade from 1.99.3 to 1.99.4
Date Fri, 14 Nov 2014 14:27:33 GMT
Repository: sqoop
Updated Branches:
  refs/heads/sqoop2 bddc9874d -> af8571b64


SQOOP-1731: Sqoop2: Generic JDBC upgrade from 1.99.3 to 1.99.4

(Abraham Elmahre via Jarek Jarcec Cecho)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/af8571b6
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/af8571b6
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/af8571b6

Branch: refs/heads/sqoop2
Commit: af8571b6441eb61310adecac117056ae380b6b15
Parents: bddc987
Author: Jarek Jarcec Cecho <jarcec@apache.org>
Authored: Fri Nov 14 06:27:00 2014 -0800
Committer: Jarek Jarcec Cecho <jarcec@apache.org>
Committed: Fri Nov 14 06:27:00 2014 -0800

----------------------------------------------------------------------
 .../jdbc/GenericJdbcConnectorUpgrader.java      |  70 ++++++++
 .../jdbc/TestGenericJdbcConnectorUpgrader.java  | 180 +++++++++++++++++++
 2 files changed, 250 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/af8571b6/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnectorUpgrader.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnectorUpgrader.java
b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnectorUpgrader.java
index fb92a39..6e85c03 100644
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnectorUpgrader.java
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnectorUpgrader.java
@@ -20,25 +20,95 @@ package org.apache.sqoop.connector.jdbc;
 
 import org.apache.sqoop.configurable.ConfigurableUpgradeUtil;
 import org.apache.sqoop.connector.spi.ConnectorConfigurableUpgrader;
+import org.apache.sqoop.model.MConfig;
 import org.apache.sqoop.model.MFromConfig;
+import org.apache.sqoop.model.MInput;
 import org.apache.sqoop.model.MLinkConfig;
 import org.apache.sqoop.model.MToConfig;
 
+import java.util.HashMap;
+import java.util.Map;
+
 // NOTE: All config types have the similar upgrade path at this point
 public class GenericJdbcConnectorUpgrader extends ConnectorConfigurableUpgrader {
 
+  private static final String JOB_CONFIGURATION_FORM_NAME = "table";
+  private static final String CONNECTION_CONFIGURATION_FORM_NAME = "connection";
+  private static final Map<String, String> CONNECTION_TO_LINK_CONFIG_INPUT_MAP;
+  private static final Map<String, String> IMPORT_JOB_TABLE_TO_FROM_CONFIG_INPUT_MAP;
+  private static final Map<String, String> EXPORT_JOB_TABLE_TO_TO_CONFIG_INPUT_MAP;
+
+  static {
+    CONNECTION_TO_LINK_CONFIG_INPUT_MAP = new HashMap<String, String>();
+    CONNECTION_TO_LINK_CONFIG_INPUT_MAP.put(CONNECTION_CONFIGURATION_FORM_NAME + ".jdbcDriver",
"linkConfig.jdbcDriver");
+    CONNECTION_TO_LINK_CONFIG_INPUT_MAP.put(CONNECTION_CONFIGURATION_FORM_NAME + ".connectionString",
"linkConfig.connectionString");
+    CONNECTION_TO_LINK_CONFIG_INPUT_MAP.put(CONNECTION_CONFIGURATION_FORM_NAME + ".username",
"linkConfig.username");
+    CONNECTION_TO_LINK_CONFIG_INPUT_MAP.put(CONNECTION_CONFIGURATION_FORM_NAME + ".password",
"linkConfig.password");
+    CONNECTION_TO_LINK_CONFIG_INPUT_MAP.put(CONNECTION_CONFIGURATION_FORM_NAME + ".jdbcProperties",
"linkConfig.jdbcProperties");
+
+    IMPORT_JOB_TABLE_TO_FROM_CONFIG_INPUT_MAP = new HashMap<String, String>();
+    IMPORT_JOB_TABLE_TO_FROM_CONFIG_INPUT_MAP.put(JOB_CONFIGURATION_FORM_NAME + ".schemaName",
"fromJobConfig.schemaName");
+    IMPORT_JOB_TABLE_TO_FROM_CONFIG_INPUT_MAP.put(JOB_CONFIGURATION_FORM_NAME + ".tableName",
"fromJobConfig.tableName");
+    IMPORT_JOB_TABLE_TO_FROM_CONFIG_INPUT_MAP.put(JOB_CONFIGURATION_FORM_NAME + ".sql", "fromJobConfig.sql");
+    IMPORT_JOB_TABLE_TO_FROM_CONFIG_INPUT_MAP.put(JOB_CONFIGURATION_FORM_NAME + ".columns",
"fromJobConfig.columns");
+    IMPORT_JOB_TABLE_TO_FROM_CONFIG_INPUT_MAP.put(JOB_CONFIGURATION_FORM_NAME + ".partitionColumn",
"fromJobConfig.partitionColumn");
+    IMPORT_JOB_TABLE_TO_FROM_CONFIG_INPUT_MAP.put(JOB_CONFIGURATION_FORM_NAME + ".partitionColumnNull",
"fromJobConfig.allowNullValueInPartitionColumn");
+    IMPORT_JOB_TABLE_TO_FROM_CONFIG_INPUT_MAP.put(JOB_CONFIGURATION_FORM_NAME + ".boundaryQuery",
"fromJobConfig.boundaryQuery");
+
+    EXPORT_JOB_TABLE_TO_TO_CONFIG_INPUT_MAP = new HashMap<String, String>();
+    EXPORT_JOB_TABLE_TO_TO_CONFIG_INPUT_MAP.put(JOB_CONFIGURATION_FORM_NAME + ".schemaName",
"toJobConfig.schemaName");
+    EXPORT_JOB_TABLE_TO_TO_CONFIG_INPUT_MAP.put(JOB_CONFIGURATION_FORM_NAME + ".tableName",
"toJobConfig.tableName");
+    EXPORT_JOB_TABLE_TO_TO_CONFIG_INPUT_MAP.put(JOB_CONFIGURATION_FORM_NAME + ".sql", "toJobConfig.sql");
+    EXPORT_JOB_TABLE_TO_TO_CONFIG_INPUT_MAP.put(JOB_CONFIGURATION_FORM_NAME + ".columns",
"toJobConfig.columns");
+    EXPORT_JOB_TABLE_TO_TO_CONFIG_INPUT_MAP.put(JOB_CONFIGURATION_FORM_NAME + ".stageTableName",
"toJobConfig.stageTableName");
+    EXPORT_JOB_TABLE_TO_TO_CONFIG_INPUT_MAP.put(JOB_CONFIGURATION_FORM_NAME + ".clearStageTable",
"toJobConfig.shouldClearStageTable");
+  }
+
   @Override
   public void upgradeLinkConfig(MLinkConfig original, MLinkConfig upgradeTarget) {
+    // Upgrade from 1.99.3 to 1.99.4
+    for (MConfig config : original.getConfigs()) {
+      if (config.getName().equals(CONNECTION_CONFIGURATION_FORM_NAME)) {
+        for (MInput originalInput : config.getInputs()) {
+          String inputName = CONNECTION_TO_LINK_CONFIG_INPUT_MAP.get(originalInput.getName());
+          MInput input = upgradeTarget.getInput(inputName);
+          input.setValue(originalInput.getValue());
+        }
+      }
+    }
+
     ConfigurableUpgradeUtil.doUpgrade(original.getConfigs(), upgradeTarget.getConfigs());
   }
 
   @Override
   public void upgradeFromJobConfig(MFromConfig original, MFromConfig upgradeTarget) {
+    // Upgrade from 1.99.3 to 1.99.4
+    for (MConfig config : original.getConfigs()) {
+      if (config.getName().equals(JOB_CONFIGURATION_FORM_NAME)) {
+        for (MInput originalInput : config.getInputs()) {
+          String inputName = IMPORT_JOB_TABLE_TO_FROM_CONFIG_INPUT_MAP.get(originalInput.getName());
+          MInput input = upgradeTarget.getInput(inputName);
+          input.setValue(originalInput.getValue());
+        }
+      }
+    }
+
     ConfigurableUpgradeUtil.doUpgrade(original.getConfigs(), upgradeTarget.getConfigs());
   }
 
   @Override
   public void upgradeToJobConfig(MToConfig original, MToConfig upgradeTarget) {
+    // Upgrade from 1.99.3 to 1.99.4
+    for (MConfig config : original.getConfigs()) {
+      if (config.getName().equals(JOB_CONFIGURATION_FORM_NAME)) {
+        for (MInput originalInput : config.getInputs()) {
+          String inputName = EXPORT_JOB_TABLE_TO_TO_CONFIG_INPUT_MAP.get(originalInput.getName());
+          MInput input = upgradeTarget.getInput(inputName);
+          input.setValue(originalInput.getValue());
+        }
+      }
+    }
+
     ConfigurableUpgradeUtil.doUpgrade(original.getConfigs(), upgradeTarget.getConfigs());
   }
 }

http://git-wip-us.apache.org/repos/asf/sqoop/blob/af8571b6/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestGenericJdbcConnectorUpgrader.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestGenericJdbcConnectorUpgrader.java
b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestGenericJdbcConnectorUpgrader.java
new file mode 100644
index 0000000..d6fe504
--- /dev/null
+++ b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestGenericJdbcConnectorUpgrader.java
@@ -0,0 +1,180 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.connector.jdbc;
+
+import org.apache.sqoop.connector.jdbc.configuration.FromJobConfiguration;
+import org.apache.sqoop.connector.jdbc.configuration.LinkConfiguration;
+import org.apache.sqoop.connector.jdbc.configuration.ToJobConfiguration;
+import org.apache.sqoop.model.ConfigUtils;
+import org.apache.sqoop.model.MBooleanInput;
+import org.apache.sqoop.model.MConfig;
+import org.apache.sqoop.model.MFromConfig;
+import org.apache.sqoop.model.MInput;
+import org.apache.sqoop.model.MLinkConfig;
+import org.apache.sqoop.model.MStringInput;
+import org.apache.sqoop.model.MToConfig;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.LinkedList;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Test upgrader.
+ */
+public class TestGenericJdbcConnectorUpgrader {
+
+  private GenericJdbcConnectorUpgrader upgrader;
+
+  @Before
+  public void setUp() {
+    upgrader = new GenericJdbcConnectorUpgrader();
+  }
+
+  @Test
+  public void testFromConfig() {
+    // No upgrade
+    MFromConfig originalConfigs = new MFromConfig(ConfigUtils.toConfigs(FromJobConfiguration.class));
+    MFromConfig newConfigs = new MFromConfig(ConfigUtils.toConfigs(FromJobConfiguration.class));
+    originalConfigs.getInput("fromJobConfig.schemaName").setValue("test-schema");
+    originalConfigs.getInput("fromJobConfig.tableName").setValue("test-tableName");
+    originalConfigs.getInput("fromJobConfig.sql").setValue("test-sql");
+    originalConfigs.getInput("fromJobConfig.columns").setValue("test-columns");
+    originalConfigs.getInput("fromJobConfig.partitionColumn").setValue("test-partitionColumn");
+    originalConfigs.getInput("fromJobConfig.allowNullValueInPartitionColumn").setValue("test-allowNullValueInPartitionColumn");
+    upgrader.upgradeFromJobConfig(originalConfigs, newConfigs);
+    assertEquals(originalConfigs, newConfigs);
+    assertEquals("test-schema", newConfigs.getInput("fromJobConfig.schemaName").getValue());
+    assertEquals("test-tableName", newConfigs.getInput("fromJobConfig.tableName").getValue());
+    assertEquals("test-sql", newConfigs.getInput("fromJobConfig.sql").getValue());
+    assertEquals("test-columns", newConfigs.getInput("fromJobConfig.columns").getValue());
+    assertEquals("test-partitionColumn", newConfigs.getInput("fromJobConfig.partitionColumn").getValue());
+    assertEquals("test-allowNullValueInPartitionColumn", newConfigs.getInput("fromJobConfig.allowNullValueInPartitionColumn").getValue());
+
+    // 1.99.3 upgrade to 1.99.4
+    originalConfigs = new MFromConfig(new LinkedList<MConfig>());
+    newConfigs = new MFromConfig(ConfigUtils.toConfigs(FromJobConfiguration.class));
+    originalConfigs.getConfigs().add(new MConfig("table", new LinkedList<MInput<?>>()));
+    originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("table.schemaName",
false, (short)50));
+    originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("table.tableName",
false, (short)50));
+    originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("table.sql", false,
(short)2000));
+    originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("table.columns",
false, (short)50));
+    originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("table.partitionColumn",
false, (short)50));
+    originalConfigs.getConfigs().get(0).getInputs().add(new MBooleanInput("table.partitionColumnNull",
false));
+    originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("table.boundaryQuery",
false, (short)50));
+    originalConfigs.getInput("table.schemaName").setValue("test-schema");
+    originalConfigs.getInput("table.tableName").setValue("test-tableName");
+    originalConfigs.getInput("table.sql").setValue("test-sql");
+    originalConfigs.getInput("table.columns").setValue("test-columns");
+    originalConfigs.getInput("table.partitionColumn").setValue("test-partitionColumn");
+    originalConfigs.getInput("table.partitionColumnNull").setValue("test-allowNullValueInPartitionColumn");
+    upgrader.upgradeFromJobConfig(originalConfigs, newConfigs);
+    assertEquals("test-schema", newConfigs.getInput("fromJobConfig.schemaName").getValue());
+    assertEquals("test-tableName", newConfigs.getInput("fromJobConfig.tableName").getValue());
+    assertEquals("test-sql", newConfigs.getInput("fromJobConfig.sql").getValue());
+    assertEquals("test-columns", newConfigs.getInput("fromJobConfig.columns").getValue());
+    assertEquals("test-partitionColumn", newConfigs.getInput("fromJobConfig.partitionColumn").getValue());
+    assertEquals("test-allowNullValueInPartitionColumn", newConfigs.getInput("fromJobConfig.allowNullValueInPartitionColumn").getValue());
+  }
+
+  @Test
+  public void testToConfig() {
+    // No upgrade
+    MToConfig originalConfigs = new MToConfig(ConfigUtils.toConfigs(ToJobConfiguration.class));
+    MToConfig newConfigs = new MToConfig(ConfigUtils.toConfigs(ToJobConfiguration.class));
+    originalConfigs.getInput("toJobConfig.schemaName").setValue("test-schema");
+    originalConfigs.getInput("toJobConfig.tableName").setValue("test-tableName");
+    originalConfigs.getInput("toJobConfig.sql").setValue("test-sql");
+    originalConfigs.getInput("toJobConfig.columns").setValue("test-columns");
+    originalConfigs.getInput("toJobConfig.stageTableName").setValue("test-stageTableName");
+    originalConfigs.getInput("toJobConfig.shouldClearStageTable").setValue("test-shouldClearStageTable");
+    upgrader.upgradeToJobConfig(originalConfigs, newConfigs);
+    assertEquals(originalConfigs, newConfigs);
+    assertEquals("test-schema", newConfigs.getInput("toJobConfig.schemaName").getValue());
+    assertEquals("test-tableName", newConfigs.getInput("toJobConfig.tableName").getValue());
+    assertEquals("test-sql", newConfigs.getInput("toJobConfig.sql").getValue());
+    assertEquals("test-columns", newConfigs.getInput("toJobConfig.columns").getValue());
+    assertEquals("test-stageTableName", newConfigs.getInput("toJobConfig.stageTableName").getValue());
+    assertEquals("test-shouldClearStageTable", newConfigs.getInput("toJobConfig.shouldClearStageTable").getValue());
+
+    // 1.99.3 upgrade to 1.99.4
+    originalConfigs = new MToConfig(new LinkedList<MConfig>());
+    newConfigs = new MToConfig(ConfigUtils.toConfigs(ToJobConfiguration.class));
+    originalConfigs.getConfigs().add(new MConfig("table", new LinkedList<MInput<?>>()));
+    originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("table.schemaName",
false, (short)50));
+    originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("table.tableName",
false, (short)50));
+    originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("table.sql", false,
(short)2000));
+    originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("table.columns",
false, (short)50));
+    originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("table.stageTableName",
false, (short)50));
+    originalConfigs.getConfigs().get(0).getInputs().add(new MBooleanInput("table.clearStageTable",
false));
+    originalConfigs.getInput("table.schemaName").setValue("test-schema");
+    originalConfigs.getInput("table.tableName").setValue("test-tableName");
+    originalConfigs.getInput("table.sql").setValue("test-sql");
+    originalConfigs.getInput("table.columns").setValue("test-columns");
+    originalConfigs.getInput("table.stageTableName").setValue("test-stageTableName");
+    originalConfigs.getInput("table.clearStageTable").setValue("test-shouldClearStageTable");
+    upgrader.upgradeToJobConfig(originalConfigs, newConfigs);
+    assertEquals("test-schema", newConfigs.getInput("toJobConfig.schemaName").getValue());
+    assertEquals("test-tableName", newConfigs.getInput("toJobConfig.tableName").getValue());
+    assertEquals("test-sql", newConfigs.getInput("toJobConfig.sql").getValue());
+    assertEquals("test-columns", newConfigs.getInput("toJobConfig.columns").getValue());
+    assertEquals("test-stageTableName", newConfigs.getInput("toJobConfig.stageTableName").getValue());
+    assertEquals("test-shouldClearStageTable", newConfigs.getInput("toJobConfig.shouldClearStageTable").getValue());
+  }
+
+  @Test
+  public void testLinkConfig() {
+    // No upgrade
+    MLinkConfig originalConfigs = new MLinkConfig(ConfigUtils.toConfigs(LinkConfiguration.class));
+    MLinkConfig newConfigs = new MLinkConfig(ConfigUtils.toConfigs(LinkConfiguration.class));
+    originalConfigs.getInput("linkConfig.jdbcDriver").setValue("test-jdbcDriver");
+    originalConfigs.getInput("linkConfig.connectionString").setValue("test-connectionString");
+    originalConfigs.getInput("linkConfig.username").setValue("test-username");
+    originalConfigs.getInput("linkConfig.password").setValue("test-password");
+    originalConfigs.getInput("linkConfig.jdbcProperties").setValue("test-jdbcProperties");
+    upgrader.upgradeLinkConfig(originalConfigs, newConfigs);
+    assertEquals(originalConfigs, newConfigs);
+    assertEquals("test-jdbcDriver", newConfigs.getInput("linkConfig.jdbcDriver").getValue());
+    assertEquals("test-connectionString", newConfigs.getInput("linkConfig.connectionString").getValue());
+    assertEquals("test-username", newConfigs.getInput("linkConfig.username").getValue());
+    assertEquals("test-password", newConfigs.getInput("linkConfig.password").getValue());
+    assertEquals("test-jdbcProperties", newConfigs.getInput("linkConfig.jdbcProperties").getValue());
+
+    // 1.99.3 upgrade to 1.99.4
+    originalConfigs = new MLinkConfig(new LinkedList<MConfig>());
+    newConfigs = new MLinkConfig(ConfigUtils.toConfigs(LinkConfiguration.class));
+    originalConfigs.getConfigs().add(new MConfig("connection", new LinkedList<MInput<?>>()));
+    originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("connection.jdbcDriver",
false, (short)50));
+    originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("connection.connectionString",
false, (short)50));
+    originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("connection.username",
false, (short)2000));
+    originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("connection.password",
false, (short)50));
+    originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("connection.jdbcProperties",
false, (short)50));
+    originalConfigs.getInput("connection.jdbcDriver").setValue("test-jdbcDriver");
+    originalConfigs.getInput("connection.connectionString").setValue("test-connectionString");
+    originalConfigs.getInput("connection.username").setValue("test-username");
+    originalConfigs.getInput("connection.password").setValue("test-password");
+    originalConfigs.getInput("connection.jdbcProperties").setValue("test-jdbcProperties");
+    upgrader.upgradeLinkConfig(originalConfigs, newConfigs);
+    assertEquals("test-jdbcDriver", newConfigs.getInput("linkConfig.jdbcDriver").getValue());
+    assertEquals("test-connectionString", newConfigs.getInput("linkConfig.connectionString").getValue());
+    assertEquals("test-username", newConfigs.getInput("linkConfig.username").getValue());
+    assertEquals("test-password", newConfigs.getInput("linkConfig.password").getValue());
+    assertEquals("test-jdbcProperties", newConfigs.getInput("linkConfig.jdbcProperties").getValue());
+  }
+}


Mime
View raw message