sqoop-commits mailing list archives

Site index · List index

Message view	« Date » · « Thread »
Top	« Date » · « Thread »
From	jar...@apache.org
Subject	[1/2] sqoop git commit: SQOOP-1804: Add editable and override attribute to inputs
Date	Thu, 05 Feb 2015 16:36:57 GMT
Repository: sqoop Updated Branches: refs/heads/sqoop2 433a42d7a -> 6fc50b08b http://git-wip-us.apache.org/repos/asf/sqoop/blob/6fc50b08/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestGenericJdbcConnectorUpgrader.java ---------------------------------------------------------------------- diff --git a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestGenericJdbcConnectorUpgrader.java b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestGenericJdbcConnectorUpgrader.java index 1b1fa7f..220e42e 100644 --- a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestGenericJdbcConnectorUpgrader.java +++ b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestGenericJdbcConnectorUpgrader.java @@ -17,10 +17,12 @@ / package org.apache.sqoop.connector.jdbc; +import org.apache.commons.lang.StringUtils; import org.apache.sqoop.connector.jdbc.configuration.FromJobConfiguration; import org.apache.sqoop.connector.jdbc.configuration.LinkConfiguration; import org.apache.sqoop.connector.jdbc.configuration.ToJobConfiguration; import org.apache.sqoop.model.ConfigUtils; +import org.apache.sqoop.model.InputEditable; import org.apache.sqoop.model.MBooleanInput; import org.apache.sqoop.model.MConfig; import org.apache.sqoop.model.MFromConfig; @@ -71,13 +73,13 @@ public class TestGenericJdbcConnectorUpgrader { originalConfigs = new MFromConfig(new LinkedList<MConfig>()); newConfigs = new MFromConfig(ConfigUtils.toConfigs(FromJobConfiguration.class)); originalConfigs.getConfigs().add(new MConfig("table", new LinkedList<MInput<?>>())); - originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("table.schemaName", false, (short)50)); - originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("table.tableName", false, (short)50)); - originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("table.sql", false, (short)2000)); - originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("table.columns", false, (short)50)); - originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("table.partitionColumn", false, (short)50)); - originalConfigs.getConfigs().get(0).getInputs().add(new MBooleanInput("table.partitionColumnNull", false)); - originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("table.boundaryQuery", false, (short)50)); + originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("table.schemaName", false, InputEditable.ANY, StringUtils.EMPTY, (short)50)); + originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("table.tableName", false, InputEditable.ANY, StringUtils.EMPTY, (short)50)); + originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("table.sql", false, InputEditable.ANY, StringUtils.EMPTY, (short)2000)); + originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("table.columns", false, InputEditable.ANY, StringUtils.EMPTY, (short)50)); + originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("table.partitionColumn", false, InputEditable.ANY, StringUtils.EMPTY, (short)50)); + originalConfigs.getConfigs().get(0).getInputs().add(new MBooleanInput("table.partitionColumnNull", false, InputEditable.ANY, StringUtils.EMPTY)); + originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("table.boundaryQuery", false, InputEditable.ANY, StringUtils.EMPTY, (short)50)); originalConfigs.getInput("table.schemaName").setValue("test-schema"); originalConfigs.getInput("table.tableName").setValue("test-tableName"); originalConfigs.getInput("table.sql").setValue("test-sql"); @@ -117,12 +119,13 @@ public class TestGenericJdbcConnectorUpgrader { originalConfigs = new MToConfig(new LinkedList<MConfig>()); newConfigs = new MToConfig(ConfigUtils.toConfigs(ToJobConfiguration.class)); originalConfigs.getConfigs().add(new MConfig("table", new LinkedList<MInput<?>>())); - originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("table.schemaName", false, (short)50)); - originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("table.tableName", false, (short)50)); - originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("table.sql", false, (short)2000)); - originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("table.columns", false, (short)50)); - originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("table.stageTableName", false, (short)50)); - originalConfigs.getConfigs().get(0).getInputs().add(new MBooleanInput("table.clearStageTable", false)); + + originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("table.schemaName", false, InputEditable.ANY, StringUtils.EMPTY, (short)50)); + originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("table.tableName", false, InputEditable.ANY, StringUtils.EMPTY, (short)50)); + originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("table.sql", false, InputEditable.ANY, StringUtils.EMPTY,(short)2000)); + originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("table.columns", false, InputEditable.ANY, StringUtils.EMPTY, (short)50)); + originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("table.stageTableName", false, InputEditable.ANY, StringUtils.EMPTY, (short)50)); + originalConfigs.getConfigs().get(0).getInputs().add(new MBooleanInput("table.clearStageTable", false, InputEditable.ANY, StringUtils.EMPTY)); originalConfigs.getInput("table.schemaName").setValue("test-schema"); originalConfigs.getInput("table.tableName").setValue("test-tableName"); originalConfigs.getInput("table.sql").setValue("test-sql"); @@ -160,11 +163,11 @@ public class TestGenericJdbcConnectorUpgrader { originalConfigs = new MLinkConfig(new LinkedList<MConfig>()); newConfigs = new MLinkConfig(ConfigUtils.toConfigs(LinkConfiguration.class)); originalConfigs.getConfigs().add(new MConfig("connection", new LinkedList<MInput<?>>())); - originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("connection.jdbcDriver", false, (short)50)); - originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("connection.connectionString", false, (short)50)); - originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("connection.username", false, (short)2000)); - originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("connection.password", false, (short)50)); - originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("connection.jdbcProperties", false, (short)50)); + originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("connection.jdbcDriver", false, InputEditable.ANY, StringUtils.EMPTY, (short)50)); + originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("connection.connectionString", false,InputEditable.ANY, StringUtils.EMPTY, (short)50)); + originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("connection.username", false, InputEditable.ANY, StringUtils.EMPTY, (short)2000)); + originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("connection.password", false, InputEditable.ANY, StringUtils.EMPTY, (short)50)); + originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("connection.jdbcProperties", false, InputEditable.ANY, StringUtils.EMPTY, (short)50)); originalConfigs.getInput("connection.jdbcDriver").setValue("test-jdbcDriver"); originalConfigs.getInput("connection.connectionString").setValue("test-connectionString"); originalConfigs.getInput("connection.username").setValue("test-username"); http://git-wip-us.apache.org/repos/asf/sqoop/blob/6fc50b08/core/src/test/java/org/apache/sqoop/driver/TestDriverConfigUpgrader.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/sqoop/driver/TestDriverConfigUpgrader.java b/core/src/test/java/org/apache/sqoop/driver/TestDriverConfigUpgrader.java index d4522ae..819a7d0 100644 --- a/core/src/test/java/org/apache/sqoop/driver/TestDriverConfigUpgrader.java +++ b/core/src/test/java/org/apache/sqoop/driver/TestDriverConfigUpgrader.java @@ -24,6 +24,8 @@ import static org.testng.Assert.assertNull; import java.util.LinkedList; import java.util.List; +import org.apache.commons.lang.StringUtils; +import org.apache.sqoop.model.InputEditable; import org.apache.sqoop.model.MConfig; import org.apache.sqoop.model.MConfigList; import org.apache.sqoop.model.MDriverConfig; @@ -127,9 +129,11 @@ public class TestDriverConfigUpgrader { List<MInput<?>> inputs1(String formName) { List<MInput<?>> list = new LinkedList<MInput<?>>(); - list.add(new MStringInput(formName + ".s1", false, (short) 30)); - list.add(new MStringInput(formName + ".s2", false, (short) 30)); - list.add(new MIntegerInput(formName + ".i", false)); + list.add(new MStringInput(formName + ".s1", false, InputEditable.ANY, StringUtils.EMPTY, + (short) 30)); + list.add(new MStringInput(formName + ".s2", false, InputEditable.ANY, StringUtils.EMPTY, + (short) 30)); + list.add(new MIntegerInput(formName + ".i", false, InputEditable.ANY, StringUtils.EMPTY)); return list; } @@ -141,9 +145,11 @@ public class TestDriverConfigUpgrader { List<MInput<?>> inputs2(String formName) { List<MInput<?>> list = new LinkedList<MInput<?>>(); - list.add(new MStringInput(formName + ".s1", false, (short) 30)); - list.add(new MStringInput(formName + ".s2_", false, (short) 30)); - list.add(new MIntegerInput(formName + ".i", false)); + list.add(new MStringInput(formName + ".s1", false, InputEditable.ANY, StringUtils.EMPTY, + (short) 30)); + list.add(new MStringInput(formName + ".s2_", false, InputEditable.ANY, StringUtils.EMPTY, + (short) 30)); + list.add(new MIntegerInput(formName + ".i", false, InputEditable.ANY, StringUtils.EMPTY)); return list; } http://git-wip-us.apache.org/repos/asf/sqoop/blob/6fc50b08/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java ---------------------------------------------------------------------- diff --git a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java index 4feaee6..b029ab2 100644 --- a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java +++ b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java @@ -25,6 +25,7 @@ import org.apache.sqoop.common.SqoopException; import org.apache.sqoop.common.SupportedDirections; import org.apache.sqoop.driver.Driver; import org.apache.sqoop.error.code.CommonRepositoryError; +import org.apache.sqoop.model.InputEditable; import org.apache.sqoop.model.SubmissionError; import org.apache.sqoop.model.MBooleanInput; import org.apache.sqoop.model.MConfig; @@ -60,8 +61,10 @@ import java.sql.Timestamp; import java.sql.Types; import java.util.ArrayList; import java.util.Date; +import java.util.HashMap; import java.util.LinkedList; import java.util.List; +import java.util.Map; /* * Set of methods required from each JDBC based repository. @@ -184,8 +187,11 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler { PreparedStatement deleteConfig = null; PreparedStatement deleteConfigDirection = null; PreparedStatement deleteInput = null; + PreparedStatement deleteInputRelation = null; + try { updateConnectorStatement = conn.prepareStatement(crudQueries.getStmtUpdateConfigurable()); + deleteInputRelation = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_DELETE_INPUT_RELATIONS_FOR_INPUT); deleteInput = conn.prepareStatement(crudQueries.getStmtDeleteInputsForConfigurable()); deleteConfigDirection = conn.prepareStatement(crudQueries.getStmtDeleteDirectionsForConfigurable()); deleteConfig = conn.prepareStatement(crudQueries.getStmtDeleteConfigsForConfigurable()); @@ -199,9 +205,11 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler { if (updateConnectorStatement.executeUpdate() != 1) { throw new SqoopException(CommonRepositoryError.COMMON_0035); } + deleteInputRelation.setLong(1, mConnector.getPersistenceId()); deleteInput.setLong(1, mConnector.getPersistenceId()); deleteConfigDirection.setLong(1, mConnector.getPersistenceId()); deleteConfig.setLong(1, mConnector.getPersistenceId()); + deleteInputRelation.executeUpdate(); deleteInput.executeUpdate(); deleteConfigDirection.executeUpdate(); deleteConfig.executeUpdate(); @@ -227,8 +235,10 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler { PreparedStatement updateDriverStatement = null; PreparedStatement deleteConfig = null; PreparedStatement deleteInput = null; + PreparedStatement deleteInputRelation = null; try { updateDriverStatement = conn.prepareStatement(crudQueries.getStmtUpdateConfigurable()); + deleteInputRelation = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_DELETE_INPUT_RELATIONS_FOR_INPUT); deleteInput = conn.prepareStatement(crudQueries.getStmtDeleteInputsForConfigurable()); deleteConfig = conn.prepareStatement(crudQueries.getStmtDeleteConfigsForConfigurable()); updateDriverStatement.setString(1, mDriver.getUniqueName()); @@ -240,8 +250,10 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler { if (updateDriverStatement.executeUpdate() != 1) { throw new SqoopException(CommonRepositoryError.COMMON_0035); } + deleteInputRelation.setLong(1, mDriver.getPersistenceId()); deleteInput.setLong(1, mDriver.getPersistenceId()); deleteConfig.setLong(1, mDriver.getPersistenceId()); + deleteInputRelation.executeUpdate(); deleteInput.executeUpdate(); deleteConfig.executeUpdate(); @@ -261,6 +273,7 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler { * @param conn JDBC link to use for updating the configs / private void insertConfigsForDriver(MDriver mDriver, Connection conn) { + long driverId = mDriver.getPersistenceId(); PreparedStatement baseConfigStmt = null; PreparedStatement baseInputStmt = null; try { @@ -271,7 +284,7 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler { Statement.RETURN_GENERATED_KEYS); // Register a driver config as a job type with no direction - registerConfigs(null, null, mDriver.getDriverConfig().getConfigs(), + registerConfigs(driverId, null, mDriver.getDriverConfig().getConfigs(), MConfigType.JOB.name(), baseConfigStmt, baseInputStmt, conn); } catch (SQLException ex) { @@ -308,11 +321,8 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler { driverConfigInputFetchStmt = conn.prepareStatement(crudQueries.getStmtSelectInput()); List<MConfig> driverConfigs = new ArrayList<MConfig>(); - loadDriverConfigs(driverConfigs, driverConfigFetchStmt, driverConfigInputFetchStmt, 1); + loadDriverConfigs(driverConfigs, driverConfigFetchStmt, driverConfigInputFetchStmt, 1, conn); - if (driverConfigs.isEmpty()) { - return null; - } mDriver = new MDriver(new MDriverConfig(driverConfigs), driverVersion); mDriver.setPersistenceId(driverId); @@ -355,7 +365,7 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler { throw new SqoopException(CommonRepositoryError.COMMON_0008, mDriver.getUniqueName()); } mDriver.setPersistenceId(insertAndGetDriverId(mDriver, conn)); - insertConfigsforDriver(mDriver, conn); + insertConfigsForDriver(mDriver, conn); } /* @@ -1295,27 +1305,6 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler { } } - private void insertConfigsforDriver(MDriver mDriver, Connection conn) { - PreparedStatement baseConfigStmt = null; - PreparedStatement baseInputStmt = null; - try { - baseConfigStmt = conn.prepareStatement(crudQueries.getStmtInsertIntoConfig(), - Statement.RETURN_GENERATED_KEYS); - baseInputStmt = conn.prepareStatement(crudQueries.getStmtInsertIntoInput(), - Statement.RETURN_GENERATED_KEYS); - - // Register a driver config as a job type with no owner/connector and direction - registerConfigs(mDriver.getPersistenceId(), null /* no direction/, mDriver.getDriverConfig().getConfigs(), - MConfigType.JOB.name(), baseConfigStmt, baseInputStmt, conn); - - } catch (SQLException ex) { - logException(ex, mDriver); - throw new SqoopException(CommonRepositoryError.COMMON_0011, ex); - } finally { - closeStatements(baseConfigStmt, baseInputStmt); - } - } - /* * Stores counters for given submission in repository. * @@ -1595,7 +1584,7 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler { List<MConfig> fromConfig = new ArrayList<MConfig>(); List<MConfig> toConfig = new ArrayList<MConfig>(); - loadConnectorConfigTypes(linkConfig, fromConfig, toConfig, connectorConfigFetchStmt, + loadConnectorConfigs(linkConfig, fromConfig, toConfig, connectorConfigFetchStmt, connectorConfigInputFetchStmt, 1, conn); SupportedDirections supportedDirections @@ -1654,7 +1643,7 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler { List<MConfig> fromConfig = new ArrayList<MConfig>(); List<MConfig> toConfig = new ArrayList<MConfig>(); - loadConnectorConfigTypes(connectorLinkConfig, fromConfig, toConfig, connectorConfigFetchStatement, + loadConnectorConfigs(connectorLinkConfig, fromConfig, toConfig, connectorConfigFetchStatement, connectorConfigInputStatement, 2, conn); MLink link = new MLink(connectorId, new MLinkConfig(connectorLinkConfig)); @@ -1721,7 +1710,7 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler { List<MConfig> fromConnectorFromJobConfig = new ArrayList<MConfig>(); List<MConfig> fromConnectorToJobConfig = new ArrayList<MConfig>(); - loadConnectorConfigTypes(fromConnectorLinkConfig, fromConnectorFromJobConfig, fromConnectorToJobConfig, + loadConnectorConfigs(fromConnectorLinkConfig, fromConnectorFromJobConfig, fromConnectorToJobConfig, fromConfigFetchStmt, jobInputFetchStmt, 2, conn); // TO entity configs @@ -1732,10 +1721,10 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler { // ?? dont we need 2 different driver configs for the from/to? List<MConfig> driverConfig = new ArrayList<MConfig>(); - loadConnectorConfigTypes(toConnectorLinkConfig, toConnectorFromJobConfig, toConnectorToJobConfig, + loadConnectorConfigs(toConnectorLinkConfig, toConnectorFromJobConfig, toConnectorToJobConfig, toConfigFetchStmt, jobInputFetchStmt, 2, conn); - loadDriverConfigs(driverConfig, driverConfigfetchStmt, jobInputFetchStmt, 2); + loadDriverConfigs(driverConfig, driverConfigfetchStmt, jobInputFetchStmt, 2, conn); MJob job = new MJob( fromConnectorId, toConnectorId, @@ -1792,18 +1781,13 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler { * @return short number of configs registered. * @throws java.sql.SQLException / - private short registerConfigs(Long configurableId, Direction direction, - List<MConfig> configs, String type, PreparedStatement baseConfigStmt, - PreparedStatement baseInputStmt, Connection conn) - throws SQLException { + private short registerConfigs(Long configurableId, Direction direction, List<MConfig> configs, + String type, PreparedStatement baseConfigStmt, PreparedStatement baseInputStmt, + Connection conn) throws SQLException { short configIndex = 0; for (MConfig config : configs) { - if (configurableId == null) { - baseConfigStmt.setNull(1, Types.BIGINT); - } else { - baseConfigStmt.setLong(1, configurableId); - } + baseConfigStmt.setLong(1, configurableId); baseConfigStmt.setString(2, config.getName()); baseConfigStmt.setString(3, type); @@ -1828,7 +1812,26 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler { // Insert all the inputs List<MInput<?>> inputs = config.getInputs(); - registerConfigInputs(configId, inputs, baseInputStmt); + registerConfigInputs(config, inputs, baseInputStmt); + // validate all the input relations + Map<Long, List<String>> inputRelationships = new HashMap<Long, List<String>>(); + for (MInput<?> input : inputs) { + List<String> inputOverrides = validateAndGetOverridesAttribute(input, config); + if (inputOverrides != null && inputOverrides.size() > 0) { + inputRelationships.put(input.getPersistenceId(), inputOverrides); + } + } + + // Insert all input relations + if (inputRelationships != null && inputRelationships.size() > 0) { + for (Map.Entry<Long, List<String>> entry : inputRelationships.entrySet()) { + List<String> children = entry.getValue(); + for (String child : children) { + Long childId = config.getInput(child).getPersistenceId(); + insertConfigInputRelationship(entry.getKey(), childId, conn); + } + } + } } return configIndex; } @@ -1838,17 +1841,22 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler { * Use given prepare statement to save all inputs into repository. * - * @param configId Identifier for corresponding config - * @param inputs List of inputs that needs to be saved - * @param baseInputStmt Statement that we can utilize - * @throws java.sql.SQLException In case of any failure on Derby side + * @param config + * corresponding config + * @param inputs + * List of inputs that needs to be saved + * @param baseInputStmt + * Statement that we can utilize + * @throws java.sql.SQLException + * In case of any failure on Derby side / - private void registerConfigInputs(long configId, List<MInput<?>> inputs, - PreparedStatement baseInputStmt) throws SQLException { + private void registerConfigInputs(MConfig config, List<MInput<?>> inputs, + PreparedStatement baseInputStmt) throws SQLException { + short inputIndex = 0; for (MInput<?> input : inputs) { baseInputStmt.setString(1, input.getName()); - baseInputStmt.setLong(2, configId); + baseInputStmt.setLong(2, config.getPersistenceId()); baseInputStmt.setShort(3, inputIndex++); baseInputStmt.setString(4, input.getType().name()); baseInputStmt.setBoolean(5, input.isSensitive()); @@ -1859,11 +1867,14 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler { } else { baseInputStmt.setNull(6, Types.INTEGER); } + + baseInputStmt.setString(7, input.getEditable().name()); + // Enum specific column(s) - if(input.getType() == MInputType.ENUM) { - baseInputStmt.setString(7, StringUtils.join(((MEnumInput) input).getValues(), ",")); + if (input.getType() == MInputType.ENUM) { + baseInputStmt.setString(8, StringUtils.join(((MEnumInput) input).getValues(), ",")); } else { - baseInputStmt.setNull(7, Types.VARCHAR); + baseInputStmt.setNull(8, Types.VARCHAR); } int baseInputCount = baseInputStmt.executeUpdate(); @@ -1882,6 +1893,59 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler { } } + private void insertConfigInputRelationship(Long parent, Long child, Connection conn) { + PreparedStatement baseInputRelationStmt = null; + try { + baseInputRelationStmt = conn + .prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_INSERT_INTO_INPUT_RELATION); + baseInputRelationStmt.setLong(1, parent); + baseInputRelationStmt.setLong(2, child); + baseInputRelationStmt.executeUpdate(); + + } catch (SQLException ex) { + throw new SqoopException(CommonRepositoryError.COMMON_0047, ex); + } finally { + closeStatements(baseInputRelationStmt); + } + } + + /* + * Validate that the input override attribute adheres to the rules imposed + * NOTE: all input names in a config class will and must be unique + * Rule #1. + * If editable == USER_ONLY ( cannot override itself ) can override other CONNECTOR_ONLY and ANY inputs, + * but cannot overriding other USER_ONLY attributes + * Rule #2. + * If editable == CONNECTOR_ONLY or ANY ( cannot override itself ) can override any other attribute in the config object + * @param currentInput + * + / + private List<String> validateAndGetOverridesAttribute(MInput<?> currentInput, MConfig config) { + + // split the overrides string into comma separated list + String overrides = currentInput.getOverrides(); + if (StringUtils.isEmpty(overrides)) { + return null; + } + String[] overrideInputs = overrides.split("\\,"); + List<String> children = new ArrayList<String>(); + + for (String override : overrideInputs) { + if (override.equals(currentInput.getName())) { + throw new SqoopException(CommonRepositoryError.COMMON_0046, "for input :" + + currentInput.toString()); + } + if (currentInput.getEditable().equals(InputEditable.USER_ONLY)) { + if (config.getUserOnlyEditableInputNames().contains(override)) { + throw new SqoopException(CommonRepositoryError.COMMON_0045, "for input :" + + currentInput.toString()); + } + } + children.add(override); + } + return children; + } + /* * Load configs and corresponding inputs from Derby database. * @@ -1894,10 +1958,10 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler { * @param configPosition position of the config * @throws java.sql.SQLException In case of any failure on Derby side / - public void loadDriverConfigs(List<MConfig> driverConfig, + private void loadDriverConfigs(List<MConfig> driverConfig, PreparedStatement configFetchStatement, PreparedStatement inputFetchStmt, - int configPosition) throws SQLException { + int configPosition, Connection conn) throws SQLException { // Get list of structures from database ResultSet rsetConfig = configFetchStatement.executeQuery(); @@ -1923,27 +1987,31 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler { String inputType = rsetInput.getString(5); boolean inputSensitivity = rsetInput.getBoolean(6); short inputStrLength = rsetInput.getShort(7); - String inputEnumValues = rsetInput.getString(8); - String value = rsetInput.getString(9); + String editable = rsetInput.getString(8); + InputEditable editableEnum = editable != null ? InputEditable.valueOf(editable) + : InputEditable.ANY; + // get the overrides value from the SQ_INPUT_RELATION table + String overrides = getOverrides(inputId, conn); + String inputEnumValues = rsetInput.getString(9); + String value = rsetInput.getString(10); MInputType mit = MInputType.valueOf(inputType); - MInput input = null; switch (mit) { - case STRING: - input = new MStringInput(inputName, inputSensitivity, inputStrLength); + case STRING: + input = new MStringInput(inputName, inputSensitivity, editableEnum, overrides, inputStrLength); break; case MAP: - input = new MMapInput(inputName, inputSensitivity); + input = new MMapInput(inputName, inputSensitivity, editableEnum, overrides); break; case BOOLEAN: - input = new MBooleanInput(inputName, inputSensitivity); + input = new MBooleanInput(inputName, inputSensitivity, editableEnum, overrides); break; case INTEGER: - input = new MIntegerInput(inputName, inputSensitivity); + input = new MIntegerInput(inputName, inputSensitivity, editableEnum, overrides); break; case ENUM: - input = new MEnumInput(inputName, inputSensitivity, inputEnumValues.split(",")); + input = new MEnumInput(inputName, inputSensitivity, editableEnum, overrides, inputEnumValues.split(",")); break; default: throw new SqoopException(CommonRepositoryError.COMMON_0003, @@ -2037,7 +2105,7 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler { @param conn Connection object that is used to find config direction. * @throws java.sql.SQLException In case of any failure on Derby side / - public void loadConnectorConfigTypes(List<MConfig> linkConfig, List<MConfig> fromConfig, List<MConfig> toConfig, + public void loadConnectorConfigs(List<MConfig> linkConfig, List<MConfig> fromConfig, List<MConfig> toConfig, PreparedStatement configFetchStmt, PreparedStatement inputFetchStmt, int configPosition, Connection conn) throws SQLException { @@ -2065,32 +2133,38 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler { String inputType = rsetInput.getString(5); boolean inputSensitivity = rsetInput.getBoolean(6); short inputStrLength = rsetInput.getShort(7); - String inputEnumValues = rsetInput.getString(8); - String value = rsetInput.getString(9); + String editable = rsetInput.getString(8); + InputEditable editableEnum = editable != null ? InputEditable.valueOf(editable) + : InputEditable.ANY; + // get the overrides value from the SQ_INPUT_RELATION table + String overrides = getOverrides(inputId, conn); + String inputEnumValues = rsetInput.getString(9); + String value = rsetInput.getString(10); MInputType mit = MInputType.valueOf(inputType); MInput<?> input = null; switch (mit) { - case STRING: - input = new MStringInput(inputName, inputSensitivity, inputStrLength); - break; - case MAP: - input = new MMapInput(inputName, inputSensitivity); - break; - case BOOLEAN: - input = new MBooleanInput(inputName, inputSensitivity); - break; - case INTEGER: - input = new MIntegerInput(inputName, inputSensitivity); - break; - case ENUM: - input = new MEnumInput(inputName, inputSensitivity, inputEnumValues.split(",")); - break; - default: - throw new SqoopException(CommonRepositoryError.COMMON_0003, - "input-" + inputName + ":" + inputId + ":" - + "config-" + inputConfig + ":" + mit.name()); + case STRING: + input = new MStringInput(inputName, inputSensitivity, editableEnum, overrides, + inputStrLength); + break; + case MAP: + input = new MMapInput(inputName, inputSensitivity, editableEnum, overrides); + break; + case BOOLEAN: + input = new MBooleanInput(inputName, inputSensitivity, editableEnum, overrides); + break; + case INTEGER: + input = new MIntegerInput(inputName, inputSensitivity, editableEnum, overrides); + break; + case ENUM: + input = new MEnumInput(inputName, inputSensitivity, editableEnum, overrides, + inputEnumValues.split(",")); + break; + default: + throw new SqoopException(CommonRepositoryError.COMMON_0003, "input-" + inputName + ":" + + inputId + ":" + "config-" + inputConfig + ":" + mit.name()); } // Set persistent ID @@ -2170,6 +2244,60 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler { } } + /* + * @param inputId + * @param conn + * @return + / + private String getOverrides(long inputId, Connection conn) { + + PreparedStatement overridesStmt = null; + PreparedStatement inputStmt = null; + + ResultSet rsOverride = null; + ResultSet rsInput = null; + + List<String> overrides = new ArrayList<String>(); + try { + overridesStmt = conn.prepareStatement(crudQueries.getStmtSelectInputOverrides()); + inputStmt = conn.prepareStatement(crudQueries.getStmtSelectInputById()); + overridesStmt.setLong(1, inputId); + rsOverride = overridesStmt.executeQuery(); + + while (rsOverride.next()) { + long overrideId = rsOverride.getLong(1); + inputStmt.setLong(1, overrideId); + rsInput = inputStmt.executeQuery(); + if(rsInput.next()) { + overrides.add(rsInput.getString(2)); + } + } + if (overrides != null && overrides.size() > 0) { + return StringUtils.join(overrides, ","); + } else { + return StringUtils.EMPTY; + + } + } catch (SQLException ex) { + logException(ex, inputId); + throw new SqoopException(CommonRepositoryError.COMMON_0048, ex); + } finally { + if (rsOverride != null) { + closeResultSets(rsOverride); + } + if (rsInput != null) { + closeResultSets(rsInput); + } + + if (overridesStmt != null) { + closeStatements(overridesStmt); + } + if (inputStmt != null) { + closeStatements(inputStmt); + } + } + } + private void createInputValues(String query, long id, List<MConfig> configs, http://git-wip-us.apache.org/repos/asf/sqoop/blob/6fc50b08/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryInsertUpdateDeleteSelectQuery.java ---------------------------------------------------------------------- diff --git a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryInsertUpdateDeleteSelectQuery.java b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryInsertUpdateDeleteSelectQuery.java index 8626b31..d61ff0b 100644 --- a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryInsertUpdateDeleteSelectQuery.java +++ b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryInsertUpdateDeleteSelectQuery.java @@ -81,6 +81,21 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery { + " WHERE " + CommonRepoUtils.escapeColumnName(COLUMN_SQ_CFG_CONFIGURABLE) + " = ?)"; + public static final String STMT_DELETE_INPUT_RELATIONS_FOR_INPUT = + "DELETE FROM " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_INPUT_RELATION_NAME) + + " WHERE " + + CommonRepoUtils.escapeColumnName(COLUMN_SQIR_PARENT) + + " IN (SELECT " + + CommonRepoUtils.escapeColumnName(COLUMN_SQI_ID) + + " FROM " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_INPUT_NAME) + + " WHERE " + + CommonRepoUtils.escapeColumnName(COLUMN_SQI_CONFIG) + + " IN (SELECT " + + CommonRepoUtils.escapeColumnName(COLUMN_SQ_CFG_ID) + + " FROM " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_CONFIG_NAME) + + " WHERE " + + CommonRepoUtils.escapeColumnName(COLUMN_SQ_CFG_CONFIGURABLE) + " = ?))"; + //Update the configurable public static final String STMT_UPDATE_CONFIGURABLE = "UPDATE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_CONFIGURABLE_NAME) @@ -128,12 +143,29 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery { + CommonRepoUtils.escapeColumnName(COLUMN_SQI_TYPE) + ", " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_STRMASK) + ", " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_STRLENGTH) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQI_EDITABLE) + ", " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_ENUMVALS) + ", " + "cast(null as varchar(100))" + " FROM " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_INPUT_NAME) + " WHERE " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_CONFIG) + " = ?" + " ORDER BY " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_INDEX); + // DML get Input by Id + public static final String STMT_SELECT_INPUT_BY_ID = + "SELECT " + + CommonRepoUtils.escapeColumnName(COLUMN_SQI_ID) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQI_NAME) + + " FROM " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_INPUT_NAME) + + " WHERE " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_ID) + " = ?"; + + // DML get Input by name + public static final String STMT_SELECT_INPUT_BY_NAME = + "SELECT " + + CommonRepoUtils.escapeColumnName(COLUMN_SQI_ID) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQI_NAME) + + " FROM " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_INPUT_NAME) + + " WHERE " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_NAME) + " = ?"; + // DML: Insert into config input public static final String STMT_INSERT_INTO_INPUT = "INSERT INTO " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_INPUT_NAME) + " (" @@ -143,8 +175,22 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery { + CommonRepoUtils.escapeColumnName(COLUMN_SQI_TYPE) + ", " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_STRMASK) + ", " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_STRLENGTH) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQI_EDITABLE) + ", " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_ENUMVALS) - + ") VALUES (?, ?, ?, ?, ?, ?, ?)"; + + ") VALUES (?, ?, ?, ?, ?, ?, ?, ?)"; + +/******* INPUT-RELATIONSHIP TABLE **********/ + public static final String STMT_INSERT_INTO_INPUT_RELATION = + "INSERT INTO " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_INPUT_RELATION_NAME) + " (" + + CommonRepoUtils.escapeColumnName(COLUMN_SQIR_PARENT) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQIR_CHILD) + + ") VALUES (?, ?)"; + + public static final String STMT_FETCH_SQ_INPUT_OVERRIDES = + "SELECT " + + CommonRepoUtils.escapeColumnName(COLUMN_SQIR_CHILD) + + " FROM " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_INPUT_RELATION_NAME) + + " WHERE " + CommonRepoUtils.escapeColumnName(COLUMN_SQIR_PARENT) + " = ?"; / * *****LINK INPUT TABLE *********** @@ -159,6 +205,7 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery { + CommonRepoUtils.escapeColumnName(COLUMN_SQI_TYPE) + ", " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_STRMASK) + ", " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_STRLENGTH) + "," + + CommonRepoUtils.escapeColumnName(COLUMN_SQI_EDITABLE) + ", " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_ENUMVALS) + ", " + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_VALUE) + " FROM " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_INPUT_NAME) @@ -182,6 +229,7 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery { + CommonRepoUtils.escapeColumnName(COLUMN_SQI_TYPE) + ", " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_STRMASK) + ", " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_STRLENGTH) + ", " + + CommonRepoUtils.escapeColumnName(COLUMN_SQI_EDITABLE) + ", " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_ENUMVALS) + ", " + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_VALUE) + " FROM " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_INPUT_NAME) @@ -607,6 +655,18 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery { return STMT_SELECT_INPUT; } + public String getStmtSelectInputById() { + return STMT_SELECT_INPUT_BY_ID; + } + + public String getStmtSelectInputByName() { + return STMT_SELECT_INPUT_BY_NAME; + } + + public String getStmtSelectInputOverrides() { + return STMT_FETCH_SQ_INPUT_OVERRIDES; + } + public String getStmtInsertIntoInput() { return STMT_INSERT_INTO_INPUT; } http://git-wip-us.apache.org/repos/asf/sqoop/blob/6fc50b08/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositorySchemaConstants.java ---------------------------------------------------------------------- diff --git a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositorySchemaConstants.java b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositorySchemaConstants.java index 4ab07b2..5ce9488 100644 --- a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositorySchemaConstants.java +++ b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositorySchemaConstants.java @@ -117,6 +117,19 @@ public final class CommonRepositorySchemaConstants { public static final String COLUMN_SQI_ENUMVALS = "SQI_ENUMVALS"; + public static final String COLUMN_SQI_EDITABLE = "SQI_EDITABLE"; + + // SQ_INPUT_RELATION + + public static final String TABLE_SQ_INPUT_RELATION_NAME = "SQ_INPUT_RELATION"; + + public static final String COLUMN_SQIR_ID = "SQIR_ID"; + + public static final String COLUMN_SQIR_PARENT = "SQIR_PARENT_ID"; + + public static final String COLUMN_SQIR_CHILD = "SQIR_CHILD_ID"; + + public static final String TABLE_SQ_LINK_NAME = "SQ_LINK"; public static final String TABLE_SQ_LINK = SCHEMA_PREFIX + TABLE_SQ_LINK_NAME; http://git-wip-us.apache.org/repos/asf/sqoop/blob/6fc50b08/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java ---------------------------------------------------------------------- diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java index 2f05fcb..c991e5c 100644 --- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java +++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java @@ -293,10 +293,13 @@ public class DerbyRepositoryHandler extends CommonRepositoryHandler { if (repositoryVersion < 5) { runQuery(QUERY_UPGRADE_TABLE_SQ_CONFIG_ADD_UNIQUE_CONSTRAINT_NAME_TYPE_AND_CONFIGURABLE_ID, conn); runQuery(QUERY_UPGRADE_TABLE_SQ_INPUT_ADD_UNIQUE_CONSTRAINT_NAME_TYPE_AND_CONFIG_ID, conn); - // table column rename + // submission table column rename runQuery(QUERY_UPGRADE_RENAME_TABLE_SQ_JOB_SUBMISSION_COLUMN_1, conn); runQuery(QUERY_UPGRADE_RENAME_TABLE_SQ_JOB_SUBMISSION_COLUMN_2, conn); - + // SQOOP-1804 + runQuery(QUERY_UPGRADE_TABLE_SQ_INPUT_ADD_COLUMN_SQI_EDITABLE, conn); + // create a new table for SQ_INPUT relationships + runQuery(QUERY_CREATE_TABLE_SQ_INPUT_RELATION, conn); } // last step upgrade the repository version to the latest value in the code http://git-wip-us.apache.org/repos/asf/sqoop/blob/6fc50b08/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaConstants.java ---------------------------------------------------------------------- diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaConstants.java b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaConstants.java index a551094..8504091 100644 --- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaConstants.java +++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaConstants.java @@ -160,6 +160,15 @@ public final class DerbySchemaConstants { public static final String CONSTRAINT_SQ_INPUT_UNIQUE = SCHEMA_PREFIX + CONSTRAINT_SQ_INPUT_UNIQUE_NAME_TYPE_CONFIG; + // SQ_INPUT_RELATION FK + + public static final String CONSTRAINT_SQIR_PARENT_NAME = CONSTRAINT_PREFIX + "SQIR_PARENT_ID"; + public static final String CONSTRAINT_SQIR_PARENT = SCHEMA_PREFIX + CONSTRAINT_SQIR_PARENT_NAME; + + public static final String CONSTRAINT_SQIR_CHILD_NAME = CONSTRAINT_PREFIX + "SQIR_CHILD_ID"; + public static final String CONSTRAINT_SQIR_CHILD = SCHEMA_PREFIX + CONSTRAINT_SQIR_CHILD_NAME; + + // SQ_LINK @Deprecated // used only for upgrade http://git-wip-us.apache.org/repos/asf/sqoop/blob/6fc50b08/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaCreateQuery.java ---------------------------------------------------------------------- diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaCreateQuery.java b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaCreateQuery.java index ad80797..d501a5d 100644 --- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaCreateQuery.java +++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaCreateQuery.java @@ -122,6 +122,19 @@ import static org.apache.sqoop.repository.derby.DerbySchemaConstants.; \| SQI_STRMASK: BOOLEAN \| * \| SQI_STRLENGTH: SMALLINT \| * \| SQI_ENUMVALS: VARCHAR(100) \| + * \| SQI_EDITABLE: VARCHAR(32) \| + * +----------------------------+ + * </pre> +* <p> + * <strong>SQ_INPUT_RELATION</strong>: Input to Input relationship + * + * <pre> + * +----------------------------+ + * \| SQ_INPUT_RELATION \| + * +----------------------------+ + * \| SQIR_ID: BIGINT PK AUTO-GEN \| + * \| SQIR_PARENT_ID: BIGINT \|FK SQ_INPUT(SQI_ID) + * \| SQIR_CHILD_ID: BIGINT \|FK SQ_INPUT(SQI_ID) * +----------------------------+ * </pre> * @@ -369,6 +382,20 @@ public final class DerbySchemaCreateQuery { + "REFERENCES " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_FORM_NAME) + " (" + CommonRepoUtils.escapeColumnName(COLUMN_SQF_ID) + ")" + ")"; + // DDL : Create table SQ_INPUT_RELATION + public static final String QUERY_CREATE_TABLE_SQ_INPUT_RELATION = + "CREATE TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_INPUT_RELATION_NAME) + " (" + + CommonRepoUtils.escapeColumnName(COLUMN_SQIR_ID) + " BIGINT GENERATED ALWAYS AS IDENTITY (START WITH 1, INCREMENT BY 1) PRIMARY KEY, " + + CommonRepoUtils.escapeColumnName(COLUMN_SQIR_PARENT) + " BIGINT, " + + CommonRepoUtils.escapeColumnName(COLUMN_SQIR_CHILD) + " BIGINT, " + + "CONSTRAINT " + CONSTRAINT_SQIR_PARENT + " " + + "FOREIGN KEY (" + CommonRepoUtils.escapeColumnName(COLUMN_SQIR_PARENT) + ") " + + "REFERENCES " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_INPUT_NAME) + " (" + CommonRepoUtils.escapeColumnName(COLUMN_SQI_ID) + ")," + + "CONSTRAINT " + CONSTRAINT_SQIR_CHILD + " " + + "FOREIGN KEY (" + CommonRepoUtils.escapeColumnName(COLUMN_SQIR_CHILD) + ") " + + "REFERENCES " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_INPUT_NAME) + " (" + CommonRepoUtils.escapeColumnName(COLUMN_SQI_ID) + ")" + + ")"; + // DDL: Create table SQ_CONNECTION public static final String QUERY_CREATE_TABLE_SQ_CONNECTION = "CREATE TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_CONNECTION_NAME) + " (" http://git-wip-us.apache.org/repos/asf/sqoop/blob/6fc50b08/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaUpgradeQuery.java ---------------------------------------------------------------------- diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaUpgradeQuery.java b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaUpgradeQuery.java index fa6710b..606fc68 100644 --- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaUpgradeQuery.java +++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaUpgradeQuery.java @@ -510,7 +510,12 @@ public final class DerbySchemaUpgradeQuery { public static final String QUERY_UPGRADE_RENAME_TABLE_SQ_JOB_SUBMISSION_COLUMN_2 = "RENAME COLUMN " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_SUBMISSION_NAME) + "." + CommonRepoUtils.escapeColumnName(COLUMN_SQS_EXCEPTION_TRACE) + " TO " + CommonRepoUtils.escapeColumnName(COLUMN_SQS_ERROR_DETAILS); - private DerbySchemaUpgradeQuery() { + // SQOOP-1804, column add for SQ_INPUT + public static final String QUERY_UPGRADE_TABLE_SQ_INPUT_ADD_COLUMN_SQI_EDITABLE = "ALTER TABLE " + + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_INPUT_NAME) + " ADD COLUMN " + + CommonRepoUtils.escapeColumnName(COLUMN_SQI_EDITABLE) + " VARCHAR(32)"; + + private DerbySchemaUpgradeQuery() { // Disable explicit object creation } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sqoop/blob/6fc50b08/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java ---------------------------------------------------------------------- diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java index be8c23e..bea5cd7 100644 --- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java +++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java @@ -32,8 +32,10 @@ import java.sql.Statement; import java.util.LinkedList; import java.util.List; +import org.apache.commons.lang.StringUtils; import org.apache.sqoop.common.Direction; import org.apache.sqoop.json.DriverBean; +import org.apache.sqoop.model.InputEditable; import org.apache.sqoop.model.MConfig; import org.apache.sqoop.model.MConnector; import org.apache.sqoop.model.MDriver; @@ -150,10 +152,10 @@ abstract public class DerbyTestCase { } /** - * Create derby schema. FIX(SQOOP-1583): This code needs heavy refactoring. - * Details are in the ticket. + Create derby schema. FIX(SQOOP-1583): This code needs heavy refactoring. + Details are in the ticket. * - * @throws Exception + @throws Exception / protected void createOrUpgradeSchema(int version) throws Exception { if (version > 0) { @@ -215,6 +217,9 @@ abstract public class DerbyTestCase { // add submission table column name renames runQuery(QUERY_UPGRADE_RENAME_TABLE_SQ_JOB_SUBMISSION_COLUMN_1); runQuery(QUERY_UPGRADE_RENAME_TABLE_SQ_JOB_SUBMISSION_COLUMN_2); + // SQOOP-1804 + runQuery(QUERY_UPGRADE_TABLE_SQ_INPUT_ADD_COLUMN_SQI_EDITABLE); + runQuery(QUERY_CREATE_TABLE_SQ_INPUT_RELATION); } // deprecated repository version @@ -230,9 +235,11 @@ abstract public class DerbyTestCase { } /** - * Run arbitrary query on derby memory repository. - * @param query Query to execute - * @throws Exception + Run arbitrary query on derby memory repository. + + @param query + Query to execute + @throws Exception / protected void runQuery(String query, Object... args) throws Exception { PreparedStatement stmt = null; @@ -258,11 +265,12 @@ abstract public class DerbyTestCase { } /** - * Run single, arbitrary insert query on derby memory repository. + Run single, arbitrary insert query on derby memory repository. - * @param query Query to execute - * @return Long id of newly inserted row (-1 if none). - * @throws Exception + @param query + Query to execute + @return Long id of newly inserted row (-1 if none). + @throws Exception / protected Long runInsertQuery(String query, Object... args) throws Exception { PreparedStatement stmt = null; @@ -431,19 +439,19 @@ abstract public class DerbyTestCase { for (int i = 0; i < 3; i++) { // First config runInsertQuery("INSERT INTO SQOOP.SQ_INPUT" - + "(SQI_NAME, SQI_CONFIG, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH)" - + " VALUES('I1', " + (i 2 + 1) + ", 0, 'STRING', false, 30)"); + + "(SQI_NAME, SQI_CONFIG, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH, SQI_EDITABLE)" + + " VALUES('I1', " + (i * 2 + 1) + ", 0, 'STRING', false, 30, 'CONNECTOR_ONLY')"); runInsertQuery("INSERT INTO SQOOP.SQ_INPUT" - + "(SQI_NAME, SQI_CONFIG, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH)" - + " VALUES('I2', " + (i * 2 + 1) + ", 1, 'MAP', false, 30)"); + + "(SQI_NAME, SQI_CONFIG, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH, SQI_EDITABLE)" + + " VALUES('I2', " + (i * 2 + 1) + ", 1, 'MAP', false, 30, 'CONNECTOR_ONLY')"); // Second config runInsertQuery("INSERT INTO SQOOP.SQ_INPUT" - + "(SQI_NAME, SQI_CONFIG, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH)" - + " VALUES('I3', " + (i * 2 + 2) + ", 0, 'STRING', false, 30)"); + + "(SQI_NAME, SQI_CONFIG, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH, SQI_EDITABLE)" + + " VALUES('I3', " + (i * 2 + 2) + ", 0, 'STRING', false, 30, 'CONNECTOR_ONLY')"); runInsertQuery("INSERT INTO SQOOP.SQ_INPUT" - + "(SQI_NAME, SQI_CONFIG, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH)" - + " VALUES('I4', " + (i * 2 + 2) + ", 1, 'MAP', false, 30)"); + + "(SQI_NAME, SQI_CONFIG, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH, SQI_EDITABLE)" + + " VALUES('I4', " + (i * 2 + 2) + ", 1, 'MAP', false, 30, 'CONNECTOR_ONLY')"); } } @@ -517,28 +525,28 @@ abstract public class DerbyTestCase { for (int i = 0; i < 4; i++) { // First config runInsertQuery("INSERT INTO SQOOP.SQ_INPUT" - + "(SQI_NAME, SQI_CONFIG, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH)" - + " VALUES('I1', " + (i * 2 + 1) + ", 0, 'STRING', false, 30)"); + + "(SQI_NAME, SQI_CONFIG, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH, SQI_EDITABLE)" + + " VALUES('I1', " + (i * 2 + 1) + ", 0, 'STRING', false, 30, 'CONNECTOR_ONLY')"); runInsertQuery("INSERT INTO SQOOP.SQ_INPUT" - + "(SQI_NAME, SQI_CONFIG, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH)" - + " VALUES('I2', " + (i * 2 + 1) + ", 1, 'MAP', false, 30)"); + + "(SQI_NAME, SQI_CONFIG, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH, SQI_EDITABLE)" + + " VALUES('I2', " + (i * 2 + 1) + ", 1, 'MAP', false, 30, 'CONNECTOR_ONLY')"); // Second config runInsertQuery("INSERT INTO SQOOP.SQ_INPUT" - + "(SQI_NAME, SQI_CONFIG, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH)" - + " VALUES('I3', " + (i * 2 + 2) + ", 0, 'STRING', false, 30)"); + + "(SQI_NAME, SQI_CONFIG, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH, SQI_EDITABLE)" + + " VALUES('I3', " + (i * 2 + 2) + ", 0, 'STRING', false, 30, 'CONNECTOR_ONLY')"); runInsertQuery("INSERT INTO SQOOP.SQ_INPUT" - + "(SQI_NAME, SQI_CONFIG, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH)" - + " VALUES('I4', " + (i * 2 + 2) + ", 1, 'MAP', false, 30)"); + + "(SQI_NAME, SQI_CONFIG, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH, SQI_EDITABLE)" + + " VALUES('I4', " + (i * 2 + 2) + ", 1, 'MAP', false, 30, 'CONNECTOR_ONLY')"); } } /** - * Load testing connector and driver config into repository. + Load testing connector and driver config into repository. - * @param version - * system version (2 or 4) - * @throws Exception + @param version + system version (2 or 4) + @throws Exception / protected void loadConnectorAndDriverConfig(int version) throws Exception { switch (version) { @@ -561,9 +569,11 @@ abstract public class DerbyTestCase { } /** - * Load testing link objects into repository. - * @param version system version (2 or 4) - * @throws Exception + Load testing link objects into repository. + + @param version + system version (2 or 4) + @throws Exception / public void loadConnectionsOrLinks(int version) throws Exception { switch (version) { @@ -606,9 +616,11 @@ abstract public class DerbyTestCase { } /** - * Load testing job objects into repository. - * @param version system version (2 or 4) - * @throws Exception + Load testing job objects into repository. + + @param version + system version (2 or 4) + @throws Exception / public void loadJobs(int version) throws Exception { int index = 0; @@ -665,8 +677,9 @@ abstract public class DerbyTestCase { } /** - * testing job with non unique name objects into repository. - * @throws Exception + testing job with non unique name objects into repository. + + @throws Exception / public void loadNonUniqueJobsInVersion4() throws Exception { int index = 0; @@ -678,8 +691,9 @@ abstract public class DerbyTestCase { } /** - * testing link with non unique name objects into repository. - * @throws Exception + testing link with non unique name objects into repository. + + @throws Exception / public void loadNonUniqueLinksInVersion4() throws Exception { @@ -690,8 +704,9 @@ abstract public class DerbyTestCase { } /** - * testing configurable with non unique name objects into repository. - * @throws Exception + testing configurable with non unique name objects into repository. + + @throws Exception / public void loadNonUniqueConfigurablesInVersion4() throws Exception { @@ -704,8 +719,9 @@ abstract public class DerbyTestCase { } /** - * testing config with non unique name/type objects into repository. - * @throws Exception + testing config with non unique name/type objects into repository. + + @throws Exception / public void loadNonUniqueConfigNameTypeInVersion4() throws Exception { @@ -718,8 +734,9 @@ abstract public class DerbyTestCase { } /** - * testing input with non unique name/type objects into repository. - * @throws Exception + testing input with non unique name/type objects into repository. + + @throws Exception / public void loadNonUniqueInputNameTypeInVersion4() throws Exception { @@ -727,16 +744,17 @@ abstract public class DerbyTestCase { + "(SQ_CFG_CONFIGURABLE, SQ_CFG_NAME, SQ_CFG_TYPE, SQ_CFG_INDEX) " + "VALUES(1, 'C1', 'LINK', 0)"); runInsertQuery("INSERT INTO SQOOP.SQ_INPUT" - + "(SQI_NAME, SQI_CONFIG, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH)" - + " VALUES('I1', 1, 0, 'STRING', false, 30)"); + + "(SQI_NAME, SQI_CONFIG, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH,SQI_EDITABLE)" + + " VALUES('I1', 1, 0, 'STRING', false, 30, 'CONNECTOR_ONLY')"); runInsertQuery("INSERT INTO SQOOP.SQ_INPUT" - + "(SQI_NAME, SQI_CONFIG, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH)" - + " VALUES('I1', 1, 0, 'STRING', false, 30)"); + + "(SQI_NAME, SQI_CONFIG, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH, SQI_EDITABLE)" + + " VALUES('I1', 1, 0, 'STRING', false, 30, 'CONNECTOR_ONLY')"); } /** - * testing config with non unique name/type objects into repository. - * @throws Exception + testing config with non unique name/type objects into repository. + + @throws Exception / public void loadNonUniqueConfigNameButUniqueTypeInVersion4() throws Exception { @@ -749,8 +767,9 @@ abstract public class DerbyTestCase { } /** - * testing config with non unique name/type objects into repository. - * @throws Exception + testing config with non unique name/type objects into repository. + + @throws Exception / public void loadNonUniqueConfigNameAndTypeButUniqueConfigurableInVersion4() throws Exception { @@ -763,8 +782,9 @@ abstract public class DerbyTestCase { } /** - * testing input with non unique name/type objects into repository. - * @throws Exception + testing input with non unique name/type objects into repository. + + @throws Exception / public void loadNonUniqueInputNameAndTypeButUniqueConfigInVersion4() throws Exception { @@ -776,11 +796,11 @@ abstract public class DerbyTestCase { + "VALUES(2, 'C2', 'LINK', 0)"); runInsertQuery("INSERT INTO SQOOP.SQ_INPUT" - + "(SQI_NAME, SQI_CONFIG, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH)" - + " VALUES('C1.A', 1, 0, 'STRING', false, 30)"); + + "(SQI_NAME, SQI_CONFIG, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH, SQI_EDITABLE)" + + " VALUES('C1.A', 1, 0, 'STRING', false, 30, 'ANY')"); runInsertQuery("INSERT INTO SQOOP.SQ_INPUT" - + "(SQI_NAME, SQI_CONFIG, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH)" - + " VALUES('C1.A', 2, 1, 'STRING', false, 30)"); + + "(SQI_NAME, SQI_CONFIG, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH, SQI_EDITABLE)" + + " VALUES('C1.A', 2, 1, 'STRING', false, 30, 'ANY')"); } public void loadJobsForLatestVersion() throws Exception { @@ -788,7 +808,7 @@ abstract public class DerbyTestCase { } /** - * Add a second connector for testing with multiple connectors + Add a second connector for testing with multiple connectors / public void addConnectorB() throws Exception { // Connector entry @@ -801,8 +821,9 @@ abstract public class DerbyTestCase { } /** - * Load testing submissions into the repository. - * @throws Exception + Load testing submissions into the repository. + + @throws Exception / public void loadSubmissions() throws Exception { runQuery("INSERT INTO SQOOP.SQ_COUNTER_GROUP " + "(SQG_NAME) " + "VALUES" + "('gA'), ('gB')"); @@ -832,6 +853,57 @@ abstract public class DerbyTestCase { return getConnector(true, true); } + protected MConnector getConnectorWithIncorrectOverridesAttribute() { + return getBadConnector(true, true); + } + + protected MConnector getConnectorWithIncorrectOverridesAttribute2() { + return getBadConnector2(true, true); + } + + protected MConnector getConnectorWithMultipleOverridesAttribute() { + return getConnectorWithMultipleOverrides(true, true); + } + + protected MConnector getConnectorWithMultipleOverrides(boolean from, boolean to) { + MFromConfig fromConfig = null; + MToConfig toConfig = null; + if (from) { + fromConfig = getMultipleOverridesFromConfig(); + } + if (to) { + toConfig = getToConfig(); + } + return new MConnector("A", "org.apache.sqoop.test.A", "1.0-test", getLinkConfig(), fromConfig, + toConfig); + } + + protected MConnector getBadConnector(boolean from, boolean to) { + MFromConfig fromConfig = null; + MToConfig toConfig = null; + if (from) { + fromConfig = getBadFromConfig(); + } + if (to) { + toConfig = getToConfig(); + } + return new MConnector("A", "org.apache.sqoop.test.A", "1.0-test", getLinkConfig(), fromConfig, + toConfig); + } + + protected MConnector getBadConnector2(boolean from, boolean to) { + MFromConfig fromConfig = null; + MToConfig toConfig = null; + if (from) { + fromConfig = getNonExistentOverridesFromConfig(); + } + if (to) { + toConfig = getToConfig(); + } + return new MConnector("A", "org.apache.sqoop.test.A", "1.0-test", getLinkConfig(), fromConfig, + toConfig); + } + protected MConnector getConnector(boolean from, boolean to) { MFromConfig fromConfig = null; MToConfig toConfig = null; @@ -849,6 +921,10 @@ abstract public class DerbyTestCase { return new MDriver(getDriverConfig(), DriverBean.CURRENT_DRIVER_VERSION); } + protected MDriver getBadDriver() { + return new MDriver(getBadDriverConfig(), DriverBean.CURRENT_DRIVER_VERSION); + } + protected void fillLink(MLink link) { List<MConfig> configs = link.getConnectorLinkConfig().getConfigs(); ((MStringInput) configs.get(0).getInputs().get(0)).setValue("Value1"); @@ -877,6 +953,18 @@ abstract public class DerbyTestCase { return new MFromConfig(getConfigs("from1", "from2")); } + protected MFromConfig getBadFromConfig() { + return new MFromConfig(getBadConfigs("from1", "from2")); + } + + protected MFromConfig getMultipleOverridesFromConfig() { + return new MFromConfig(getMultipleOverrideConfigs("from1", "from2")); + } + + protected MFromConfig getNonExistentOverridesFromConfig() { + return new MFromConfig(getBadConfigsWithNonExistingInputOverrides("from1", "from2")); + } + protected MToConfig getToConfig() { return new MToConfig(getConfigs("to1", "to2")); } @@ -885,20 +973,111 @@ abstract public class DerbyTestCase { return new MDriverConfig(getConfigs("d1", "d2")); } + protected MDriverConfig getBadDriverConfig() { + return new MDriverConfig(getBadConfigsWithSelfOverrides("d1", "d2")); + } + protected List<MConfig> getConfigs(String configName1, String configName2) { List<MConfig> configs = new LinkedList<MConfig>(); List<MInput<?>> inputs = new LinkedList<MInput<?>>(); - MInput input = new MStringInput("I1", false, (short) 30); + MInput input = new MStringInput("I1", false, InputEditable.ANY, "I2", (short) 30); + inputs.add(input); + input = new MMapInput("I2", false, InputEditable.ANY, StringUtils.EMPTY); + inputs.add(input); + configs.add(new MConfig(configName1, inputs)); + + inputs = new LinkedList<MInput<?>>(); + input = new MStringInput("I3", false, InputEditable.ANY, "I4", (short) 30); + inputs.add(input); + input = new MMapInput("I4", false, InputEditable.USER_ONLY, "I3"); + inputs.add(input); + configs.add(new MConfig(configName2, inputs)); + + return configs; + } + + protected List<MConfig> getBadConfigs(String configName1, String configName2) { + List<MConfig> configs = new LinkedList<MConfig>(); + + List<MInput<?>> inputs = new LinkedList<MInput<?>>(); + // I1 overrides another user_only attribute, hence a bad config + MInput input = new MStringInput("I1", false, InputEditable.USER_ONLY, "I2", (short) 30); + inputs.add(input); + input = new MMapInput("I2", false, InputEditable.USER_ONLY, "I1"); + inputs.add(input); + configs.add(new MConfig(configName1, inputs)); + + inputs = new LinkedList<MInput<?>>(); + input = new MStringInput("I3", false, InputEditable.ANY, StringUtils.EMPTY, (short) 30); + inputs.add(input); + input = new MMapInput("I4", false, InputEditable.ANY, StringUtils.EMPTY); + inputs.add(input); + configs.add(new MConfig(configName2, inputs)); + + return configs; + } + + protected List<MConfig> getBadConfigsWithSelfOverrides(String configName1, String configName2) { + List<MConfig> configs = new LinkedList<MConfig>(); + + List<MInput<?>> inputs = new LinkedList<MInput<?>>(); + // I1 overrides another user_only attribute, hence a bad config + MInput input = new MStringInput("I1", false, InputEditable.USER_ONLY, "I2", (short) 30); inputs.add(input); - input = new MMapInput("I2", false); + input = new MMapInput("I2", false, InputEditable.USER_ONLY, "I2"); inputs.add(input); configs.add(new MConfig(configName1, inputs)); inputs = new LinkedList<MInput<?>>(); - input = new MStringInput("I3", false, (short) 30); + input = new MStringInput("I3", false, InputEditable.ANY, StringUtils.EMPTY, (short) 30); inputs.add(input); - input = new MMapInput("I4", false); + input = new MMapInput("I4", false, InputEditable.ANY, StringUtils.EMPTY); + inputs.add(input); + configs.add(new MConfig(configName2, inputs)); + + return configs; + } + + protected List<MConfig> getMultipleOverrideConfigs(String configName1, String configName2) { + List<MConfig> configs = new LinkedList<MConfig>(); + + List<MInput<?>> inputs = new LinkedList<MInput<?>>(); + // I1 overrides another user_only attribute, hence a bad config + MInput input = new MStringInput("I1", false, InputEditable.USER_ONLY, "I2", (short) 30); + inputs.add(input); + input = new MMapInput("I2", false, InputEditable.ANY, "I1,I3"); + inputs.add(input); + input = new MMapInput("I3", false, InputEditable.CONNECTOR_ONLY, "I1"); + inputs.add(input); + configs.add(new MConfig(configName1, inputs)); + + inputs = new LinkedList<MInput<?>>(); + input = new MStringInput("I3", false, InputEditable.ANY, StringUtils.EMPTY, (short) 30); + inputs.add(input); + input = new MMapInput("I4", false, InputEditable.ANY, StringUtils.EMPTY); + inputs.add(input); + configs.add(new MConfig(configName2, inputs)); + + return configs; + } + + protected List<MConfig> getBadConfigsWithNonExistingInputOverrides(String configName1, + String configName2) { + List<MConfig> configs = new LinkedList<MConfig>(); + + List<MInput<?>> inputs = new LinkedList<MInput<?>>(); + // I1 overrides another user_only attribute, hence a bad config + MInput input = new MStringInput("I1", false, InputEditable.USER_ONLY, "I2", (short) 30); + inputs.add(input); + input = new MMapInput("I2", false, InputEditable.USER_ONLY, "Foo"); + inputs.add(input); + configs.add(new MConfig(configName1, inputs)); + + inputs = new LinkedList<MInput<?>>(); + input = new MStringInput("I3", false, InputEditable.ANY, StringUtils.EMPTY, (short) 30); + inputs.add(input); + input = new MMapInput("I4", false, InputEditable.ANY, StringUtils.EMPTY); inputs.add(input); configs.add(new MConfig(configName2, inputs)); @@ -906,11 +1085,12 @@ abstract public class DerbyTestCase { } /** - * Find out number of entries in given table. + Find out number of entries in given table. - * @param table Table name - * @return Number of rows in the table - * @throws Exception + @param table + Table name + @return Number of rows in the table + @throws Exception / protected long countForTable(String table) throws Exception { Statement stmt = null; @@ -934,10 +1114,13 @@ abstract public class DerbyTestCase { } /* - * Assert row count for given table. - * @param table Table name - * @param expected Expected number of rows - * @throws Exception + Assert row count for given table. + + @param table + Table name + @param expected + Expected number of rows + @throws Exception / protected void assertCountForTable(String table, long expected) throws Exception { long count = countForTable(table); @@ -945,12 +1128,12 @@ abstract public class DerbyTestCase { } /** - * Printout repository content for advance debugging. + Printout repository content for advance debugging. - * This method is currently unused, but might be helpful in the future, so I'm - * letting it here. + This method is currently unused, but might be helpful in the future, so I'm + letting it here. * - * @throws Exception + @throws Exception / protected void generateDatabaseState() throws Exception { for (String tbl : new String[] { "SQ_CONNECTOR", "SQ_CONFIG", "SQ_INPUT", "SQ_LINK", @@ -960,9 +1143,11 @@ abstract public class DerbyTestCase { } /** - * Printout one single table. - * @param table Table name - * @throws Exception + Printout one single table. + + @param table + Table name + @throws Exception / protected void generateTableState(String table) throws Exception { PreparedStatement ps = null; http://git-wip-us.apache.org/repos/asf/sqoop/blob/6fc50b08/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestConnectorHandling.java ---------------------------------------------------------------------- diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestConnectorHandling.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestConnectorHandling.java index ca24398..1056c69 100644 --- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestConnectorHandling.java +++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestConnectorHandling.java @@ -17,6 +17,7 @@ / package org.apache.sqoop.repository.derby; +import org.apache.sqoop.common.SqoopException; import org.apache.sqoop.model.MConnector; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -91,10 +92,27 @@ public class TestConnectorHandling extends DerbyTestCase { assertEquals(connector, retrieved); } + @Test(expectedExceptions = SqoopException.class) + public void testRegisterConnectorWithIncorrectInputOverridesAttribute() throws Exception { + MConnector connector = getConnectorWithIncorrectOverridesAttribute(); + handler.registerConnector(connector, getDerbyDatabaseConnection()); + } + + @Test(expectedExceptions = SqoopException.class) + public void testRegisterConnectorWithIncorrectInputOverridesAttribute2() throws Exception { + MConnector connector = getConnectorWithIncorrectOverridesAttribute2(); + handler.registerConnector(connector, getDerbyDatabaseConnection()); + } + + @Test + public void testRegisterConnectorWithMultipleInputOverridesAttribute() throws Exception { + MConnector connector = getConnectorWithMultipleOverridesAttribute(); + handler.registerConnector(connector, getDerbyDatabaseConnection()); + } + @Test public void testFromDirection() throws Exception { MConnector connector = getConnector(true, false); - handler.registerConnector(connector, getDerbyDatabaseConnection()); // Connector should get persistence ID http://git-wip-us.apache.org/repos/asf/sqoop/blob/6fc50b08/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestDriverHandling.java ---------------------------------------------------------------------- diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestDriverHandling.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestDriverHandling.java index e12bf46..a2c50f8 100644 --- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestDriverHandling.java +++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestDriverHandling.java @@ -21,6 +21,7 @@ import static org.testng.AssertJUnit.assertEquals; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; +import org.apache.sqoop.common.SqoopException; import org.apache.sqoop.model.MDriver; import org.apache.sqoop.model.MDriverConfig; import org.testng.annotations.BeforeMethod; @@ -82,6 +83,11 @@ public class TestDriverHandling extends DerbyTestCase { assertEquals(driver.getVersion(), retrieved.getVersion()); } + @Test(expectedExceptions = SqoopException.class) + public void testRegisterBadDriver() throws Exception { + MDriver driver = getBadDriver(); + handler.registerDriver(driver, getDerbyDatabaseConnection()); + } @Test public void testDriverVersionUpgrade() throws Exception { http://git-wip-us.apache.org/repos/asf/sqoop/blob/6fc50b08/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInputTypes.java ---------------------------------------------------------------------- diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInputTypes.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInputTypes.java index fb07152..fe872ca 100644 --- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInputTypes.java +++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInputTypes.java @@ -26,6 +26,8 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import org.apache.commons.lang.StringUtils; +import org.apache.sqoop.model.InputEditable; import org.apache.sqoop.model.MBooleanInput; import org.apache.sqoop.model.MConfig; import org.apache.sqoop.model.MConnector; @@ -99,11 +101,22 @@ public class TestInputTypes extends DerbyTestCase { // Connection object with all various values MLink link = new MLink(connector.getPersistenceId(), connector.getLinkConfig()); MLinkConfig linkConfig = link.getConnectorLinkConfig(); - linkConfig.getStringInput("l1.String").setValue("A"); - linkConfig.getMapInput("l1.Map").setValue(map); - linkConfig.getIntegerInput("l1.Integer").setValue(1); - linkConfig.getBooleanInput("l1.Boolean").setValue(true); - linkConfig.getEnumInput("l1.Enum").setValue("YES"); + assertEquals(linkConfig.getStringInput("l1.I1").getEditable(), InputEditable.ANY); + assertEquals(linkConfig.getStringInput("l1.I1").getOverrides(), "l1.I2"); + assertEquals(linkConfig.getMapInput("l1.I2").getEditable(), InputEditable.CONNECTOR_ONLY); + assertEquals(linkConfig.getMapInput("l1.I2").getOverrides(), "l1.I5"); + assertEquals(linkConfig.getIntegerInput("l1.I3").getEditable(), InputEditable.ANY); + assertEquals(linkConfig.getIntegerInput("l1.I3").getOverrides(), "l1.I1"); + assertEquals(linkConfig.getBooleanInput("l1.I4").getEditable(), InputEditable.USER_ONLY); + assertEquals(linkConfig.getBooleanInput("l1.I4").getOverrides(), ""); + assertEquals(linkConfig.getEnumInput("l1.I5").getEditable(), InputEditable.ANY); + assertEquals(linkConfig.getEnumInput("l1.I5").getOverrides(), "l1.I4,l1.I3"); + + linkConfig.getStringInput("l1.I1").setValue("A"); + linkConfig.getMapInput("l1.I2").setValue(map); + linkConfig.getIntegerInput("l1.I3").setValue(1); + linkConfig.getBooleanInput("l1.I4").setValue(true); + linkConfig.getEnumInput("l1.I5").setValue("YES"); // Create the link in repository handler.createLink(link, getDerbyDatabaseConnection()); @@ -112,11 +125,14 @@ public class TestInputTypes extends DerbyTestCase { // Retrieve created link MLink retrieved = handler.findLink(link.getPersistenceId(), getDerbyDatabaseConnection()); linkConfig = retrieved.getConnectorLinkConfig(); - assertEquals("A", linkConfig.getStringInput("l1.String").getValue()); - assertEquals(map, linkConfig.getMapInput("l1.Map").getValue()); - assertEquals(1, (int) linkConfig.getIntegerInput("l1.Integer").getValue()); - assertEquals(true, (boolean) linkConfig.getBooleanInput("l1.Boolean").getValue()); - assertEquals("YES", linkConfig.getEnumInput("l1.Enum").getValue()); + assertEquals("A", linkConfig.getStringInput("l1.I1").getValue()); + assertEquals(map, linkConfig.getMapInput("l1.I2").getValue()); + assertEquals(1, (int) linkConfig.getIntegerInput("l1.I3").getValue()); + assertEquals(true, (boolean) linkConfig.getBooleanInput("l1.I4").getValue()); + assertEquals("YES", linkConfig.getEnumInput("l1.I5").getValue()); + assertEquals(linkConfig.getEnumInput("l1.I5").getEditable(), InputEditable.ANY); + assertEquals(linkConfig.getEnumInput("l1.I5").getOverrides(), "l1.I4,l1.I3"); + } /* @@ -133,19 +149,23 @@ public class TestInputTypes extends DerbyTestCase { inputs = new LinkedList<MInput<?>>(); - input = new MStringInput(configName1 + ".String", false, (short) 30); + input = new MStringInput(configName1 + ".I1", false, InputEditable.ANY, configName1 + ".I2", + (short) 30); inputs.add(input); - input = new MMapInput(configName1 + ".Map", false); + input = new MMapInput(configName1 + ".I2", false, InputEditable.CONNECTOR_ONLY, configName1 + + ".I5"); inputs.add(input); - input = new MIntegerInput(configName1 + ".Integer", false); + input = new MIntegerInput(configName1 + ".I3", false, InputEditable.ANY, configName1 + ".I1"); inputs.add(input); - input = new MBooleanInput(configName1 + ".Boolean", false); + input = new MBooleanInput(configName1 + ".I4", false, InputEditable.USER_ONLY, + StringUtils.EMPTY); inputs.add(input); - input = new MEnumInput(configName1 + ".Enum", false, new String[] { "YES", "NO" }); + input = new MEnumInput(configName1 + ".I5", false, InputEditable.ANY, configName1 + ".I4," + + configName1 + ".I3", new String[] { "YES", "NO" }); inputs.add(input); configs.add(new MConfig(configName1, inputs)); http://git-wip-us.apache.org/repos/asf/sqoop/blob/6fc50b08/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/PostgresqlTestCase.java ---------------------------------------------------------------------- diff --git a/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/PostgresqlTestCase.java b/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/PostgresqlTestCase.java index 08a3342..79381d8 100644 --- a/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/PostgresqlTestCase.java +++ b/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/PostgresqlTestCase.java @@ -17,6 +17,7 @@ / package org.apache.sqoop.integration.repository.postgresql; +import org.apache.commons.lang.StringUtils; import org.apache.sqoop.common.Direction; import org.apache.sqoop.common.test.db.DatabaseProvider; import org.apache.sqoop.common.test.db.PostgreSQLProvider; @@ -148,16 +149,16 @@ abstract public class PostgresqlTestCase { List<MConfig> configs = new LinkedList<MConfig>(); List<MInput<?>> inputs = new LinkedList<MInput<?>>(); - MInput input = new MStringInput("I1", false, (short) 30); + MInput input = new MStringInput("I1", false, InputEditable.ANY, StringUtils.EMPTY, (short) 30); inputs.add(input); - input = new MMapInput("I2", false); + input = new MMapInput("I2", false, InputEditable.ANY, StringUtils.EMPTY); inputs.add(input); configs.add(new MConfig(configName1, inputs)); inputs = new LinkedList<MInput<?>>(); - input = new MStringInput("I3", false, (short) 30); + input = new MStringInput("I3", false, InputEditable.ANY, StringUtils.EMPTY, (short) 30); inputs.add(input); - input = new MMapInput("I4", false); + input = new MMapInput("I4", false, InputEditable.ANY, StringUtils.EMPTY); inputs.add(input); configs.add(new MConfig(configName2, inputs)); http://git-wip-us.apache.org/repos/asf/sqoop/blob/6fc50b08/shell/src/main/java/org/apache/sqoop/shell/core/Constants.java ---------------------------------------------------------------------- diff --git a/shell/src/main/java/org/apache/sqoop/shell/core/Constants.java b/shell/src/main/java/org/apache/sqoop/shell/core/Constants.java index ca387d8..46f2b7f 100644 --- a/shell/src/main/java/org/apache/sqoop/shell/core/Constants.java +++ b/shell/src/main/java/org/apache/sqoop/shell/core/Constants.java @@ -372,6 +372,10 @@ public class Constants { "config.displayer.type"; public static final String RES_CONFIG_DISPLAYER_SENSITIVE = "config.displayer.sensitive"; + public static final String RES_CONFIG_DISPLAYER_EDITABLE = + "config.displayer.editable"; + public static final String RES_CONFIG_DISPLAYER_OVERRIDES = + "config.displayer.overrides"; public static final String RES_CONFIG_DISPLAYER_SIZE = "config.displayer.size"; public static final String RES_CONFIG_DISPLAYER_POSSIBLE_VALUES = http://git-wip-us.apache.org/repos/asf/sqoop/blob/6fc50b08/shell/src/main/java/org/apache/sqoop/shell/utils/ConfigDisplayer.java ---------------------------------------------------------------------- diff --git a/shell/src/main/java/org/apache/sqoop/shell/utils/ConfigDisplayer.java b/shell/src/main/java/org/apache/sqoop/shell/utils/ConfigDisplayer.java index e240163..aeb6d41 100644 --- a/shell/src/main/java/org/apache/sqoop/shell/utils/ConfigDisplayer.java +++ b/shell/src/main/java/org/apache/sqoop/shell/utils/ConfigDisplayer.java @@ -115,6 +115,11 @@ public final class ConfigDisplayer { println(input.getType()); print(" %s: ", resourceString(Constants.RES_CONFIG_DISPLAYER_SENSITIVE)); println(input.isSensitive()); + print(" %s: ", resourceString(Constants.RES_CONFIG_DISPLAYER_EDITABLE)); + println(input.getEditable()); + print(" %s: ", resourceString(Constants.RES_CONFIG_DISPLAYER_OVERRIDES)); + println(input.getOverrides()); + if (input.getType() == MInputType.STRING) { print(" %s: ", resourceString(Constants.RES_CONFIG_DISPLAYER_SIZE)); println(((MStringInput)input).getMaxLength()); http://git-wip-us.apache.org/repos/asf/sqoop/blob/6fc50b08/shell/src/main/java/org/apache/sqoop/shell/utils/ConfigFiller.java ---------------------------------------------------------------------- diff --git a/shell/src/main/java/org/apache/sqoop/shell/utils/ConfigFiller.java b/shell/src/main/java/org/apache/sqoop/shell/utils/ConfigFiller.java index de40818..02278f5 100644 --- a/shell/src/main/java/org/apache/sqoop/shell/utils/ConfigFiller.java +++ b/shell/src/main/java/org/apache/sqoop/shell/utils/ConfigFiller.java @@ -22,6 +22,7 @@ import jline.ConsoleReader; import org.apache.commons.cli.CommandLine; import org.apache.commons.lang.StringUtils; import org.apache.sqoop.common.Direction; +import org.apache.sqoop.model.InputEditable; import org.apache.sqoop.model.MBooleanInput; import org.apache.sqoop.model.MLink; import org.apache.sqoop.model.MEnumInput; @@ -53,7 +54,7 @@ public final class ConfigFiller { Internal input that will be reused for loading names for link and * job objects. / - private static MStringInput nameInput = new MStringInput("object-name", false, (short)25); + private static MStringInput nameInput = new MStringInput("object-name", false, InputEditable.ANY, StringUtils.EMPTY, (short)25); /* * Fill job object based on CLI options. http://git-wip-us.apache.org/repos/asf/sqoop/blob/6fc50b08/shell/src/main/resources/shell-resource.properties ---------------------------------------------------------------------- diff --git a/shell/src/main/resources/shell-resource.properties b/shell/src/main/resources/shell-resource.properties index 2b5a9b7..9c88235 100644 --- a/shell/src/main/resources/shell-resource.properties +++ b/shell/src/main/resources/shell-resource.properties @@ -211,6 +211,8 @@ config.displayer.help = Help config.displayer.input = Input config.displayer.type = Type config.displayer.sensitive = Sensitive +config.displayer.editable = Editable By +config.displayer.overrides = Overrides config.displayer.size = Size config.displayer.possible_values = Possible values config.displayer.unsupported_datatype = Unsupported data type
Mime	Unnamed text/plain (inline, 7-Bit, 88671 bytes)
	View raw message