sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From b...@apache.org
Subject svn commit: r1195852 [6/7] - in /incubator/sqoop/trunk/src/java: com/cloudera/sqoop/orm/ com/cloudera/sqoop/tool/ org/apache/sqoop/orm/ org/apache/sqoop/tool/
Date Tue, 01 Nov 2011 07:10:14 GMT
Added: incubator/sqoop/trunk/src/java/org/apache/sqoop/tool/ImportTool.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/org/apache/sqoop/tool/ImportTool.java?rev=1195852&view=auto
==============================================================================
--- incubator/sqoop/trunk/src/java/org/apache/sqoop/tool/ImportTool.java (added)
+++ incubator/sqoop/trunk/src/java/org/apache/sqoop/tool/ImportTool.java Tue Nov  1 07:10:13 2011
@@ -0,0 +1,897 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.tool;
+
+import java.io.IOException;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.Types;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import org.apache.hadoop.util.StringUtils;
+
+import com.cloudera.sqoop.Sqoop;
+import com.cloudera.sqoop.SqoopOptions;
+import com.cloudera.sqoop.SqoopOptions.FileLayout;
+import com.cloudera.sqoop.SqoopOptions.InvalidOptionsException;
+import com.cloudera.sqoop.cli.RelatedOptions;
+import com.cloudera.sqoop.cli.ToolOptions;
+import com.cloudera.sqoop.hive.HiveImport;
+import com.cloudera.sqoop.manager.ImportJobContext;
+
+import com.cloudera.sqoop.metastore.JobData;
+import com.cloudera.sqoop.metastore.JobStorage;
+import com.cloudera.sqoop.metastore.JobStorageFactory;
+import com.cloudera.sqoop.util.AppendUtils;
+import com.cloudera.sqoop.util.ImportException;
+import org.apache.hadoop.fs.Path;
+
+/**
+ * Tool that performs database imports to HDFS.
+ */
+public class ImportTool extends com.cloudera.sqoop.tool.BaseSqoopTool {
+
+  public static final Log LOG = LogFactory.getLog(ImportTool.class.getName());
+
+  private CodeGenTool codeGenerator;
+
+  // true if this is an all-tables import. Set by a subclass which
+  // overrides the run() method of this tool (which can only do
+  // a single table).
+  private boolean allTables;
+
+  // store check column type for incremental option
+  private int checkColumnType;
+
+  public ImportTool() {
+    this("import", false);
+  }
+
+  public ImportTool(String toolName, boolean allTables) {
+    super(toolName);
+    this.codeGenerator = new CodeGenTool();
+    this.allTables = allTables;
+  }
+
+  @Override
+  protected boolean init(SqoopOptions sqoopOpts) {
+    boolean ret = super.init(sqoopOpts);
+    codeGenerator.setManager(manager);
+    return ret;
+  }
+
+  /**
+   * @return a list of jar files generated as part of this import process
+   */
+  public List<String> getGeneratedJarFiles() {
+    return this.codeGenerator.getGeneratedJarFiles();
+  }
+
+  /**
+   * @return true if the supplied options specify an incremental import.
+   */
+  private boolean isIncremental(SqoopOptions options) {
+    return !options.getIncrementalMode().equals(
+        SqoopOptions.IncrementalMode.None);
+  }
+
+  /**
+   * If this is an incremental import, then we should save the
+   * user's state back to the metastore (if this job was run
+   * from the metastore). Otherwise, log to the user what data
+   * they need to supply next time.
+   */
+  private void saveIncrementalState(SqoopOptions options)
+      throws IOException {
+    if (!isIncremental(options)) {
+      return;
+    }
+
+    Map<String, String> descriptor = options.getStorageDescriptor();
+    String jobName = options.getJobName();
+
+    if (null != jobName && null != descriptor) {
+      // Actually save it back to the metastore.
+      LOG.info("Saving incremental import state to the metastore");
+      JobStorageFactory ssf = new JobStorageFactory(options.getConf());
+      JobStorage storage = ssf.getJobStorage(descriptor);
+      storage.open(descriptor);
+      try {
+        // Save the 'parent' SqoopOptions; this does not contain the mutations
+        // to the SqoopOptions state that occurred over the course of this
+        // execution, except for the one we specifically want to memorize:
+        // the latest value of the check column.
+        JobData data = new JobData(options.getParent(), this);
+        storage.update(jobName, data);
+        LOG.info("Updated data for job: " + jobName);
+      } finally {
+        storage.close();
+      }
+    } else {
+      // If there wasn't a parent SqoopOptions, then the incremental
+      // state data was stored in the current SqoopOptions.
+      LOG.info("Incremental import complete! To run another incremental "
+          + "import of all data following this import, supply the "
+          + "following arguments:");
+      SqoopOptions.IncrementalMode incrementalMode =
+          options.getIncrementalMode();
+      switch (incrementalMode) {
+      case AppendRows:
+        LOG.info(" --incremental append");
+        break;
+      case DateLastModified:
+        LOG.info(" --incremental lastmodified");
+        break;
+      default:
+        LOG.warn("Undefined incremental mode: " + incrementalMode);
+        break;
+      }
+      LOG.info("  --check-column " + options.getIncrementalTestColumn());
+      LOG.info("  --last-value " + options.getIncrementalLastValue());
+      LOG.info("(Consider saving this with 'sqoop job --create')");
+    }
+  }
+
+  /**
+   * Return the max value in the incremental-import test column. This
+   * value must be numeric.
+   */
+  private Object getMaxColumnId(SqoopOptions options) throws SQLException {
+    StringBuilder sb = new StringBuilder();
+    sb.append("SELECT MAX(");
+    sb.append(options.getIncrementalTestColumn());
+    sb.append(") FROM ");
+    sb.append(options.getTableName());
+
+    String where = options.getWhereClause();
+    if (null != where) {
+      sb.append(" WHERE ");
+      sb.append(where);
+    }
+
+    Connection conn = manager.getConnection();
+    Statement s = null;
+    ResultSet rs = null;
+    try {
+      s = conn.createStatement();
+      rs = s.executeQuery(sb.toString());
+      if (!rs.next()) {
+        // This probably means the table is empty.
+        LOG.warn("Unexpected: empty results for max value query?");
+        return null;
+      }
+
+      ResultSetMetaData rsmd = rs.getMetaData();
+      checkColumnType = rsmd.getColumnType(1);
+      if (checkColumnType == Types.TIMESTAMP) {
+        return rs.getTimestamp(1);
+      } else if (checkColumnType == Types.DATE) {
+        return rs.getDate(1);
+      } else if (checkColumnType == Types.TIME) {
+        return rs.getTime(1);
+      } else {
+        return rs.getObject(1);
+      }
+    } finally {
+      try {
+        if (null != rs) {
+          rs.close();
+        }
+      } catch (SQLException sqlE) {
+        LOG.warn("SQL Exception closing resultset: " + sqlE);
+      }
+
+      try {
+        if (null != s) {
+          s.close();
+        }
+      } catch (SQLException sqlE) {
+        LOG.warn("SQL Exception closing statement: " + sqlE);
+      }
+    }
+  }
+
+  /**
+   * Determine if a column is date/time.
+   * @return true if column type is TIMESTAMP, DATE, or TIME.
+   */
+  private boolean isDateTimeColumn(int columnType) {
+    return (columnType == Types.TIMESTAMP)
+        || (columnType == Types.DATE)
+        || (columnType == Types.TIME);
+  }
+
+  /**
+   * Initialize the constraints which set the incremental import range.
+   * @return false if an import is not necessary, because the dataset has not
+   * changed.
+   */
+  private boolean initIncrementalConstraints(SqoopOptions options,
+      ImportJobContext context) throws ImportException, IOException {
+
+    // If this is an incremental import, determine the constraints
+    // to inject in the WHERE clause or $CONDITIONS for a query.
+    // Also modify the 'last value' field of the SqoopOptions to
+    // specify the current job start time / start row.
+
+    if (!isIncremental(options)) {
+      return true;
+    }
+
+    SqoopOptions.IncrementalMode incrementalMode = options.getIncrementalMode();
+    String nextIncrementalValue = null;
+
+    Object nextVal;
+    switch (incrementalMode) {
+    case AppendRows:
+      try {
+        nextVal = getMaxColumnId(options);
+        if (isDateTimeColumn(checkColumnType)) {
+          nextIncrementalValue = (nextVal == null) ? null
+            : manager.datetimeToQueryString(nextVal.toString(),
+                                            checkColumnType);
+        } else {
+          nextIncrementalValue = (nextVal == null) ? null : nextVal.toString();
+        }
+      } catch (SQLException sqlE) {
+        throw new IOException(sqlE);
+      }
+      break;
+    case DateLastModified:
+      checkColumnType = Types.TIMESTAMP;
+      nextVal = manager.getCurrentDbTimestamp();
+      if (null == nextVal) {
+        throw new IOException("Could not get current time from database");
+      }
+      nextIncrementalValue = manager.datetimeToQueryString(nextVal.toString(),
+          checkColumnType);
+      break;
+    default:
+      throw new ImportException("Undefined incremental import type: "
+          + incrementalMode);
+    }
+
+    // Build the WHERE clause components that are used to import
+    // only this incremental section.
+    StringBuilder sb = new StringBuilder();
+    String prevEndpoint = options.getIncrementalLastValue();
+
+    if (isDateTimeColumn(checkColumnType) && null != prevEndpoint
+        && !prevEndpoint.startsWith("\'") && !prevEndpoint.endsWith("\'")) {
+      // Incremental imports based on date/time should be 'quoted' in
+      // ANSI SQL. If the user didn't specify single-quotes, put them
+      // around, here.
+      prevEndpoint = manager.datetimeToQueryString(prevEndpoint,
+          checkColumnType);
+    }
+
+    String checkColName = manager.escapeColName(
+        options.getIncrementalTestColumn());
+    LOG.info("Incremental import based on column " + checkColName);
+    if (null != prevEndpoint) {
+      if (prevEndpoint.equals(nextIncrementalValue)) {
+        LOG.info("No new rows detected since last import.");
+        return false;
+      }
+      LOG.info("Lower bound value: " + prevEndpoint);
+      sb.append(checkColName);
+      switch (incrementalMode) {
+      case AppendRows:
+        sb.append(" > ");
+        break;
+      case DateLastModified:
+        sb.append(" >= ");
+        break;
+      default:
+        throw new ImportException("Undefined comparison");
+      }
+      sb.append(prevEndpoint);
+      sb.append(" AND ");
+    }
+
+    if (null != nextIncrementalValue) {
+      sb.append(checkColName);
+      switch (incrementalMode) {
+      case AppendRows:
+        sb.append(" <= ");
+        break;
+      case DateLastModified:
+        sb.append(" < ");
+        break;
+      default:
+        throw new ImportException("Undefined comparison");
+      }
+      sb.append(nextIncrementalValue);
+    } else {
+      sb.append(checkColName);
+      sb.append(" IS NULL ");
+    }
+
+    LOG.info("Upper bound value: " + nextIncrementalValue);
+
+    String prevWhereClause = options.getWhereClause();
+    if (null != prevWhereClause) {
+      sb.append(" AND (");
+      sb.append(prevWhereClause);
+      sb.append(")");
+    }
+
+    String newConstraints = sb.toString();
+    options.setWhereClause(newConstraints);
+
+    // Save this state for next time.
+    SqoopOptions recordOptions = options.getParent();
+    if (null == recordOptions) {
+      recordOptions = options;
+    }
+    recordOptions.setIncrementalLastValue(
+        (nextVal == null) ? null : nextVal.toString());
+
+    return true;
+  }
+
+  /**
+   * Import a table or query.
+   * @return true if an import was performed, false otherwise.
+   */
+  protected boolean importTable(SqoopOptions options, String tableName,
+      HiveImport hiveImport) throws IOException, ImportException {
+    String jarFile = null;
+
+    // Generate the ORM code for the tables.
+    jarFile = codeGenerator.generateORM(options, tableName);
+
+    // Do the actual import.
+    ImportJobContext context = new ImportJobContext(tableName, jarFile,
+        options, getOutputPath(options, tableName));
+
+    // If we're doing an incremental import, set up the
+    // filtering conditions used to get the latest records.
+    if (!initIncrementalConstraints(options, context)) {
+      return false;
+    }
+
+    if (null != tableName) {
+      manager.importTable(context);
+    } else {
+      manager.importQuery(context);
+    }
+
+    if (options.isAppendMode()) {
+      AppendUtils app = new AppendUtils(context);
+      app.append();
+    }
+
+    // If the user wants this table to be in Hive, perform that post-load.
+    if (options.doHiveImport()) {
+      hiveImport.importTable(tableName, options.getHiveTableName(), false);
+    }
+
+    saveIncrementalState(options);
+
+    return true;
+  }
+
+  /**
+   * @return the output path for the imported files;
+   * in append mode this will point to a temporary folder.
+   * if importing to hbase, this may return null.
+   */
+  private Path getOutputPath(SqoopOptions options, String tableName) {
+    // Get output directory
+    String hdfsWarehouseDir = options.getWarehouseDir();
+    String hdfsTargetDir = options.getTargetDir();
+    Path outputPath = null;
+    if (options.isAppendMode()) {
+      // Use temporary path, later removed when appending
+      outputPath = AppendUtils.getTempAppendDir(tableName);
+      LOG.debug("Using temporary folder: " + outputPath.getName());
+    } else {
+      // Try in this order: target-dir or warehouse-dir
+      if (hdfsTargetDir != null) {
+        outputPath = new Path(hdfsTargetDir);
+      } else if (hdfsWarehouseDir != null) {
+        outputPath = new Path(hdfsWarehouseDir, tableName);
+      } else if (null != tableName) {
+        outputPath = new Path(tableName);
+      }
+    }
+
+    return outputPath;
+  }
+
+  @Override
+  /** {@inheritDoc} */
+  public int run(SqoopOptions options) {
+    HiveImport hiveImport = null;
+
+    if (allTables) {
+      // We got into this method, but we should be in a subclass.
+      // (This method only handles a single table)
+      // This should not be reached, but for sanity's sake, test here.
+      LOG.error("ImportTool.run() can only handle a single table.");
+      return 1;
+    }
+
+    if (!init(options)) {
+      return 1;
+    }
+
+    codeGenerator.setManager(manager);
+
+    try {
+      if (options.doHiveImport()) {
+        hiveImport = new HiveImport(options, manager, options.getConf(), false);
+      }
+
+      // Import a single table (or query) the user specified.
+      importTable(options, options.getTableName(), hiveImport);
+    } catch (IllegalArgumentException iea) {
+        LOG.error("Imported Failed: " + iea.getMessage());
+        if (System.getProperty(Sqoop.SQOOP_RETHROW_PROPERTY) != null) {
+          throw iea;
+        }
+        return 1;
+    } catch (IOException ioe) {
+      LOG.error("Encountered IOException running import job: "
+          + StringUtils.stringifyException(ioe));
+      if (System.getProperty(Sqoop.SQOOP_RETHROW_PROPERTY) != null) {
+        throw new RuntimeException(ioe);
+      } else {
+        return 1;
+      }
+    } catch (ImportException ie) {
+      LOG.error("Error during import: " + ie.toString());
+      if (System.getProperty(Sqoop.SQOOP_RETHROW_PROPERTY) != null) {
+        throw new RuntimeException(ie);
+      } else {
+        return 1;
+      }
+    } finally {
+      destroy(options);
+    }
+
+    return 0;
+  }
+
+  /**
+   * Construct the set of options that control imports, either of one
+   * table or a batch of tables.
+   * @return the RelatedOptions that can be used to parse the import
+   * arguments.
+   */
+  protected RelatedOptions getImportOptions() {
+    // Imports
+    RelatedOptions importOpts = new RelatedOptions("Import control arguments");
+
+    importOpts.addOption(OptionBuilder
+        .withDescription("Use direct import fast path")
+        .withLongOpt(DIRECT_ARG)
+        .create());
+
+    if (!allTables) {
+      importOpts.addOption(OptionBuilder.withArgName("table-name")
+          .hasArg().withDescription("Table to read")
+          .withLongOpt(TABLE_ARG)
+          .create());
+      importOpts.addOption(OptionBuilder.withArgName("col,col,col...")
+          .hasArg().withDescription("Columns to import from table")
+          .withLongOpt(COLUMNS_ARG)
+          .create());
+      importOpts.addOption(OptionBuilder.withArgName("column-name")
+          .hasArg()
+          .withDescription("Column of the table used to split work units")
+          .withLongOpt(SPLIT_BY_ARG)
+          .create());
+      importOpts.addOption(OptionBuilder.withArgName("where clause")
+          .hasArg().withDescription("WHERE clause to use during import")
+          .withLongOpt(WHERE_ARG)
+          .create());
+      importOpts.addOption(OptionBuilder
+          .withDescription("Imports data in append mode")
+          .withLongOpt(APPEND_ARG)
+          .create());
+      importOpts.addOption(OptionBuilder.withArgName("dir")
+          .hasArg().withDescription("HDFS plain table destination")
+          .withLongOpt(TARGET_DIR_ARG)
+          .create());
+      importOpts.addOption(OptionBuilder.withArgName("statement")
+          .hasArg()
+          .withDescription("Import results of SQL 'statement'")
+          .withLongOpt(SQL_QUERY_ARG)
+          .create(SQL_QUERY_SHORT_ARG));
+      importOpts.addOption(OptionBuilder.withArgName("statement")
+          .hasArg()
+          .withDescription("Set boundary query for retrieving max and min"
+              + " value of the primary key")
+          .withLongOpt(SQL_QUERY_BOUNDARY)
+          .create());
+    }
+
+    importOpts.addOption(OptionBuilder.withArgName("dir")
+        .hasArg().withDescription("HDFS parent for table destination")
+        .withLongOpt(WAREHOUSE_DIR_ARG)
+        .create());
+    importOpts.addOption(OptionBuilder
+        .withDescription("Imports data to SequenceFiles")
+        .withLongOpt(FMT_SEQUENCEFILE_ARG)
+        .create());
+    importOpts.addOption(OptionBuilder
+        .withDescription("Imports data as plain text (default)")
+        .withLongOpt(FMT_TEXTFILE_ARG)
+        .create());
+    importOpts.addOption(OptionBuilder
+        .withDescription("Imports data to Avro data files")
+        .withLongOpt(FMT_AVRODATAFILE_ARG)
+        .create());
+    importOpts.addOption(OptionBuilder.withArgName("n")
+        .hasArg().withDescription("Use 'n' map tasks to import in parallel")
+        .withLongOpt(NUM_MAPPERS_ARG)
+        .create(NUM_MAPPERS_SHORT_ARG));
+    importOpts.addOption(OptionBuilder
+        .withDescription("Enable compression")
+        .withLongOpt(COMPRESS_ARG)
+        .create(COMPRESS_SHORT_ARG));
+    importOpts.addOption(OptionBuilder.withArgName("codec")
+        .hasArg()
+        .withDescription("Compression codec to use for import")
+        .withLongOpt(COMPRESSION_CODEC_ARG)
+        .create());
+    importOpts.addOption(OptionBuilder.withArgName("n")
+        .hasArg()
+        .withDescription("Split the input stream every 'n' bytes "
+        + "when importing in direct mode")
+        .withLongOpt(DIRECT_SPLIT_SIZE_ARG)
+        .create());
+    importOpts.addOption(OptionBuilder.withArgName("n")
+        .hasArg()
+        .withDescription("Set the maximum size for an inline LOB")
+        .withLongOpt(INLINE_LOB_LIMIT_ARG)
+        .create());
+    importOpts.addOption(OptionBuilder.withArgName("n")
+        .hasArg()
+        .withDescription("Set number 'n' of rows to fetch from the "
+        + "database when more rows are needed")
+        .withLongOpt(FETCH_SIZE_ARG)
+        .create());
+
+    return importOpts;
+  }
+
+  /**
+   * Return options for incremental import.
+   */
+  protected RelatedOptions getIncrementalOptions() {
+    RelatedOptions incrementalOpts =
+        new RelatedOptions("Incremental import arguments");
+
+    incrementalOpts.addOption(OptionBuilder.withArgName("import-type")
+        .hasArg()
+        .withDescription(
+        "Define an incremental import of type 'append' or 'lastmodified'")
+        .withLongOpt(INCREMENT_TYPE_ARG)
+        .create());
+    incrementalOpts.addOption(OptionBuilder.withArgName("column")
+        .hasArg()
+        .withDescription("Source column to check for incremental change")
+        .withLongOpt(INCREMENT_COL_ARG)
+        .create());
+    incrementalOpts.addOption(OptionBuilder.withArgName("value")
+        .hasArg()
+        .withDescription("Last imported value in the incremental check column")
+        .withLongOpt(INCREMENT_LAST_VAL_ARG)
+        .create());
+
+    return incrementalOpts;
+  }
+
+  @Override
+  /** Configure the command-line arguments we expect to receive */
+  public void configureOptions(ToolOptions toolOptions) {
+
+    toolOptions.addUniqueOptions(getCommonOptions());
+    toolOptions.addUniqueOptions(getImportOptions());
+    if (!allTables) {
+      toolOptions.addUniqueOptions(getIncrementalOptions());
+    }
+    toolOptions.addUniqueOptions(getOutputFormatOptions());
+    toolOptions.addUniqueOptions(getInputFormatOptions());
+    toolOptions.addUniqueOptions(getHiveOptions(true));
+    toolOptions.addUniqueOptions(getHBaseOptions());
+
+    // get common codegen opts.
+    RelatedOptions codeGenOpts = getCodeGenOpts(allTables);
+
+    // add import-specific codegen opts:
+    codeGenOpts.addOption(OptionBuilder.withArgName("file")
+        .hasArg()
+        .withDescription("Disable code generation; use specified jar")
+        .withLongOpt(JAR_FILE_NAME_ARG)
+        .create());
+
+    toolOptions.addUniqueOptions(codeGenOpts);
+  }
+
+  @Override
+  /** {@inheritDoc} */
+  public void printHelp(ToolOptions toolOptions) {
+    super.printHelp(toolOptions);
+    System.out.println("");
+    if (allTables) {
+      System.out.println("At minimum, you must specify --connect");
+    } else {
+      System.out.println(
+          "At minimum, you must specify --connect and --table");
+    }
+
+    System.out.println(
+        "Arguments to mysqldump and other subprograms may be supplied");
+    System.out.println(
+        "after a '--' on the command line.");
+  }
+
+  private void applyIncrementalOptions(CommandLine in, SqoopOptions out)
+      throws InvalidOptionsException  {
+    if (in.hasOption(INCREMENT_TYPE_ARG)) {
+      String incrementalTypeStr = in.getOptionValue(INCREMENT_TYPE_ARG);
+      if ("append".equals(incrementalTypeStr)) {
+        out.setIncrementalMode(SqoopOptions.IncrementalMode.AppendRows);
+        // This argument implies ability to append to the same directory.
+        out.setAppendMode(true);
+      } else if ("lastmodified".equals(incrementalTypeStr)) {
+        out.setIncrementalMode(SqoopOptions.IncrementalMode.DateLastModified);
+      } else {
+        throw new InvalidOptionsException("Unknown incremental import mode: "
+            + incrementalTypeStr + ". Use 'append' or 'lastmodified'."
+            + HELP_STR);
+      }
+    }
+
+    if (in.hasOption(INCREMENT_COL_ARG)) {
+      out.setIncrementalTestColumn(in.getOptionValue(INCREMENT_COL_ARG));
+    }
+
+    if (in.hasOption(INCREMENT_LAST_VAL_ARG)) {
+      out.setIncrementalLastValue(in.getOptionValue(INCREMENT_LAST_VAL_ARG));
+    }
+  }
+
+  @Override
+  /** {@inheritDoc} */
+  public void applyOptions(CommandLine in, SqoopOptions out)
+      throws InvalidOptionsException {
+
+    try {
+      applyCommonOptions(in, out);
+
+      if (in.hasOption(DIRECT_ARG)) {
+        out.setDirectMode(true);
+      }
+
+      if (!allTables) {
+        if (in.hasOption(TABLE_ARG)) {
+          out.setTableName(in.getOptionValue(TABLE_ARG));
+        }
+
+        if (in.hasOption(COLUMNS_ARG)) {
+          String[] cols= in.getOptionValue(COLUMNS_ARG).split(",");
+          for (int i=0; i<cols.length; i++) {
+            cols[i] = cols[i].trim();
+          }
+          out.setColumns(cols);
+        }
+
+        if (in.hasOption(SPLIT_BY_ARG)) {
+          out.setSplitByCol(in.getOptionValue(SPLIT_BY_ARG));
+        }
+
+        if (in.hasOption(WHERE_ARG)) {
+          out.setWhereClause(in.getOptionValue(WHERE_ARG));
+        }
+
+        if (in.hasOption(TARGET_DIR_ARG)) {
+          out.setTargetDir(in.getOptionValue(TARGET_DIR_ARG));
+        }
+
+        if (in.hasOption(APPEND_ARG)) {
+          out.setAppendMode(true);
+        }
+
+        if (in.hasOption(SQL_QUERY_ARG)) {
+          out.setSqlQuery(in.getOptionValue(SQL_QUERY_ARG));
+        }
+
+        if(in.hasOption(SQL_QUERY_BOUNDARY)) {
+          out.setBoundaryQuery(in.getOptionValue(SQL_QUERY_BOUNDARY));
+        }
+      }
+
+      if (in.hasOption(WAREHOUSE_DIR_ARG)) {
+        out.setWarehouseDir(in.getOptionValue(WAREHOUSE_DIR_ARG));
+      }
+
+      if (in.hasOption(FMT_SEQUENCEFILE_ARG)) {
+        out.setFileLayout(SqoopOptions.FileLayout.SequenceFile);
+      }
+
+      if (in.hasOption(FMT_TEXTFILE_ARG)) {
+        out.setFileLayout(SqoopOptions.FileLayout.TextFile);
+      }
+
+      if (in.hasOption(FMT_AVRODATAFILE_ARG)) {
+        out.setFileLayout(SqoopOptions.FileLayout.AvroDataFile);
+      }
+
+      if (in.hasOption(NUM_MAPPERS_ARG)) {
+        out.setNumMappers(Integer.parseInt(in.getOptionValue(NUM_MAPPERS_ARG)));
+      }
+
+      if (in.hasOption(COMPRESS_ARG)) {
+        out.setUseCompression(true);
+      }
+
+      if (in.hasOption(COMPRESSION_CODEC_ARG)) {
+        out.setCompressionCodec(in.getOptionValue(COMPRESSION_CODEC_ARG));
+      }
+
+      if (in.hasOption(DIRECT_SPLIT_SIZE_ARG)) {
+        out.setDirectSplitSize(Long.parseLong(in.getOptionValue(
+            DIRECT_SPLIT_SIZE_ARG)));
+      }
+
+      if (in.hasOption(INLINE_LOB_LIMIT_ARG)) {
+        out.setInlineLobLimit(Long.parseLong(in.getOptionValue(
+            INLINE_LOB_LIMIT_ARG)));
+      }
+
+      if (in.hasOption(FETCH_SIZE_ARG)) {
+        out.setFetchSize(new Integer(in.getOptionValue(FETCH_SIZE_ARG)));
+      }
+
+      if (in.hasOption(JAR_FILE_NAME_ARG)) {
+        out.setExistingJarName(in.getOptionValue(JAR_FILE_NAME_ARG));
+      }
+
+      applyIncrementalOptions(in, out);
+      applyHiveOptions(in, out);
+      applyOutputFormatOptions(in, out);
+      applyInputFormatOptions(in, out);
+      applyCodeGenOptions(in, out, allTables);
+      applyHBaseOptions(in, out);
+    } catch (NumberFormatException nfe) {
+      throw new InvalidOptionsException("Error: expected numeric argument.\n"
+          + "Try --help for usage.");
+    }
+  }
+
+  /**
+   * Validate import-specific arguments.
+   * @param options the configured SqoopOptions to check
+   */
+  protected void validateImportOptions(SqoopOptions options)
+      throws InvalidOptionsException {
+    if (!allTables && options.getTableName() == null
+        && options.getSqlQuery() == null) {
+      throw new InvalidOptionsException(
+          "--table or --" + SQL_QUERY_ARG + " is required for import. "
+          + "(Or use sqoop import-all-tables.)"
+          + HELP_STR);
+    } else if (options.getExistingJarName() != null
+        && options.getClassName() == null) {
+      throw new InvalidOptionsException("Jar specified with --jar-file, but no "
+          + "class specified with --class-name." + HELP_STR);
+    } else if (options.getTargetDir() != null
+        && options.getWarehouseDir() != null) {
+      throw new InvalidOptionsException(
+          "--target-dir with --warehouse-dir are incompatible options."
+          + HELP_STR);
+    } else if (options.getTableName() != null
+        && options.getSqlQuery() != null) {
+      throw new InvalidOptionsException(
+          "Cannot specify --" + SQL_QUERY_ARG + " and --table together."
+          + HELP_STR);
+    } else if (options.getSqlQuery() != null
+        && options.getTargetDir() == null && options.getHBaseTable() == null) {
+      throw new InvalidOptionsException(
+          "Must specify destination with --target-dir."
+          + HELP_STR);
+    } else if (options.getSqlQuery() != null && options.doHiveImport()
+        && options.getHiveTableName() == null) {
+      throw new InvalidOptionsException(
+          "When importing a query to Hive, you must specify --"
+          + HIVE_TABLE_ARG + "." + HELP_STR);
+    } else if (options.getSqlQuery() != null && options.getNumMappers() > 1
+        && options.getSplitByCol() == null) {
+      throw new InvalidOptionsException(
+          "When importing query results in parallel, you must specify --"
+          + SPLIT_BY_ARG + "." + HELP_STR);
+    } else if (options.isDirect()
+            && options.getFileLayout() != SqoopOptions.FileLayout.TextFile
+            && options.getConnectString().contains("jdbc:mysql://")) {
+      throw new InvalidOptionsException(
+            "MySQL direct export currently supports only text output format."
+             + "Parameters --as-sequencefile and --as-avrodatafile are not "
+             + "supported with --direct params in MySQL case.");
+    } else if (!options.getMapColumnJava().isEmpty()
+            && options.getFileLayout() == FileLayout.AvroDataFile) {
+      throw new InvalidOptionsException(
+              "Overriding column types is currently not supported with avro.");
+    }
+  }
+
+  /**
+   * Validate the incremental import options.
+   */
+  private void validateIncrementalOptions(SqoopOptions options)
+      throws InvalidOptionsException {
+    if (options.getIncrementalMode() != SqoopOptions.IncrementalMode.None
+        && options.getIncrementalTestColumn() == null) {
+      throw new InvalidOptionsException(
+          "For an incremental import, the check column must be specified "
+          + "with --" + INCREMENT_COL_ARG + ". " + HELP_STR);
+    }
+
+    if (options.getIncrementalMode() == SqoopOptions.IncrementalMode.None
+        && options.getIncrementalTestColumn() != null) {
+      throw new InvalidOptionsException(
+          "You must specify an incremental import mode with --"
+          + INCREMENT_TYPE_ARG + ". " + HELP_STR);
+    }
+
+    if (options.getIncrementalMode() != SqoopOptions.IncrementalMode.None
+        && options.getTableName() == null) {
+      throw new InvalidOptionsException("Incremental imports require a table."
+          + HELP_STR);
+    }
+  }
+
+  @Override
+  /** {@inheritDoc} */
+  public void validateOptions(SqoopOptions options)
+      throws InvalidOptionsException {
+
+    // If extraArguments is full, check for '--' followed by args for
+    // mysqldump or other commands we rely on.
+    options.setExtraArgs(getSubcommandArgs(extraArguments));
+    int dashPos = getDashPosition(extraArguments);
+
+    if (hasUnrecognizedArgs(extraArguments, 0, dashPos)) {
+      throw new InvalidOptionsException(HELP_STR);
+    }
+
+    validateImportOptions(options);
+    validateIncrementalOptions(options);
+    validateCommonOptions(options);
+    validateCodeGenOptions(options);
+    validateOutputFormatOptions(options);
+    validateHBaseOptions(options);
+    validateHiveOptions(options);
+  }
+}
+

Propchange: incubator/sqoop/trunk/src/java/org/apache/sqoop/tool/ImportTool.java
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Added: incubator/sqoop/trunk/src/java/org/apache/sqoop/tool/JobTool.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/org/apache/sqoop/tool/JobTool.java?rev=1195852&view=auto
==============================================================================
--- incubator/sqoop/trunk/src/java/org/apache/sqoop/tool/JobTool.java (added)
+++ incubator/sqoop/trunk/src/java/org/apache/sqoop/tool/JobTool.java Tue Nov  1 07:10:13 2011
@@ -0,0 +1,406 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.tool;
+
+import java.io.IOException;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.TreeMap;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.ToolRunner;
+
+import org.apache.log4j.Category;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+import com.cloudera.sqoop.Sqoop;
+import com.cloudera.sqoop.SqoopOptions;
+import com.cloudera.sqoop.SqoopOptions.InvalidOptionsException;
+import com.cloudera.sqoop.cli.ToolOptions;
+import com.cloudera.sqoop.metastore.hsqldb.HsqldbJobStorage;
+import com.cloudera.sqoop.metastore.JobData;
+import com.cloudera.sqoop.metastore.JobStorage;
+import com.cloudera.sqoop.metastore.JobStorageFactory;
+
+/**
+ * Tool that creates and executes saved jobs.
+ */
+public class JobTool extends com.cloudera.sqoop.tool.BaseSqoopTool {
+
+  public static final Log LOG = LogFactory.getLog(
+      JobTool.class.getName());
+
+  private enum JobOp {
+    JobCreate,
+    JobDelete,
+    JobExecute,
+    JobList,
+    JobShow,
+  };
+
+  private Map<String, String> storageDescriptor;
+  private String jobName;
+  private JobOp operation;
+  private JobStorage storage;
+
+  public JobTool() {
+    super("job");
+  }
+
+  /**
+   * Given an array of strings, return all elements of this
+   * array up to (but not including) the first instance of "--".
+   */
+  private String [] getElementsUpToDoubleDash(String [] array) {
+    String [] parseableChildArgv = null;
+    for (int i = 0; i < array.length; i++) {
+      if ("--".equals(array[i])) {
+        parseableChildArgv = Arrays.copyOfRange(array, 0, i);
+        break;
+      }
+    }
+
+    if (parseableChildArgv == null) {
+      // Didn't find any nested '--'.
+      parseableChildArgv = array;
+    }
+
+    return parseableChildArgv;
+  }
+
+  /**
+   * Given an array of strings, return the first instance
+   * of "--" and all following elements.
+   * If no "--" exists, return null.
+   */
+  private String [] getElementsAfterDoubleDash(String [] array) {
+    String [] extraChildArgv = null;
+    for (int i = 0; i < array.length; i++) {
+      if ("--".equals(array[i])) {
+        extraChildArgv = Arrays.copyOfRange(array, i, array.length);
+        break;
+      }
+    }
+
+    return extraChildArgv;
+  }
+
+  private int configureChildTool(SqoopOptions childOptions,
+      SqoopTool childTool, String [] childArgv) {
+    // Within the child arguments there may be a '--' followed by
+    // dependent args. Stash them off to the side.
+
+    // Everything up to the '--'.
+    String [] parseableChildArgv = getElementsUpToDoubleDash(childArgv);
+
+    // The '--' and any subsequent args.
+    String [] extraChildArgv = getElementsAfterDoubleDash(childArgv);
+
+    // Now feed the arguments into the tool itself.
+    try {
+      childOptions = childTool.parseArguments(parseableChildArgv,
+          null, childOptions, false);
+      childTool.appendArgs(extraChildArgv);
+      childTool.validateOptions(childOptions);
+    } catch (ParseException pe) {
+      LOG.error("Error parsing arguments to the job-specific tool.");
+      LOG.error("See 'sqoop help <tool>' for usage.");
+      return 1;
+    } catch (SqoopOptions.InvalidOptionsException e) {
+      System.err.println(e.getMessage());
+      return 1;
+    }
+
+    return 0; // Success.
+  }
+
+  private int createJob(SqoopOptions options) throws IOException {
+    // In our extraArguments array, we should have a '--' followed by
+    // a tool name, and any tool-specific arguments.
+    // Create an instance of the named tool and then configure it to
+    // get a SqoopOptions out which we will serialize into a job.
+    int dashPos = getDashPosition(extraArguments);
+    int toolArgPos = dashPos + 1;
+    if (null == extraArguments || toolArgPos < 0
+        || toolArgPos >= extraArguments.length) {
+      LOG.error("No tool specified; cannot create a job.");
+      LOG.error("Use: sqoop job --create <job-name> "
+          + "-- <tool-name> [tool-args]");
+      return 1;
+    }
+
+    String jobToolName = extraArguments[toolArgPos];
+    SqoopTool jobTool = SqoopTool.getTool(jobToolName);
+    if (null == jobTool) {
+      LOG.error("No such tool available: " + jobToolName);
+      return 1;
+    }
+
+    // Create a SqoopOptions and Configuration based on the current one,
+    // but deep-copied. This will be populated within the job.
+    SqoopOptions jobOptions = new SqoopOptions();
+    jobOptions.setConf(new Configuration(options.getConf()));
+
+    // Get the arguments to feed to the child tool.
+    String [] childArgs = Arrays.copyOfRange(extraArguments, toolArgPos + 1,
+        extraArguments.length);
+
+    int confRet = configureChildTool(jobOptions, jobTool, childArgs);
+    if (0 != confRet) {
+      // Error.
+      return confRet;
+    }
+
+    // Now that the tool is fully configured, materialize the job.
+    //TODO(jarcec): Remove the cast when JobData will be moved to apache package
+    JobData jobData = new JobData(jobOptions,
+            (com.cloudera.sqoop.tool.SqoopTool)jobTool);
+    this.storage.create(jobName, jobData);
+    return 0; // Success.
+  }
+
+  private int listJobs(SqoopOptions opts) throws IOException {
+    List<String> jobNames = storage.list();
+    System.out.println("Available jobs:");
+    for (String name : jobNames) {
+      System.out.println("  " + name);
+    }
+    return 0;
+  }
+
+  private int deleteJob(SqoopOptions opts) throws IOException {
+    this.storage.delete(jobName);
+    return 0;
+  }
+
+  private int execJob(SqoopOptions opts) throws IOException {
+    JobData data = this.storage.read(jobName);
+    if (null == data) {
+      LOG.error("No such job: " + jobName);
+      return 1;
+    }
+
+    SqoopOptions childOpts = data.getSqoopOptions();
+    SqoopTool childTool = data.getSqoopTool();
+
+    // Don't overwrite the original SqoopOptions with the
+    // arguments; make a child options.
+
+    SqoopOptions clonedOpts = (SqoopOptions) childOpts.clone();
+    clonedOpts.setParent(childOpts);
+
+    int dashPos = getDashPosition(extraArguments);
+    String [] childArgv;
+    if (dashPos >= extraArguments.length) {
+      childArgv = new String[0];
+    } else {
+      childArgv = Arrays.copyOfRange(extraArguments, dashPos + 1,
+          extraArguments.length);
+    }
+
+    int confRet = configureChildTool(clonedOpts, childTool, childArgv);
+    if (0 != confRet) {
+      // Error.
+      return confRet;
+    }
+
+    return childTool.run(clonedOpts);
+  }
+
+  private int showJob(SqoopOptions opts) throws IOException {
+    JobData data = this.storage.read(jobName);
+    if (null == data) {
+      LOG.error("No such job: " + jobName);
+      return 1;
+    }
+
+    SqoopOptions childOpts = data.getSqoopOptions();
+    SqoopTool childTool = data.getSqoopTool();
+
+    System.out.println("Job: " + jobName);
+    System.out.println("Tool: " + childTool.getToolName());
+
+    System.out.println("Options:");
+    System.out.println("----------------------------");
+    Properties props = childOpts.writeProperties();
+    for (Map.Entry<Object, Object> entry : props.entrySet()) {
+      System.out.println(entry.getKey().toString() + " = " + entry.getValue());
+    }
+
+    // TODO: This does not show entries in the Configuration
+    // (SqoopOptions.getConf()) which were stored as different from the
+    // default.
+
+    return 0;
+  }
+
+  @Override
+  /** {@inheritDoc} */
+  public int run(SqoopOptions options) {
+    // Get a JobStorage instance to use to materialize this job.
+    JobStorageFactory ssf = new JobStorageFactory(options.getConf());
+    this.storage = ssf.getJobStorage(storageDescriptor);
+    if (null == this.storage) {
+      LOG.error("There is no JobStorage implementation available");
+      LOG.error("that can read your specified storage descriptor.");
+      LOG.error("Don't know where to save this job info! You may");
+      LOG.error("need to specify the connect string with --meta-connect.");
+      return 1;
+    }
+
+    try {
+      // Open the storage layer.
+      this.storage.open(this.storageDescriptor);
+
+      // And now determine what operation to perform with it.
+      switch (operation) {
+      case JobCreate:
+        return createJob(options);
+      case JobDelete:
+        return deleteJob(options);
+      case JobExecute:
+        return execJob(options);
+      case JobList:
+        return listJobs(options);
+      case JobShow:
+        return showJob(options);
+      default:
+        LOG.error("Undefined job operation: " + operation);
+        return 1;
+      }
+    } catch (IOException ioe) {
+      LOG.error("I/O error performing job operation: "
+          + StringUtils.stringifyException(ioe));
+      return 1;
+    } finally {
+      if (null != this.storage) {
+        try {
+          storage.close();
+        } catch (IOException ioe) {
+          LOG.warn("IOException closing JobStorage: "
+              + StringUtils.stringifyException(ioe));
+        }
+      }
+    }
+  }
+
+  @Override
+  /** Configure the command-line arguments we expect to receive */
+  public void configureOptions(ToolOptions toolOptions) {
+    toolOptions.addUniqueOptions(getJobOptions());
+  }
+
+  @Override
+  /** {@inheritDoc} */
+  public void applyOptions(CommandLine in, SqoopOptions out)
+      throws InvalidOptionsException {
+
+    if (in.hasOption(VERBOSE_ARG)) {
+      // Immediately switch into DEBUG logging.
+      Category sqoopLogger = Logger.getLogger(
+          Sqoop.class.getName()).getParent();
+      sqoopLogger.setLevel(Level.DEBUG);
+      LOG.debug("Enabled debug logging.");
+    }
+
+    if (in.hasOption(HELP_ARG)) {
+      ToolOptions toolOpts = new ToolOptions();
+      configureOptions(toolOpts);
+      printHelp(toolOpts);
+      throw new InvalidOptionsException("");
+    }
+
+    this.storageDescriptor = new TreeMap<String, String>();
+
+    if (in.hasOption(STORAGE_METASTORE_ARG)) {
+      this.storageDescriptor.put(HsqldbJobStorage.META_CONNECT_KEY,
+          in.getOptionValue(STORAGE_METASTORE_ARG));
+    }
+
+    // These are generated via an option group; exactly one
+    // of this exhaustive list will always be selected.
+    if (in.hasOption(JOB_CMD_CREATE_ARG)) {
+      this.operation = JobOp.JobCreate;
+      this.jobName = in.getOptionValue(JOB_CMD_CREATE_ARG);
+    } else if (in.hasOption(JOB_CMD_DELETE_ARG)) {
+      this.operation = JobOp.JobDelete;
+      this.jobName = in.getOptionValue(JOB_CMD_DELETE_ARG);
+    } else if (in.hasOption(JOB_CMD_EXEC_ARG)) {
+      this.operation = JobOp.JobExecute;
+      this.jobName = in.getOptionValue(JOB_CMD_EXEC_ARG);
+    } else if (in.hasOption(JOB_CMD_LIST_ARG)) {
+      this.operation = JobOp.JobList;
+    } else if (in.hasOption(JOB_CMD_SHOW_ARG)) {
+      this.operation = JobOp.JobShow;
+      this.jobName = in.getOptionValue(JOB_CMD_SHOW_ARG);
+    }
+  }
+
+  @Override
+  /** {@inheritDoc} */
+  public void validateOptions(SqoopOptions options)
+      throws InvalidOptionsException {
+
+    if (null == operation
+        || (null == this.jobName && operation != JobOp.JobList)) {
+      throw new InvalidOptionsException("No job operation specified"
+          + HELP_STR);
+    }
+
+    if (operation == JobOp.JobCreate) {
+      // Check that we have a '--' followed by at least a tool name.
+      if (extraArguments == null || extraArguments.length == 0) {
+        throw new InvalidOptionsException(
+            "Expected: -- <tool-name> [tool-args] "
+            + HELP_STR);
+      }
+    }
+
+    int dashPos = getDashPosition(extraArguments);
+    if (hasUnrecognizedArgs(extraArguments, 0, dashPos)) {
+      throw new InvalidOptionsException(HELP_STR);
+    }
+  }
+
+  @Override
+  /** {@inheritDoc} */
+  public void printHelp(ToolOptions opts) {
+    System.out.println("usage: sqoop " + getToolName()
+        + " [GENERIC-ARGS] [JOB-ARGS] [-- [<tool-name>] [TOOL-ARGS]]");
+    System.out.println("");
+
+    opts.printHelp();
+
+    System.out.println("");
+    System.out.println("Generic Hadoop command-line arguments:");
+    System.out.println("(must preceed any tool-specific arguments)");
+    ToolRunner.printGenericCommandUsage(System.out);
+  }
+}
+

Propchange: incubator/sqoop/trunk/src/java/org/apache/sqoop/tool/JobTool.java
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Added: incubator/sqoop/trunk/src/java/org/apache/sqoop/tool/ListDatabasesTool.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/org/apache/sqoop/tool/ListDatabasesTool.java?rev=1195852&view=auto
==============================================================================
--- incubator/sqoop/trunk/src/java/org/apache/sqoop/tool/ListDatabasesTool.java (added)
+++ incubator/sqoop/trunk/src/java/org/apache/sqoop/tool/ListDatabasesTool.java Tue Nov  1 07:10:13 2011
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.tool;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import com.cloudera.sqoop.SqoopOptions;
+import com.cloudera.sqoop.SqoopOptions.InvalidOptionsException;
+import com.cloudera.sqoop.cli.ToolOptions;
+
+/**
+ * Tool that lists available databases on a server.
+ */
+public class ListDatabasesTool extends com.cloudera.sqoop.tool.BaseSqoopTool {
+
+  public static final Log LOG = LogFactory.getLog(
+      ListDatabasesTool.class.getName());
+
+  public ListDatabasesTool() {
+    super("list-databases");
+  }
+
+  @Override
+  /** {@inheritDoc} */
+  public int run(SqoopOptions options) {
+    if (!init(options)) {
+      return 1;
+    }
+
+    try {
+      String [] databases = manager.listDatabases();
+      if (null == databases) {
+        System.err.println("Could not retrieve database list from server");
+        LOG.error("manager.listDatabases() returned null");
+        return 1;
+      } else {
+        for (String db : databases) {
+          System.out.println(db);
+        }
+      }
+    } finally {
+      destroy(options);
+    }
+
+    return 0;
+  }
+
+  @Override
+  /** Configure the command-line arguments we expect to receive */
+  public void configureOptions(ToolOptions toolOptions) {
+    toolOptions.addUniqueOptions(getCommonOptions());
+  }
+
+  @Override
+  /** {@inheritDoc} */
+  public void applyOptions(CommandLine in, SqoopOptions out)
+      throws InvalidOptionsException {
+    applyCommonOptions(in, out);
+  }
+
+  @Override
+  /** {@inheritDoc} */
+  public void validateOptions(SqoopOptions options)
+      throws InvalidOptionsException {
+
+    if (hasUnrecognizedArgs(extraArguments)) {
+      throw new InvalidOptionsException(HELP_STR);
+    }
+    validateCommonOptions(options);
+  }
+}
+

Propchange: incubator/sqoop/trunk/src/java/org/apache/sqoop/tool/ListDatabasesTool.java
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Added: incubator/sqoop/trunk/src/java/org/apache/sqoop/tool/ListTablesTool.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/org/apache/sqoop/tool/ListTablesTool.java?rev=1195852&view=auto
==============================================================================
--- incubator/sqoop/trunk/src/java/org/apache/sqoop/tool/ListTablesTool.java (added)
+++ incubator/sqoop/trunk/src/java/org/apache/sqoop/tool/ListTablesTool.java Tue Nov  1 07:10:13 2011
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.tool;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import com.cloudera.sqoop.SqoopOptions;
+import com.cloudera.sqoop.SqoopOptions.InvalidOptionsException;
+import com.cloudera.sqoop.cli.ToolOptions;
+
+/**
+ * Tool that lists available tables in a database.
+ */
+public class ListTablesTool extends com.cloudera.sqoop.tool.BaseSqoopTool {
+
+  public static final Log LOG = LogFactory.getLog(
+      ListTablesTool.class.getName());
+
+  public ListTablesTool() {
+    super("list-tables");
+  }
+
+  @Override
+  /** {@inheritDoc} */
+  public int run(SqoopOptions options) {
+    if (!init(options)) {
+      return 1;
+    }
+
+    try {
+      String [] tables = manager.listTables();
+      if (null == tables) {
+        System.err.println("Could not retrieve tables list from server");
+        LOG.error("manager.listTables() returned null");
+        return 1;
+      } else {
+        for (String tbl : tables) {
+          System.out.println(tbl);
+        }
+      }
+    } finally {
+      destroy(options);
+    }
+
+    return 0;
+  }
+
+  @Override
+  /** Configure the command-line arguments we expect to receive */
+  public void configureOptions(ToolOptions toolOptions) {
+    toolOptions.addUniqueOptions(getCommonOptions());
+  }
+
+  @Override
+  /** {@inheritDoc} */
+  public void applyOptions(CommandLine in, SqoopOptions out)
+      throws InvalidOptionsException {
+    applyCommonOptions(in, out);
+  }
+
+  @Override
+  /** {@inheritDoc} */
+  public void validateOptions(SqoopOptions options)
+      throws InvalidOptionsException {
+    if (hasUnrecognizedArgs(extraArguments)) {
+      throw new InvalidOptionsException(HELP_STR);
+    }
+
+    validateCommonOptions(options);
+  }
+}
+

Propchange: incubator/sqoop/trunk/src/java/org/apache/sqoop/tool/ListTablesTool.java
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Added: incubator/sqoop/trunk/src/java/org/apache/sqoop/tool/MergeTool.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/org/apache/sqoop/tool/MergeTool.java?rev=1195852&view=auto
==============================================================================
--- incubator/sqoop/trunk/src/java/org/apache/sqoop/tool/MergeTool.java (added)
+++ incubator/sqoop/trunk/src/java/org/apache/sqoop/tool/MergeTool.java Tue Nov  1 07:10:13 2011
@@ -0,0 +1,235 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.tool;
+
+import java.io.IOException;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.log4j.Category;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+import com.cloudera.sqoop.Sqoop;
+import com.cloudera.sqoop.SqoopOptions;
+import com.cloudera.sqoop.SqoopOptions.InvalidOptionsException;
+import com.cloudera.sqoop.cli.RelatedOptions;
+import com.cloudera.sqoop.cli.ToolOptions;
+import com.cloudera.sqoop.mapreduce.MergeJob;
+
+/**
+ * Tool that merges a more recent dataset on top of an older one.
+ */
+public class MergeTool extends com.cloudera.sqoop.tool.BaseSqoopTool {
+
+  public static final Log LOG = LogFactory.getLog(MergeTool.class.getName());
+
+  public MergeTool() {
+    this("merge");
+  }
+
+  public MergeTool(String toolName) {
+    super(toolName);
+  }
+
+  @Override
+  /** {@inheritDoc} */
+  public int run(SqoopOptions options) {
+    try {
+      // Configure and execute a MapReduce job to merge these datasets.
+      MergeJob mergeJob = new MergeJob(options);
+      if (!mergeJob.runMergeJob()) {
+        LOG.error("MapReduce job failed!");
+        return 1;
+      }
+    } catch (IOException ioe) {
+      LOG.error("Encountered IOException running import job: "
+          + StringUtils.stringifyException(ioe));
+      if (System.getProperty(Sqoop.SQOOP_RETHROW_PROPERTY) != null) {
+        throw new RuntimeException(ioe);
+      } else {
+        return 1;
+      }
+    }
+
+    return 0;
+  }
+
+  /**
+   * Construct the set of options that control imports, either of one
+   * table or a batch of tables.
+   * @return the RelatedOptions that can be used to parse the import
+   * arguments.
+   */
+  protected RelatedOptions getMergeOptions() {
+    // Imports
+    RelatedOptions mergeOpts = new RelatedOptions("Merge arguments");
+
+    mergeOpts.addOption(OptionBuilder.withArgName("file")
+        .hasArg().withDescription("Load class from specified jar file")
+        .withLongOpt(JAR_FILE_NAME_ARG)
+        .create());
+
+    mergeOpts.addOption(OptionBuilder.withArgName("name")
+        .hasArg().withDescription("Specify record class name to load")
+        .withLongOpt(CLASS_NAME_ARG)
+        .create());
+
+    mergeOpts.addOption(OptionBuilder.withArgName("path")
+        .hasArg().withDescription("Path to the more recent data set")
+        .withLongOpt(NEW_DATASET_ARG)
+        .create());
+
+    mergeOpts.addOption(OptionBuilder.withArgName("path")
+        .hasArg().withDescription("Path to the older data set")
+        .withLongOpt(OLD_DATASET_ARG)
+        .create());
+
+    mergeOpts.addOption(OptionBuilder.withArgName("path")
+        .hasArg().withDescription("Destination path for merged results")
+        .withLongOpt(TARGET_DIR_ARG)
+        .create());
+
+    mergeOpts.addOption(OptionBuilder.withArgName("column")
+        .hasArg().withDescription("Key column to use to join results")
+        .withLongOpt(MERGE_KEY_ARG)
+        .create());
+
+    // Since the "common" options aren't used in the merge tool,
+    // add these settings here.
+    mergeOpts.addOption(OptionBuilder
+        .withDescription("Print more information while working")
+        .withLongOpt(VERBOSE_ARG)
+        .create());
+    mergeOpts.addOption(OptionBuilder
+        .withDescription("Print usage instructions")
+        .withLongOpt(HELP_ARG)
+        .create());
+
+    return mergeOpts;
+  }
+
+
+  @Override
+  /** Configure the command-line arguments we expect to receive */
+  public void configureOptions(ToolOptions toolOptions) {
+    toolOptions.addUniqueOptions(getMergeOptions());
+  }
+
+
+  @Override
+  /** {@inheritDoc} */
+  public void applyOptions(CommandLine in, SqoopOptions out)
+      throws InvalidOptionsException {
+
+    if (in.hasOption(VERBOSE_ARG)) {
+      // Immediately switch into DEBUG logging.
+      Category sqoopLogger = Logger.getLogger(
+          Sqoop.class.getName()).getParent();
+      sqoopLogger.setLevel(Level.DEBUG);
+      LOG.debug("Enabled debug logging.");
+    }
+
+    if (in.hasOption(HELP_ARG)) {
+      ToolOptions toolOpts = new ToolOptions();
+      configureOptions(toolOpts);
+      printHelp(toolOpts);
+      throw new InvalidOptionsException("");
+    }
+
+    if (in.hasOption(JAR_FILE_NAME_ARG)) {
+      out.setExistingJarName(in.getOptionValue(JAR_FILE_NAME_ARG));
+    }
+
+    if (in.hasOption(CLASS_NAME_ARG)) {
+      out.setClassName(in.getOptionValue(CLASS_NAME_ARG));
+    }
+
+    if (in.hasOption(NEW_DATASET_ARG)) {
+      out.setMergeNewPath(in.getOptionValue(NEW_DATASET_ARG));
+    }
+
+    if (in.hasOption(OLD_DATASET_ARG)) {
+      out.setMergeOldPath(in.getOptionValue(OLD_DATASET_ARG));
+    }
+
+    if (in.hasOption(TARGET_DIR_ARG)) {
+      out.setTargetDir(in.getOptionValue(TARGET_DIR_ARG));
+    }
+
+    if (in.hasOption(MERGE_KEY_ARG)) {
+      out.setMergeKeyCol(in.getOptionValue(MERGE_KEY_ARG));
+    }
+  }
+
+  /**
+   * Validate merge-specific arguments.
+   * @param options the configured SqoopOptions to check
+   */
+  protected void validateMergeOptions(SqoopOptions options)
+      throws InvalidOptionsException {
+
+    if (options.getMergeNewPath() == null) {
+      throw new InvalidOptionsException("Must set the new dataset path with --"
+          + NEW_DATASET_ARG + "." + HELP_STR);
+    }
+
+    if (options.getMergeOldPath() == null) {
+      throw new InvalidOptionsException("Must set the old dataset path with --"
+          + OLD_DATASET_ARG + "." + HELP_STR);
+    }
+
+    if (options.getMergeKeyCol() == null) {
+      throw new InvalidOptionsException("Must set the merge key column with --"
+          + MERGE_KEY_ARG + "." + HELP_STR);
+    }
+
+    if (options.getTargetDir() == null) {
+      throw new InvalidOptionsException("Must set the target directory with --"
+          + TARGET_DIR_ARG + "." + HELP_STR);
+    }
+
+    if (options.getClassName() == null) {
+      throw new InvalidOptionsException("Must set the SqoopRecord class "
+          + "implementation to use with --" + CLASS_NAME_ARG + "."
+          + HELP_STR);
+    }
+  }
+
+  @Override
+  /** {@inheritDoc} */
+  public void validateOptions(SqoopOptions options)
+      throws InvalidOptionsException {
+
+    // If extraArguments is full, check for '--' followed by args for
+    // mysqldump or other commands we rely on.
+    options.setExtraArgs(getSubcommandArgs(extraArguments));
+    int dashPos = getDashPosition(extraArguments);
+
+    if (hasUnrecognizedArgs(extraArguments, 0, dashPos)) {
+      throw new InvalidOptionsException(HELP_STR);
+    }
+
+    validateMergeOptions(options);
+  }
+}
+

Propchange: incubator/sqoop/trunk/src/java/org/apache/sqoop/tool/MergeTool.java
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Added: incubator/sqoop/trunk/src/java/org/apache/sqoop/tool/MetastoreTool.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/org/apache/sqoop/tool/MetastoreTool.java?rev=1195852&view=auto
==============================================================================
--- incubator/sqoop/trunk/src/java/org/apache/sqoop/tool/MetastoreTool.java (added)
+++ incubator/sqoop/trunk/src/java/org/apache/sqoop/tool/MetastoreTool.java Tue Nov  1 07:10:13 2011
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.tool;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import com.cloudera.sqoop.SqoopOptions;
+import com.cloudera.sqoop.SqoopOptions.InvalidOptionsException;
+import com.cloudera.sqoop.cli.RelatedOptions;
+import com.cloudera.sqoop.cli.ToolOptions;
+import com.cloudera.sqoop.metastore.hsqldb.HsqldbMetaStore;
+
+/**
+ * Tool that runs a standalone Sqoop metastore.
+ */
+public class MetastoreTool extends com.cloudera.sqoop.tool.BaseSqoopTool {
+
+  public static final Log LOG = LogFactory.getLog(
+      MetastoreTool.class.getName());
+
+  private HsqldbMetaStore metastore;
+
+  // If set to true, shut an existing metastore down.
+  private boolean shutdown = false;
+
+  public MetastoreTool() {
+    super("metastore");
+  }
+
+  @Override
+  /** {@inheritDoc} */
+  public int run(SqoopOptions options) {
+    metastore = new HsqldbMetaStore(options.getConf());
+    if (shutdown) {
+      LOG.info("Shutting down metastore...");
+      metastore.shutdown();
+    } else {
+      metastore.start();
+      metastore.waitForServer();
+      LOG.info("Server thread has quit.");
+    }
+    return 0;
+  }
+
+  @Override
+  /** Configure the command-line arguments we expect to receive */
+  public void configureOptions(ToolOptions toolOptions) {
+    RelatedOptions opts = new RelatedOptions("metastore arguments");
+    opts.addOption(OptionBuilder
+        .withDescription("Cleanly shut down a running metastore")
+        .withLongOpt(METASTORE_SHUTDOWN_ARG)
+        .create());
+
+    toolOptions.addUniqueOptions(opts);
+  }
+
+  @Override
+  /** {@inheritDoc} */
+  public void applyOptions(CommandLine in, SqoopOptions out)
+      throws InvalidOptionsException {
+    if (in.hasOption(METASTORE_SHUTDOWN_ARG)) {
+      this.shutdown = true;
+    }
+  }
+
+  @Override
+  /** {@inheritDoc} */
+  public void validateOptions(SqoopOptions options)
+      throws InvalidOptionsException {
+  }
+}
+

Propchange: incubator/sqoop/trunk/src/java/org/apache/sqoop/tool/MetastoreTool.java
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Added: incubator/sqoop/trunk/src/java/org/apache/sqoop/tool/SqoopTool.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/org/apache/sqoop/tool/SqoopTool.java?rev=1195852&view=auto
==============================================================================
--- incubator/sqoop/trunk/src/java/org/apache/sqoop/tool/SqoopTool.java (added)
+++ incubator/sqoop/trunk/src/java/org/apache/sqoop/tool/SqoopTool.java Tue Nov  1 07:10:13 2011
@@ -0,0 +1,507 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.tool;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.TreeMap;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.sqoop.util.ClassLoaderStack;
+
+import com.cloudera.sqoop.SqoopOptions;
+import com.cloudera.sqoop.SqoopOptions.InvalidOptionsException;
+import com.cloudera.sqoop.cli.SqoopParser;
+import com.cloudera.sqoop.cli.ToolOptions;
+import com.cloudera.sqoop.config.ConfigurationHelper;
+import com.cloudera.sqoop.tool.ToolDesc;
+
+/**
+ * Base class for Sqoop subprograms (e.g., SqoopImport, SqoopExport, etc.)
+ * Allows subprograms to configure the arguments they accept and
+ * provides an entry-point to the subprogram.
+ */
+public abstract class SqoopTool {
+
+  public static final Log LOG = LogFactory.getLog(SqoopTool.class.getName());
+
+  /**
+   * Configuration key that specifies the set of ToolPlugin instances to load
+   * before determining which SqoopTool instance to load.
+   */
+  public static final String TOOL_PLUGINS_KEY = "sqoop.tool.plugins";
+
+  private static final Map<String, Class<? extends SqoopTool>> TOOLS;
+  private static final Map<String, String> DESCRIPTIONS;
+
+  static {
+    // All SqoopTool instances should be registered here so that
+    // they can be found internally.
+    TOOLS = new TreeMap<String, Class<? extends SqoopTool>>();
+    DESCRIPTIONS = new TreeMap<String, String>();
+
+    registerTool("codegen", CodeGenTool.class,
+        "Generate code to interact with database records");
+    registerTool("create-hive-table", CreateHiveTableTool.class,
+        "Import a table definition into Hive");
+    registerTool("eval", EvalSqlTool.class,
+        "Evaluate a SQL statement and display the results");
+    registerTool("export", ExportTool.class,
+        "Export an HDFS directory to a database table");
+    registerTool("import", ImportTool.class,
+        "Import a table from a database to HDFS");
+    registerTool("import-all-tables", ImportAllTablesTool.class,
+        "Import tables from a database to HDFS");
+    registerTool("help", HelpTool.class, "List available commands");
+    registerTool("list-databases", ListDatabasesTool.class,
+        "List available databases on a server");
+    registerTool("list-tables", ListTablesTool.class,
+        "List available tables in a database");
+    registerTool("merge", MergeTool.class,
+        "Merge results of incremental imports");
+    registerTool("metastore", MetastoreTool.class,
+        "Run a standalone Sqoop metastore");
+    registerTool("job", JobTool.class,
+        "Work with saved jobs");
+    registerTool("version", VersionTool.class,
+        "Display version information");
+  }
+
+  /**
+   * Add a tool to the available set of SqoopTool instances.
+   * @param toolName the name the user access the tool through.
+   * @param cls the class providing the tool.
+   * @param description a user-friendly description of the tool's function.
+   */
+  private static void registerTool(String toolName,
+      Class<? extends SqoopTool> cls, String description) {
+    Class<? extends SqoopTool> existing = TOOLS.get(toolName);
+    if (null != existing) {
+      // Already have a tool with this name. Refuse to start.
+      throw new RuntimeException("A plugin is attempting to register a tool "
+          + "with name " + toolName + ", but this tool already exists ("
+          + existing.getName() + ")");
+    }
+
+    TOOLS.put(toolName, cls);
+    DESCRIPTIONS.put(toolName, description);
+  }
+
+  /**
+   * Add tool to available set of SqoopTool instances using the ToolDesc
+   * struct as the sole argument.
+   */
+  private static void registerTool(ToolDesc toolDescription) {
+    registerTool(toolDescription.getName(), toolDescription.getToolClass(),
+        toolDescription.getDesc());
+  }
+
+  /**
+   * Load plugins referenced in sqoop-site.xml or other config (e.g., tools.d/),
+   * to allow external tool definitions.
+   *
+   * @return the Configuration used to load the plugins.
+   */
+  public static Configuration loadPlugins(Configuration conf) {
+    conf = loadPluginsFromConfDir(conf);
+    List<ToolPlugin> plugins = conf.getInstances(TOOL_PLUGINS_KEY,
+        ToolPlugin.class);
+    for (ToolPlugin plugin : plugins) {
+      LOG.debug("Loading plugin: " + plugin.getClass().getName());
+      List<ToolDesc> descriptions = plugin.getTools();
+      for (ToolDesc desc : descriptions) {
+        LOG.debug("  Adding tool: " + desc.getName()
+            + " -> " + desc.getToolClass().getName());
+        registerTool(desc);
+      }
+    }
+
+    return conf;
+  }
+
+  /**
+   * If $SQOOP_CONF_DIR/tools.d/ exists and sqoop.tool.plugins is not set,
+   * then we look through the files in that directory; they should contain
+   * lines of the form 'plugin.class.name[=/path/to/containing.jar]'.
+   *
+   * <p>Put all plugin.class.names into the Configuration, and load any
+   * specified jars into the ClassLoader.
+   * </p>
+   *
+   * @param conf the current configuration to populate with class names.
+   * @return conf again, after possibly populating sqoop.tool.plugins.
+   */
+  private static Configuration loadPluginsFromConfDir(Configuration conf) {
+    if (conf.get(TOOL_PLUGINS_KEY) != null) {
+      LOG.debug(TOOL_PLUGINS_KEY + " is set; ignoring tools.d");
+      return conf;
+    }
+
+    String confDirName = System.getenv("SQOOP_CONF_DIR");
+    if (null == confDirName) {
+      LOG.warn("$SQOOP_CONF_DIR has not been set in the environment. "
+          + "Cannot check for additional configuration.");
+      return conf;
+    }
+
+    File confDir = new File(confDirName);
+    File toolsDir = new File(confDir, "tools.d");
+
+    if (toolsDir.exists() && toolsDir.isDirectory()) {
+      // We have a tools.d subdirectory. Get the file list, sort it,
+      // and process them in order.
+      String [] fileNames = toolsDir.list();
+      Arrays.sort(fileNames);
+
+      for (String fileName : fileNames) {
+        File f = new File(toolsDir, fileName);
+        if (f.isFile()) {
+          loadPluginsFromFile(conf, f);
+        }
+      }
+    }
+
+    // Set the classloader in this configuration so that it will use
+    // the jars we just loaded in.
+    conf.setClassLoader(Thread.currentThread().getContextClassLoader());
+    return conf;
+  }
+
+  /**
+   * Read the specified file and extract any ToolPlugin implementation
+   * names from there.
+   * @param conf the configuration to populate.
+   * @param f the file containing the configuration data to add.
+   */
+  private static void loadPluginsFromFile(Configuration conf, File f) {
+    Reader r = null;
+    try {
+      // The file format is actually Java properties-file syntax.
+      r = new InputStreamReader(new FileInputStream(f));
+      Properties props = new Properties();
+      props.load(r);
+
+      for (Map.Entry<Object, Object> entry : props.entrySet()) {
+        // Each key is a ToolPlugin class name.
+        // Each value, if set, is the jar that contains it.
+        String plugin = entry.getKey().toString();
+        addPlugin(conf, plugin);
+
+        String jarName = entry.getValue().toString();
+        if (jarName.length() > 0) {
+          ClassLoaderStack.addJarFile(jarName, plugin);
+          LOG.debug("Added plugin " + plugin + " in jar " + jarName
+              + " specified by " + f);
+        } else if (LOG.isDebugEnabled()) {
+          LOG.debug("Added plugin " + plugin + " specified by " + f);
+        }
+      }
+    } catch (IOException ioe) {
+      LOG.error("Error loading ToolPlugin information from file "
+          + f + ": " + StringUtils.stringifyException(ioe));
+    } finally {
+      if (null != r) {
+        try {
+          r.close();
+        } catch (IOException ioe) {
+          LOG.warn("Error closing file " + f + ": " + ioe);
+        }
+      }
+    }
+  }
+
+  /**
+   * Add the specified plugin class name to the configuration string
+   * listing plugin classes.
+   */
+  private static void addPlugin(Configuration conf, String pluginName) {
+    String existingPlugins = conf.get(TOOL_PLUGINS_KEY);
+    String newPlugins = null;
+    if (null == existingPlugins || existingPlugins.length() == 0) {
+      newPlugins = pluginName;
+    } else {
+      newPlugins = existingPlugins + "," + pluginName;
+    }
+
+    conf.set(TOOL_PLUGINS_KEY, newPlugins);
+  }
+
+  /**
+   * @return the list of available tools.
+   */
+  public static Set<String> getToolNames() {
+    return TOOLS.keySet();
+  }
+
+  /**
+   * @return the SqoopTool instance with the provided name, or null
+   * if no such tool exists.
+   */
+  public static SqoopTool getTool(String toolName) {
+    Class<? extends SqoopTool> cls = TOOLS.get(toolName);
+    try {
+      if (null != cls) {
+        SqoopTool tool = cls.newInstance();
+        tool.setToolName(toolName);
+        return tool;
+      }
+    } catch (Exception e) {
+      LOG.error(StringUtils.stringifyException(e));
+      return null;
+    }
+
+    return null;
+  }
+
+  /**
+   * @return the user-friendly description for a tool, or null if the tool
+   * cannot be found.
+   */
+  public static String getToolDescription(String toolName) {
+    return DESCRIPTIONS.get(toolName);
+  }
+
+  /** The name of the current tool. */
+  private String toolName;
+
+  /** Arguments that remained unparsed after parseArguments. */
+  protected String [] extraArguments;
+
+  public SqoopTool() {
+    this.toolName = "<" + this.getClass().getName() + ">";
+  }
+
+  public SqoopTool(String name) {
+    this.toolName = name;
+  }
+
+  public String getToolName() {
+    return this.toolName;
+  }
+
+  protected void setToolName(String name) {
+    this.toolName = name;
+  }
+
+  /**
+   * Main body of code to run the tool.
+   * @param options the SqoopOptions configured via
+   * configureOptions()/applyOptions().
+   * @return an integer return code for external programs to consume. 0
+   * represents success; nonzero means failure.
+   */
+  public abstract int run(SqoopOptions options);
+
+  /**
+   * Configure the command-line arguments we expect to receive.
+   * @param opts a ToolOptions that should be populated with sets of
+   * RelatedOptions for the tool.
+   */
+  public void configureOptions(ToolOptions opts) {
+    // Default implementation does nothing.
+  }
+
+  /**
+   * Print the help message for this tool.
+   * @param opts the configured tool options
+   */
+  public void printHelp(ToolOptions opts) {
+    System.out.println("usage: sqoop " + getToolName()
+        + " [GENERIC-ARGS] [TOOL-ARGS]");
+    System.out.println("");
+
+    opts.printHelp();
+
+    System.out.println("");
+    System.out.println("Generic Hadoop command-line arguments:");
+    System.out.println("(must preceed any tool-specific arguments)");
+    ToolRunner.printGenericCommandUsage(System.out);
+  }
+
+  /** Generate the SqoopOptions containing actual argument values from
+   * the extracted CommandLine arguments.
+   * @param in the CLI CommandLine that contain the user's set Options.
+   * @param out the SqoopOptions with all fields applied.
+   * @throws InvalidOptionsException if there's a problem.
+   */
+  public void applyOptions(CommandLine in, SqoopOptions out)
+      throws InvalidOptionsException {
+    // Default implementation does nothing.
+  }
+
+  /**
+   * Validates options and ensures that any required options are
+   * present and that any mutually-exclusive options are not selected.
+   * @throws InvalidOptionsException if there's a problem.
+   */
+  public void validateOptions(SqoopOptions options)
+      throws InvalidOptionsException {
+    // Default implementation does nothing.
+  }
+
+  /**
+   * Configures a SqoopOptions according to the specified arguments.
+   * Reads a set of arguments and uses them to configure a SqoopOptions
+   * and its embedded configuration (i.e., through GenericOptionsParser.)
+   * Stores any unparsed arguments in the extraArguments field.
+   *
+   * @param args the arguments to parse.
+   * @param conf if non-null, set as the configuration for the returned
+   * SqoopOptions.
+   * @param in a (perhaps partially-configured) SqoopOptions. If null,
+   * then a new SqoopOptions will be used. If this has a null configuration
+   * and conf is null, then a new Configuration will be inserted in this.
+   * @param useGenericOptions if true, will also parse generic Hadoop
+   * options into the Configuration.
+   * @return a SqoopOptions that is fully configured by a given tool.
+   */
+  public SqoopOptions parseArguments(String [] args,
+      Configuration conf, SqoopOptions in, boolean useGenericOptions)
+      throws ParseException, SqoopOptions.InvalidOptionsException {
+    SqoopOptions out = in;
+
+    if (null == out) {
+      out = new SqoopOptions();
+    }
+
+    if (null != conf) {
+      // User specified a configuration; use it and override any conf
+      // that may have been in the SqoopOptions.
+      out.setConf(conf);
+    } else if (null == out.getConf()) {
+      // User did not specify a configuration, but neither did the
+      // SqoopOptions. Fabricate a new one.
+      out.setConf(new Configuration());
+    }
+
+    // This tool is the "active" tool; bind it in the SqoopOptions.
+    //TODO(jarcec): Remove the cast when SqoopOptions will be moved
+    //              to apache package
+    out.setActiveSqoopTool((com.cloudera.sqoop.tool.SqoopTool)this);
+
+    String [] toolArgs = args; // args after generic parser is done.
+    if (useGenericOptions) {
+      try {
+        toolArgs = ConfigurationHelper.parseGenericOptions(
+            out.getConf(), args);
+      } catch (IOException ioe) {
+        ParseException pe = new ParseException(
+            "Could not parse generic arguments");
+        pe.initCause(ioe);
+        throw pe;
+      }
+    }
+
+    // Parse tool-specific arguments.
+    ToolOptions toolOptions = new ToolOptions();
+    configureOptions(toolOptions);
+    CommandLineParser parser = new SqoopParser();
+    CommandLine cmdLine = parser.parse(toolOptions.merge(), toolArgs, true);
+    applyOptions(cmdLine, out);
+    this.extraArguments = cmdLine.getArgs();
+    return out;
+  }
+
+  /**
+   * Append 'extra' to extraArguments.
+   */
+  public void appendArgs(String [] extra) {
+    int existingLen =
+        (this.extraArguments == null) ? 0 : this.extraArguments.length;
+    int newLen = (extra == null) ? 0 : extra.length;
+    String [] newExtra = new String[existingLen + newLen];
+
+    if (null != this.extraArguments) {
+      System.arraycopy(this.extraArguments, 0, newExtra, 0, existingLen);
+    }
+
+    if (null != extra) {
+      System.arraycopy(extra, 0, newExtra, existingLen, newLen);
+    }
+
+    this.extraArguments = newExtra;
+  }
+
+  /**
+   * Allow a tool to specify a set of dependency jar filenames. This is used
+   * to allow tools to bundle arbitrary dependency jars necessary for a
+   * MapReduce job executed by Sqoop. The jar containing the SqoopTool
+   * instance itself will already be handled by Sqoop.
+   *
+   * <p>Called by JobBase.cacheJars().</p>
+   *
+   * <p>
+   * This does not load the jars into the current VM; they are assumed to be
+   * already on the classpath if they are needed on the client side (or
+   * otherwise classloaded by the tool itself). This is purely to specify jars
+   * necessary to be added to the distributed cache. The tool itself can
+   * classload these jars by running loadDependencyJars().
+   * </p>
+   *
+   * <p>See also: c.c.s.util.Jars.getJarPathForClass()</p>
+   */
+  public List<String> getDependencyJars() {
+    // Default behavior: no additional dependencies.
+    return Collections.emptyList();
+  }
+
+  /**
+   * Loads dependency jars specified by getDependencyJars() into the current
+   * classloader stack. May optionally be called by a [third-party] tool
+   * before doing work, to ensure that all of its dependencies get classloaded
+   * properly. Note that dependencies will not be available until after the
+   * tool is already constructed.
+   */
+  protected void loadDependencyJars(SqoopOptions options) throws IOException {
+    List<String> deps = getDependencyJars();
+    if (null == deps) {
+      return;
+    }
+
+    for (String depFilename : deps) {
+      LOG.debug("Loading dependency: " + depFilename);
+      ClassLoaderStack.addJarFile(depFilename, null);
+    }
+
+    options.getConf().setClassLoader(
+        Thread.currentThread().getContextClassLoader());
+  }
+
+  @Override
+  public String toString() {
+    return getToolName();
+  }
+}
+

Propchange: incubator/sqoop/trunk/src/java/org/apache/sqoop/tool/SqoopTool.java
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Added: incubator/sqoop/trunk/src/java/org/apache/sqoop/tool/ToolDesc.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/org/apache/sqoop/tool/ToolDesc.java?rev=1195852&view=auto
==============================================================================
--- incubator/sqoop/trunk/src/java/org/apache/sqoop/tool/ToolDesc.java (added)
+++ incubator/sqoop/trunk/src/java/org/apache/sqoop/tool/ToolDesc.java Tue Nov  1 07:10:13 2011
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.tool;
+
+/**
+ * Describes a SqoopTool.
+ * This class should be final
+ */
+public class ToolDesc {
+  private final String toolName;
+  private final Class<? extends SqoopTool> toolClass;
+  private final String description;
+
+
+  /**
+   * Main c'tor; sets all fields that describe a SqoopTool.
+   */
+  public ToolDesc(String name, Class<? extends SqoopTool> cls, String desc) {
+    this.toolName = name;
+    this.toolClass = cls;
+    this.description = desc;
+  }
+
+  /**
+   * @return the name used to invoke the tool (e.g., 'sqoop &lt;foo&gt;')
+   */
+  public String getName() {
+    return toolName;
+  }
+
+  /**
+   * @return a human-readable description of what the tool does.
+   */
+  public String getDesc() {
+    return description;
+  }
+
+  /**
+   * @return the class that implements SqoopTool.
+   */
+  public Class<? extends SqoopTool> getToolClass() {
+    return toolClass;
+  }
+
+}

Propchange: incubator/sqoop/trunk/src/java/org/apache/sqoop/tool/ToolDesc.java
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Added: incubator/sqoop/trunk/src/java/org/apache/sqoop/tool/ToolPlugin.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/org/apache/sqoop/tool/ToolPlugin.java?rev=1195852&view=auto
==============================================================================
--- incubator/sqoop/trunk/src/java/org/apache/sqoop/tool/ToolPlugin.java (added)
+++ incubator/sqoop/trunk/src/java/org/apache/sqoop/tool/ToolPlugin.java Tue Nov  1 07:10:13 2011
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.tool;
+
+import java.util.List;
+
+import com.cloudera.sqoop.tool.ToolDesc;
+
+/**
+ * Abstract base class that defines the ToolPlugin API; additional SqoopTool
+ * implementations may be registered with the system via ToolPlugin classes.
+ */
+public abstract class ToolPlugin {
+  /**
+   * Describes the tools made available by this plugin.
+   * @return a list of ToolDesc objects containing the tool name, class,
+   * and description.
+   */
+  public abstract List<ToolDesc> getTools();
+}

Propchange: incubator/sqoop/trunk/src/java/org/apache/sqoop/tool/ToolPlugin.java
------------------------------------------------------------------------------
    svn:mime-type = text/plain



Mime
View raw message