sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jar...@apache.org
Subject sqoop git commit: SQOOP-1826: NPE in ImportTool.lastModifiedMerge during postgres import
Date Tue, 06 Jan 2015 09:15:11 GMT
Repository: sqoop
Updated Branches:
  refs/heads/trunk 8b6eb33f1 -> c32010f44


SQOOP-1826: NPE in ImportTool.lastModifiedMerge during postgres import

(Ricky Nguyen via Jarek Jarcec Cecho)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/c32010f4
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/c32010f4
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/c32010f4

Branch: refs/heads/trunk
Commit: c32010f440f139b37840df6a3433f875898136d5
Parents: 8b6eb33
Author: Jarek Jarcec Cecho <jarcec@apache.org>
Authored: Tue Jan 6 10:14:38 2015 +0100
Committer: Jarek Jarcec Cecho <jarcec@apache.org>
Committed: Tue Jan 6 10:14:38 2015 +0100

----------------------------------------------------------------------
 .../sqoop/manager/DirectPostgresqlManager.java  |  2 ++
 .../sqoop/manager/PostgresqlImportTest.java     | 37 ++++++++++++++++++--
 2 files changed, 36 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/c32010f4/src/java/org/apache/sqoop/manager/DirectPostgresqlManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/manager/DirectPostgresqlManager.java b/src/java/org/apache/sqoop/manager/DirectPostgresqlManager.java
index 8d4a097..63b0704 100644
--- a/src/java/org/apache/sqoop/manager/DirectPostgresqlManager.java
+++ b/src/java/org/apache/sqoop/manager/DirectPostgresqlManager.java
@@ -348,6 +348,8 @@ public class DirectPostgresqlManager
   public void importTable(com.cloudera.sqoop.manager.ImportJobContext context)
     throws IOException, ImportException {
 
+    context.setConnManager(this);
+
     String tableName = context.getTableName();
     SqoopOptions options = context.getOptions();
 

http://git-wip-us.apache.org/repos/asf/sqoop/blob/c32010f4/src/test/com/cloudera/sqoop/manager/PostgresqlImportTest.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/manager/PostgresqlImportTest.java b/src/test/com/cloudera/sqoop/manager/PostgresqlImportTest.java
index 3b5731b..57dd338 100644
--- a/src/test/com/cloudera/sqoop/manager/PostgresqlImportTest.java
+++ b/src/test/com/cloudera/sqoop/manager/PostgresqlImportTest.java
@@ -235,15 +235,18 @@ public class PostgresqlImportTest extends ImportJobTestCase {
     return args.toArray(new String[0]);
   }
 
-  private void doImportAndVerify(boolean isDirect, String [] expectedResults,
+  private void doImportAndVerify(boolean isDirect, String[] expectedResults,
       String tableName, String... extraArgs) throws IOException {
 
     Path warehousePath = new Path(this.getWarehouseDir());
     Path tablePath = new Path(warehousePath, tableName);
-    Path filePath = new Path(tablePath, "part-m-00000");
+
+    // if importing with merge step, directory should exist and output should be from a reducer
+    boolean isMerge = Arrays.asList(extraArgs).contains("--merge-key");
+    Path filePath = new Path(tablePath, isMerge ? "part-r-00000" : "part-m-00000");
 
     File tableFile = new File(tablePath.toString());
-    if (tableFile.exists() && tableFile.isDirectory()) {
+    if (tableFile.exists() && tableFile.isDirectory() && !isMerge) {
       // remove the directory before running the import.
       FileListing.recursiveDeleteDir(tableFile);
     }
@@ -329,6 +332,34 @@ public class PostgresqlImportTest extends ImportJobTestCase {
     doImportAndVerify(false, expectedResults, TABLE_NAME, extraArgs);
   }
 
+  public void testDirectIncrementalImport() throws IOException {
+    String [] expectedResults = { };
+
+    String [] extraArgs = { "--incremental", "lastmodified",
+            "--check-column", "start_date",
+    };
+
+    doImportAndVerify(true, expectedResults, TABLE_NAME, extraArgs);
+  }
+
+  public void testDirectIncrementalImportMerge() throws IOException {
+    String [] expectedResults = { };
+
+    String [] extraArgs = { "--incremental", "lastmodified",
+            "--check-column", "start_date",
+    };
+
+    doImportAndVerify(true, expectedResults, TABLE_NAME, extraArgs);
+
+    extraArgs = new String[] { "--incremental", "lastmodified",
+            "--check-column", "start_date",
+            "--merge-key", "id",
+            "--last-value", "2009-04-20"
+    };
+
+    doImportAndVerify(true, expectedResults, TABLE_NAME, extraArgs);
+  }
+
  @Test
   public void testDifferentSchemaImport() throws IOException {
     String [] expectedResults = {


Mime
View raw message