sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a..@apache.org
Subject sqoop git commit: SQOOP-1696: Sqoop2: TestMapReduce.testOutputFormat is failing on Hadoop 1 profile
Date Sun, 09 Nov 2014 07:31:35 GMT
Repository: sqoop
Updated Branches:
  refs/heads/sqoop2 a389c2c7c -> 0e4f46b44


SQOOP-1696: Sqoop2: TestMapReduce.testOutputFormat is failing on Hadoop 1 profile

(Jarek Jarcec Cecho via Abraham Elmahrek)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/0e4f46b4
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/0e4f46b4
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/0e4f46b4

Branch: refs/heads/sqoop2
Commit: 0e4f46b44d66e9418e3e81614b688f7352f026b3
Parents: a389c2c
Author: Abraham Elmahrek <abraham@elmahrek.com>
Authored: Sat Nov 8 23:26:08 2014 -0800
Committer: Abraham Elmahrek <abraham@elmahrek.com>
Committed: Sat Nov 8 23:26:35 2014 -0800

----------------------------------------------------------------------
 .../java/org/apache/sqoop/job/JobUtils.java     | 41 +++++++++++++++++++-
 1 file changed, 40 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/0e4f46b4/execution/mapreduce/src/test/java/org/apache/sqoop/job/JobUtils.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/test/java/org/apache/sqoop/job/JobUtils.java b/execution/mapreduce/src/test/java/org/apache/sqoop/job/JobUtils.java
index 7ce8011..dafdeb4 100644
--- a/execution/mapreduce/src/test/java/org/apache/sqoop/job/JobUtils.java
+++ b/execution/mapreduce/src/test/java/org/apache/sqoop/job/JobUtils.java
@@ -23,10 +23,16 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.NullWritable;
 import org.apache.hadoop.mapreduce.InputFormat;
 import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
 import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.OutputCommitter;
 import org.apache.hadoop.mapreduce.OutputFormat;
 import org.apache.sqoop.job.io.SqoopWritable;
 import org.apache.sqoop.job.mr.SqoopSplit;
+import org.apache.sqoop.utils.ClassUtils;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 public class JobUtils {
 
@@ -44,7 +50,40 @@ public class JobUtils {
     job.setOutputKeyClass(SqoopWritable.class);
     job.setOutputValueClass(NullWritable.class);
 
-    return job.waitForCompletion(true);
+    boolean ret = job.waitForCompletion(true);
+
+    // Hadoop 1.0 (and 0.20) have nasty bug when job committer is not called in LocalJobRuner
+    if (isHadoop1()) {
+      callOutputCommitter(job, output);
+    }
+
+    return ret;
+  }
+
+  /**
+   * Call output format on given job manually.
+   */
+  private static void callOutputCommitter(Job job, Class<? extends OutputFormat<SqoopWritable,
NullWritable>> outputFormat) throws IOException, InterruptedException {
+    OutputCommitter committer = ((OutputFormat)ClassUtils.instantiate(outputFormat)).getOutputCommitter(null);
+
+    JobContext jobContext = mock(JobContext.class);
+    when(jobContext.getConfiguration()).thenReturn(job.getConfiguration());
+
+    committer.commitJob(jobContext);
+  }
+
+  /**
+   * Detect Hadoop 1.0 installation
+   *
+   * @return True if and only if this is Hadoop 1 and below
+   */
+  public static boolean isHadoop1() {
+    String version = org.apache.hadoop.util.VersionInfo.getVersion();
+    if (version.matches("\\b0\\.20\\..+\\b")
+      || version.matches("\\b1\\.\\d\\.\\d")) {
+      return true;
+    }
+    return false;
   }
 
   private JobUtils() {


Mime
View raw message