Repository: sqoop
Updated Branches:
refs/heads/sqoop2 66a86353c -> 0c20d1f06
SQOOP-2802: Sqoop2: Logging level should be debug for integration tests
(Abraham Fine via Jarek Jarcec Cecho)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/0c20d1f0
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/0c20d1f0
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/0c20d1f0
Branch: refs/heads/sqoop2
Commit: 0c20d1f069ea77d3128f1bd05a890d2b6cb2c200
Parents: 66a8635
Author: Jarek Jarcec Cecho <jarcec@apache.org>
Authored: Fri Jan 29 12:52:02 2016 -0800
Committer: Jarek Jarcec Cecho <jarcec@apache.org>
Committed: Fri Jan 29 12:52:02 2016 -0800
----------------------------------------------------------------------
dist/src/main/conf/sqoop.properties | 3 +++
.../main/java/org/apache/sqoop/job/MRJobConstants.java | 2 ++
.../org/apache/sqoop/job/mr/MRConfigurationUtils.java | 3 +++
.../src/main/resources/META-INF/log4j.properties | 2 +-
.../org/apache/sqoop/submission/mapreduce/Constants.java | 3 +++
.../submission/mapreduce/MapreduceSubmissionEngine.java | 10 ++++++++++
.../apache/sqoop/test/minicluster/SqoopMiniCluster.java | 1 +
7 files changed, 23 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/0c20d1f0/dist/src/main/conf/sqoop.properties
----------------------------------------------------------------------
diff --git a/dist/src/main/conf/sqoop.properties b/dist/src/main/conf/sqoop.properties
index 2895530..620146d 100755
--- a/dist/src/main/conf/sqoop.properties
+++ b/dist/src/main/conf/sqoop.properties
@@ -143,6 +143,9 @@ org.apache.sqoop.submission.engine=org.apache.sqoop.submission.mapreduce.Mapredu
# Hadoop configuration directory
org.apache.sqoop.submission.engine.mapreduce.configuration.directory=/etc/hadoop/conf/
+# Log level for Sqoop Mapper/Reducer
+org.apache.sqoop.submission.engine.mapreduce.configuration.loglevel=INFO
+
#
# Execution engine configuration
#
http://git-wip-us.apache.org/repos/asf/sqoop/blob/0c20d1f0/execution/mapreduce/src/main/java/org/apache/sqoop/job/MRJobConstants.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/MRJobConstants.java b/execution/mapreduce/src/main/java/org/apache/sqoop/job/MRJobConstants.java
index 8968198..6cbdac4 100644
--- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/MRJobConstants.java
+++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/MRJobConstants.java
@@ -72,6 +72,8 @@ public final class MRJobConstants extends Constants {
public static final String JOB_CONNECTOR_TO_NAME = PREFIX_JOB_CONFIG
+ "connector.to.name";
+ public static final String SYSPROP_MR_LOG_LEVEL = "sqoop.mr.loglevel";
+
// Hadoop specific constants
// We're using constants from Hadoop 1. Hadoop 2 has different names, but
// provides backward compatibility layer for those names as well.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/0c20d1f0/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/MRConfigurationUtils.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/MRConfigurationUtils.java
b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/MRConfigurationUtils.java
index 1e1b237..8410391 100644
--- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/MRConfigurationUtils.java
+++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/MRConfigurationUtils.java
@@ -268,6 +268,9 @@ public final class MRConfigurationUtils {
}
public static void configureLogging() {
+ if (System.getProperty(MRJobConstants.SYSPROP_MR_LOG_LEVEL) == null) {
+ System.setProperty(MRJobConstants.SYSPROP_MR_LOG_LEVEL, "INFO");
+ }
try (InputStream resourceAsStream =
SqoopMapper.class.getResourceAsStream("/META-INF/log4j.properties")) {
Properties props = new Properties();
http://git-wip-us.apache.org/repos/asf/sqoop/blob/0c20d1f0/execution/mapreduce/src/main/resources/META-INF/log4j.properties
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/main/resources/META-INF/log4j.properties b/execution/mapreduce/src/main/resources/META-INF/log4j.properties
index 2a57714..e42c791 100644
--- a/execution/mapreduce/src/main/resources/META-INF/log4j.properties
+++ b/execution/mapreduce/src/main/resources/META-INF/log4j.properties
@@ -14,7 +14,7 @@
# limitations under the License.
# Set root logger level to INFO and its only appender to A1.
-log4j.logger.org.apache.sqoop=INFO, A1
+log4j.logger.org.apache.sqoop=${sqoop.mr.loglevel}, A1
# A1 is set to be a ConsoleAppender.
log4j.appender.A1=org.apache.log4j.ConsoleAppender
http://git-wip-us.apache.org/repos/asf/sqoop/blob/0c20d1f0/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/Constants.java
----------------------------------------------------------------------
diff --git a/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/Constants.java
b/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/Constants.java
index 93b0725..213da61 100644
--- a/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/Constants.java
+++ b/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/Constants.java
@@ -27,6 +27,9 @@ public class Constants {
public static final String CONF_CONFIG_DIR =
PREFIX_MAPREDUCE + "configuration.directory";
+ public static final String CONF_SQOOP_MR_LOG_LEVEL =
+ PREFIX_MAPREDUCE + "configuration.loglevel";
+
public static final String SQOOP_JOB = "sqoop.job";
private Constants() {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/0c20d1f0/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java
----------------------------------------------------------------------
diff --git a/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java
b/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java
index c03bf39..0dc4ef0 100644
--- a/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java
+++ b/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.mapred.JobID;
import org.apache.hadoop.mapred.JobStatus;
import org.apache.hadoop.mapred.RunningJob;
import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.log4j.Logger;
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.common.MapContext;
@@ -107,6 +108,15 @@ public class MapreduceSubmissionEngine extends SubmissionEngine {
}
}
+ // Log level for sqoop mapper and reducer
+ String mrLogLevel = context.getString(prefix + Constants.CONF_SQOOP_MR_LOG_LEVEL);
+ if (globalConfiguration.get(MRJobConfig.MR_AM_COMMAND_OPTS) == null) {
+ globalConfiguration.set(MRJobConfig.MR_AM_COMMAND_OPTS, MRJobConfig.DEFAULT_MR_AM_COMMAND_OPTS
+ " -Dsqoop.mr.loglevel=" + mrLogLevel);
+ } else {
+ globalConfiguration.set(MRJobConfig.MR_AM_COMMAND_OPTS, globalConfiguration.get(MRJobConfig.MR_AM_COMMAND_OPTS)
+ " -Dsqoop.mr.loglevel=" + mrLogLevel);
+ }
+
+
// Save our own property inside the job to easily identify Sqoop jobs
globalConfiguration.setBoolean(Constants.SQOOP_JOB, true);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/0c20d1f0/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java b/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java
index 4ff97e7..c8da7af 100644
--- a/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java
+++ b/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java
@@ -193,6 +193,7 @@ public abstract class SqoopMiniCluster {
properties.put("org.apache.sqoop.submission.engine", "org.apache.sqoop.submission.mapreduce.MapreduceSubmissionEngine");
properties.put("org.apache.sqoop.submission.engine.mapreduce.configuration.directory",
getConfigurationPath());
+ properties.put("org.apache.sqoop.submission.engine.mapreduce.configuration.loglevel",
"DEBUG");
return properties;
}
|