Author: sandy
Date: Fri Jan 17 17:53:27 2014
New Revision: 1559203
URL: http://svn.apache.org/r1559203
Log:
MAPREDUCE-5650. Job fails when hprof mapreduce.task.profile.map/reduce.params is specified
(Gera Shegalov via Sandy Ryza)
Added:
hadoop/common/branches/branch-2.3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithProfiler.java
- copied unchanged from r1559201, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithProfiler.java
Modified:
hadoop/common/branches/branch-2.3/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-2.3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java
hadoop/common/branches/branch-2.3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
Modified: hadoop/common/branches/branch-2.3/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.3/hadoop-mapreduce-project/CHANGES.txt?rev=1559203&r1=1559202&r2=1559203&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.3/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2.3/hadoop-mapreduce-project/CHANGES.txt Fri Jan 17 17:53:27
2014
@@ -56,6 +56,9 @@ Release 2.3.0 - UNRELEASED
MAPREDUCE-5674. Missing start and finish time in mapred.JobStatus.
(Chuan Liu via cnauroth)
+ MAPREDUCE-5650. Job fails when hprof mapreduce.task.profile.map/reduce.params
+ is specified (Gera Shegalov via Sandy Ryza)
+
Release 2.2.0 - 2013-10-13
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-2.3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java?rev=1559203&r1=1559202&r2=1559203&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java
(original)
+++ hadoop/common/branches/branch-2.3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java
Fri Jan 17 17:53:27 2014
@@ -210,19 +210,11 @@ public class MapReduceChildJVM {
if (conf.getProfileEnabled()) {
if (conf.getProfileTaskRange(task.isMapTask()
).isIncluded(task.getPartition())) {
- vargs.add(
- String.format(
- conf.getProfileParams(),
- getTaskLogFile(TaskLog.LogName.PROFILE)
- )
- );
- if (task.isMapTask()) {
- vargs.add(conf.get(MRJobConfig.TASK_MAP_PROFILE_PARAMS, ""));
- }
- else {
- vargs.add(conf.get(MRJobConfig.TASK_REDUCE_PROFILE_PARAMS, ""));
- }
-
+ final String profileParams = conf.get(task.isMapTask()
+ ? MRJobConfig.TASK_MAP_PROFILE_PARAMS
+ : MRJobConfig.TASK_REDUCE_PROFILE_PARAMS, conf.getProfileParams());
+ vargs.add(String.format(profileParams,
+ getTaskLogFile(TaskLog.LogName.PROFILE)));
}
}
Modified: hadoop/common/branches/branch-2.3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml?rev=1559203&r1=1559202&r2=1559203&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
(original)
+++ hadoop/common/branches/branch-2.3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
Fri Jan 17 17:53:27 2014
@@ -945,6 +945,31 @@
</property>
<property>
+ <name>mapreduce.task.profile.params</name>
+ <value></value>
+ <description>JVM profiler parameters used to profile map and reduce task
+ attempts. This string may contain a single format specifier %s that will
+ be replaced by the path to profile.out in the task attempt log directory.
+ To specify different profiling options for map tasks and reduce tasks,
+ more specific parameters mapreduce.task.profile.map.params and
+ mapreduce.task.profile.reduce.params should be used.</description>
+ </property>
+
+ <property>
+ <name>mapreduce.task.profile.map.params</name>
+ <value>${mapreduce.task.profile.params}</value>
+ <description>Map-task-specific JVM profiler parameters. See
+ mapreduce.task.profile.params</description>
+ </property>
+
+ <property>
+ <name>mapreduce.task.profile.reduce.params</name>
+ <value>${mapreduce.task.profile.params}</value>
+ <description>Reduce-task-specific JVM profiler parameters. See
+ mapreduce.task.profile.params</description>
+ </property>
+
+ <property>
<name>mapreduce.task.skip.start.attempts</name>
<value>2</value>
<description> The number of Task attempts AFTER which skip mode
|