Author: todd
Date: Thu Aug 4 17:09:14 2011
New Revision: 1153931
URL: http://svn.apache.org/viewvc?rev=1153931&view=rev
Log:
Merge trunk into HA branch after mavenization of hadoop-common
(no conflicts, straight SVN merge)
Added:
hadoop/common/branches/HDFS-1623/mapreduce/src/test/mapred/org/apache/hadoop/mapred/TestTaskLauncherThreaded.java
- copied unchanged from r1153927, hadoop/common/trunk/mapreduce/src/test/mapred/org/apache/hadoop/mapred/TestTaskLauncherThreaded.java
Modified:
hadoop/common/branches/HDFS-1623/mapreduce/ (props changed)
hadoop/common/branches/HDFS-1623/mapreduce/.gitignore (props changed)
hadoop/common/branches/HDFS-1623/mapreduce/CHANGES.txt (contents, props changed)
hadoop/common/branches/HDFS-1623/mapreduce/conf/ (props changed)
hadoop/common/branches/HDFS-1623/mapreduce/conf/capacity-scheduler.xml.template (props changed)
hadoop/common/branches/HDFS-1623/mapreduce/ivy.xml
hadoop/common/branches/HDFS-1623/mapreduce/src/c++/ (props changed)
hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/ (props changed)
hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/block_forensics/ (props changed)
hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/build-contrib.xml (props changed)
hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/build.xml (props changed)
hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/capacity-scheduler/ (props changed)
hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/capacity-scheduler/ivy.xml
hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/data_join/ (props changed)
hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/data_join/ivy.xml
hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/dynamic-scheduler/ (props changed)
hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/dynamic-scheduler/ivy.xml
hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/eclipse-plugin/ (props changed)
hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/eclipse-plugin/ivy.xml
hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/fairscheduler/ (props changed)
hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/fairscheduler/ivy.xml
hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/gridmix/ivy.xml
hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/index/ (props changed)
hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/index/ivy.xml
hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/mumak/ivy.xml
hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/raid/ivy.xml
hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/streaming/ (props changed)
hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/streaming/ivy.xml
hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/vaidya/ (props changed)
hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/vaidya/ivy.xml
hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/vertica/ivy.xml
hadoop/common/branches/HDFS-1623/mapreduce/src/examples/ (props changed)
hadoop/common/branches/HDFS-1623/mapreduce/src/java/ (props changed)
hadoop/common/branches/HDFS-1623/mapreduce/src/java/mapred-default.xml
hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/CompletedJobStatusStore.java
hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/MapTask.java
hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/ReduceTask.java
hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/Task.java
hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/TaskLog.java
hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/TaskTracker.java
hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapreduce/MRJobConfig.java
hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java
hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapreduce/lib/output/MultipleOutputs.java
hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapreduce/lib/partition/TotalOrderPartitioner.java
hadoop/common/branches/HDFS-1623/mapreduce/src/test/mapred/ (props changed)
hadoop/common/branches/HDFS-1623/mapreduce/src/test/mapred/org/apache/hadoop/fs/ (props changed)
hadoop/common/branches/HDFS-1623/mapreduce/src/test/mapred/org/apache/hadoop/hdfs/ (props changed)
hadoop/common/branches/HDFS-1623/mapreduce/src/test/mapred/org/apache/hadoop/io/FileBench.java (props changed)
hadoop/common/branches/HDFS-1623/mapreduce/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java (props changed)
hadoop/common/branches/HDFS-1623/mapreduce/src/test/mapred/org/apache/hadoop/ipc/ (props changed)
hadoop/common/branches/HDFS-1623/mapreduce/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java (props changed)
hadoop/common/branches/HDFS-1623/mapreduce/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java (props changed)
hadoop/common/branches/HDFS-1623/mapreduce/src/webapps/job/ (props changed)
Propchange: hadoop/common/branches/HDFS-1623/mapreduce/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug 4 17:09:14 2011
@@ -1 +1,2 @@
+/hadoop/common/trunk/mapreduce:1152502-1153927
/hadoop/core/branches/branch-0.19/mapred:713112
Propchange: hadoop/common/branches/HDFS-1623/mapreduce/.gitignore
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug 4 17:09:14 2011
@@ -1,2 +1,3 @@
+/hadoop/common/trunk/mapreduce/.gitignore:1152502-1153927
/hadoop/core/branches/branch-0.19/mapred/.gitignore:713112
/hadoop/core/trunk/.gitignore:784664-785643
Modified: hadoop/common/branches/HDFS-1623/mapreduce/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/CHANGES.txt?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/CHANGES.txt Thu Aug 4 17:09:14 2011
@@ -40,6 +40,9 @@ Trunk (unreleased changes)
IMPROVEMENTS
+ MAPREDUCE-2187. Reporter sends progress during sort/merge. (Anupam Seth via
+ acmurthy)
+
MAPREDUCE-2365. Add counters to track bytes (read,written) via
File(Input,Output)Format. (Siddharth Seth via acmurthy)
@@ -206,12 +209,18 @@ Trunk (unreleased changes)
MAPREDUCE-2602. Allow setting of end-of-record delimiter for
TextInputFormat for the old API. (Ahmed Radwan via todd)
+ MAPREDUCE-2705. Permits parallel multiple task launches.
+ (Thomas Graves via ddas)
+
OPTIMIZATIONS
MAPREDUCE-2026. Make JobTracker.getJobCounters() and
JobInProgress.getCounters() aquire locks in a shorter time period.
(Joydeep Sen Sarma via schen)
+ MAPREDUCE-2740. MultipleOutputs in new API creates needless
+ TaskAttemptContexts. (todd)
+
BUG FIXES
MAPREDUCE-2603. Disable High-Ram emulation in system tests.
@@ -362,6 +371,16 @@ Trunk (unreleased changes)
MAPREDUCE-2463. Job history files are not moved to done folder when job
history location is hdfs. (Devaraj K via szetszwo)
+ MAPREDUCE-2243. Close streams propely in a finally-block to avoid leakage
+ in CompletedJobStatusStore, TaskLog, EventWriter and TotalOrderPartitioner.
+ (Devaraj K via szetszwo)
+
+ MAPREDUCE-2741. Make ant build system work with hadoop-common JAR
+ generated by Maven. (Alejandro Abdelnur via tomwhite)
+
+ MAPREDUCE-2760. mapreduce.jobtracker.split.metainfo.maxsize typoed
+ in mapred-default.xml. (todd via eli)
+
Release 0.22.0 - Unreleased
INCOMPATIBLE CHANGES
Propchange: hadoop/common/branches/HDFS-1623/mapreduce/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug 4 17:09:14 2011
@@ -1,2 +1,3 @@
+/hadoop/common/trunk/mapreduce/CHANGES.txt:1152502-1153927
/hadoop/core/branches/branch-0.19/mapred/CHANGES.txt:713112
/hadoop/mapreduce/branches/HDFS-641/CHANGES.txt:817878-835964
Propchange: hadoop/common/branches/HDFS-1623/mapreduce/conf/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug 4 17:09:14 2011
@@ -1,2 +1,3 @@
+/hadoop/common/trunk/mapreduce/conf:1152502-1153927
/hadoop/core/branches/branch-0.19/mapred/conf:713112
/hadoop/core/trunk/conf:784664-785643
Propchange: hadoop/common/branches/HDFS-1623/mapreduce/conf/capacity-scheduler.xml.template
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug 4 17:09:14 2011
@@ -1,2 +1,3 @@
+/hadoop/common/trunk/mapreduce/conf/capacity-scheduler.xml.template:1152502-1153927
/hadoop/core/branches/branch-0.19/mapred/conf/capacity-scheduler.xml.template:713112
/hadoop/core/trunk/conf/capacity-scheduler.xml.template:776175-785643
Modified: hadoop/common/branches/HDFS-1623/mapreduce/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/ivy.xml?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/ivy.xml (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/ivy.xml Thu Aug 4 17:09:14 2011
@@ -57,6 +57,7 @@
<artifact conf="master"/>
</publications>
<dependencies>
+ <dependency org="org.apache.hadoop" name="hadoop-annotations" rev="${hadoop-common.version}" conf="compile->default"/>
<dependency org="org.apache.hadoop" name="hadoop-common"
rev="${hadoop-common.version}" conf="compile->default"/>
<dependency org="org.apache.hadoop" name="hadoop-common-test"
@@ -105,6 +106,8 @@
<exclude module="jetty"/>
<exclude module="slf4j-simple"/>
</dependency>
+ <dependency org="junit" name="junit" rev="${junit.version}"
+ conf="test->default"/>
<dependency org="org.mockito" name="mockito-all" rev="${mockito-all.version}"
conf="test->default"/>
<dependency org="org.vafer" name="jdeb" rev="${jdeb.version}" conf="package->master"/>
@@ -116,6 +119,12 @@
<dependency org="org.aspectj" name="aspectjtools" rev="${aspectj.version}"
conf="compile->default"/>
+ <!-- Exclusions for transitive dependencies pulled in by log4j -->
+ <exclude org="com.sun.jdmk"/>
+ <exclude org="com.sun.jmx"/>
+ <exclude org="javax.jms"/>
+ <exclude org="javax.mail"/>
+
</dependencies>
</ivy-module>
Propchange: hadoop/common/branches/HDFS-1623/mapreduce/src/c++/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug 4 17:09:14 2011
@@ -1,2 +1,3 @@
+/hadoop/common/trunk/mapreduce/src/c++:1152502-1153927
/hadoop/core/branches/branch-0.19/mapred/src/c++:713112
/hadoop/core/trunk/src/c++:776175-784663
Propchange: hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug 4 17:09:14 2011
@@ -1,2 +1,3 @@
+/hadoop/common/trunk/mapreduce/src/contrib:1152502-1153927
/hadoop/core/branches/branch-0.19/mapred/src/contrib:713112
/hadoop/core/trunk/src/contrib:784664-785643
Propchange: hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/block_forensics/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug 4 17:09:14 2011
@@ -1 +1,2 @@
+/hadoop/common/trunk/mapreduce/src/contrib/block_forensics:1152502-1153927
/hadoop/core/branches/branch-0.19/hdfs/src/contrib/block_forensics:713112
Propchange: hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/build-contrib.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug 4 17:09:14 2011
@@ -1,2 +1,3 @@
+/hadoop/common/trunk/mapreduce/src/contrib/build-contrib.xml:1152502-1153927
/hadoop/core/branches/branch-0.19/mapred/src/contrib/build-contrib.xml:713112
/hadoop/core/trunk/src/contrib/build-contrib.xml:776175-786373
Propchange: hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/build.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug 4 17:09:14 2011
@@ -1,2 +1,3 @@
+/hadoop/common/trunk/mapreduce/src/contrib/build.xml:1152502-1153927
/hadoop/core/branches/branch-0.19/mapred/src/contrib/build.xml:713112
/hadoop/core/trunk/src/contrib/build.xml:776175-786373
Propchange: hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/capacity-scheduler/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug 4 17:09:14 2011
@@ -1,2 +1,3 @@
+/hadoop/common/trunk/mapreduce/src/contrib/capacity-scheduler:1152502-1153927
/hadoop/core/branches/branch-0.19/mapred/src/contrib/capacity-scheduler:713112
/hadoop/core/trunk/src/contrib/capacity-scheduler:776175-786373
Modified: hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/capacity-scheduler/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/capacity-scheduler/ivy.xml?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/capacity-scheduler/ivy.xml (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/capacity-scheduler/ivy.xml Thu Aug 4 17:09:14 2011
@@ -41,6 +41,7 @@
<artifact conf="master"/>
</publications>
<dependencies>
+ <dependency org="org.apache.hadoop" name="hadoop-annotations" rev="${hadoop-common.version}" conf="common->default"/>
<dependency org="org.apache.hadoop" name="hadoop-common"
rev="${hadoop-common.version}" conf="common->default"/>
<dependency org="org.apache.hadoop" name="hadoop-common-test"
@@ -75,5 +76,12 @@
<exclude module="jetty"/>
<exclude module="slf4j-simple"/>
</dependency>
+
+ <!-- Exclusions for transitive dependencies pulled in by log4j -->
+ <exclude org="com.sun.jdmk"/>
+ <exclude org="com.sun.jmx"/>
+ <exclude org="javax.jms"/>
+ <exclude org="javax.mail"/>
+
</dependencies>
</ivy-module>
Propchange: hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/data_join/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug 4 17:09:14 2011
@@ -1,2 +1,3 @@
+/hadoop/common/trunk/mapreduce/src/contrib/data_join:1152502-1153927
/hadoop/core/branches/branch-0.19/mapred/src/contrib/data_join:713112
/hadoop/core/trunk/src/contrib/data_join:776175-786373
Modified: hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/data_join/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/data_join/ivy.xml?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/data_join/ivy.xml (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/data_join/ivy.xml Thu Aug 4 17:09:14 2011
@@ -41,11 +41,20 @@
<artifact conf="master"/>
</publications>
<dependencies>
+ <dependency org="org.apache.hadoop" name="hadoop-annotations" rev="${hadoop-common.version}" conf="common->default"/>
<dependency org="org.apache.hadoop" name="hadoop-common" rev="${hadoop-common.version}" conf="common->default"/>
<dependency org="org.apache.hadoop" name="hadoop-common-test" rev="${hadoop-common.version}" conf="common->default"/>
<dependency org="org.apache.hadoop" name="hadoop-hdfs" rev="${hadoop-hdfs.version}" conf="common->default"/>
<dependency org="org.apache.hadoop" name="hadoop-hdfs-test" rev="${hadoop-hdfs.version}" conf="common->default"/>
<dependency org="commons-logging" name="commons-logging" rev="${commons-logging.version}" conf="common->default"/>
<dependency org="log4j" name="log4j" rev="${log4j.version}" conf="common->master"/>
+ <dependency org="junit" name="junit" rev="${junit.version}" conf="common->default"/>
+
+ <!-- Exclusions for transitive dependencies pulled in by log4j -->
+ <exclude org="com.sun.jdmk"/>
+ <exclude org="com.sun.jmx"/>
+ <exclude org="javax.jms"/>
+ <exclude org="javax.mail"/>
+
</dependencies>
</ivy-module>
Propchange: hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/dynamic-scheduler/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug 4 17:09:14 2011
@@ -1,2 +1,3 @@
+/hadoop/common/trunk/mapreduce/src/contrib/dynamic-scheduler:1152502-1153927
/hadoop/core/branches/branch-0.19/src/contrib/dynamic-scheduler:713112
/hadoop/core/trunk/src/contrib/dynamic-scheduler:784975-786373
Modified: hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/dynamic-scheduler/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/dynamic-scheduler/ivy.xml?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/dynamic-scheduler/ivy.xml (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/dynamic-scheduler/ivy.xml Thu Aug 4 17:09:14 2011
@@ -37,6 +37,7 @@
</publications>
<dependencies>
+ <dependency org="org.apache.hadoop" name="hadoop-annotations" rev="${hadoop-common.version}" conf="common->default"/>
<dependency org="org.apache.hadoop" name="hadoop-common" rev="${hadoop-common.version}" conf="common->default"/>
<dependency org="org.mortbay.jetty" name="jetty" rev="${jetty.version}" conf="common->master"/>
<dependency org="org.mortbay.jetty" name="jetty-util" rev="${jetty-util.version}" conf="common->master"/>
@@ -50,5 +51,12 @@
<dependency org="org.mortbay.jetty" name="servlet-api-2.5" rev="${servlet-api-2.5.version}" conf="common->master"/>
<dependency org="junit" name="junit" rev="${junit.version}" conf="common->default"/>
<dependency org="org.slf4j" name="slf4j-api" rev="${slf4j-api.version}" conf="common->master"/>
+
+ <!-- Exclusions for transitive dependencies pulled in by log4j -->
+ <exclude org="com.sun.jdmk"/>
+ <exclude org="com.sun.jmx"/>
+ <exclude org="javax.jms"/>
+ <exclude org="javax.mail"/>
+
</dependencies>
</ivy-module>
Propchange: hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/eclipse-plugin/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug 4 17:09:14 2011
@@ -1,2 +1,3 @@
+/hadoop/common/trunk/mapreduce/src/contrib/eclipse-plugin:1152502-1153927
/hadoop/core/branches/branch-0.19/core/src/contrib/eclipse-plugin:713112
/hadoop/core/trunk/src/contrib/eclipse-plugin:776175-784663
Modified: hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/eclipse-plugin/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/eclipse-plugin/ivy.xml?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/eclipse-plugin/ivy.xml (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/eclipse-plugin/ivy.xml Thu Aug 4 17:09:14 2011
@@ -41,9 +41,17 @@
<artifact conf="master"/>
</publications>
<dependencies>
+ <dependency org="org.apache.hadoop" name="hadoop-annotations" rev="${hadoop-common.version}" conf="common->default"/>
<dependency org="org.apache.hadoop" name="hadoop-common" rev="${hadoop-common.version}" conf="common->default"/>
<dependency org="org.apache.hadoop" name="hadoop-hdfs" rev="${hadoop-hdfs.version}" conf="common->default"/>
<dependency org="commons-logging" name="commons-logging" rev="${commons-logging.version}" conf="common->default"/>
<dependency org="log4j" name="log4j" rev="${log4j.version}" conf="common->master"/>
+
+ <!-- Exclusions for transitive dependencies pulled in by log4j -->
+ <exclude org="com.sun.jdmk"/>
+ <exclude org="com.sun.jmx"/>
+ <exclude org="javax.jms"/>
+ <exclude org="javax.mail"/>
+
</dependencies>
</ivy-module>
Propchange: hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/fairscheduler/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug 4 17:09:14 2011
@@ -1,2 +1,3 @@
+/hadoop/common/trunk/mapreduce/src/contrib/fairscheduler:1152502-1153927
/hadoop/core/branches/branch-0.19/mapred/src/contrib/fairscheduler:713112
/hadoop/core/trunk/src/contrib/fairscheduler:776175-786373
Modified: hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/fairscheduler/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/fairscheduler/ivy.xml?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/fairscheduler/ivy.xml (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/fairscheduler/ivy.xml Thu Aug 4 17:09:14 2011
@@ -40,6 +40,7 @@
<artifact conf="master"/>
</publications>
<dependencies>
+ <dependency org="org.apache.hadoop" name="hadoop-annotations" rev="${hadoop-common.version}" conf="common->default"/>
<dependency org="org.apache.hadoop" name="hadoop-common"
rev="${hadoop-common.version}" conf="common->default"/>
<dependency org="org.apache.hadoop" name="hadoop-common-test"
@@ -100,5 +101,12 @@
name="jsp-2.1"
rev="${jetty.version}"
conf="common->master"/>
+
+ <!-- Exclusions for transitive dependencies pulled in by log4j -->
+ <exclude org="com.sun.jdmk"/>
+ <exclude org="com.sun.jmx"/>
+ <exclude org="javax.jms"/>
+ <exclude org="javax.mail"/>
+
</dependencies>
</ivy-module>
Modified: hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/gridmix/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/gridmix/ivy.xml?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/gridmix/ivy.xml (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/gridmix/ivy.xml Thu Aug 4 17:09:14 2011
@@ -36,6 +36,10 @@
<artifact conf="master"/>
</publications>
<dependencies>
+ <dependency org="org.apache.hadoop"
+ name="hadoop-annotations"
+ rev="${hadoop-common.version}"
+ conf="common->default"/>
<dependency org="org.apache.hadoop"
name="hadoop-common"
rev="${hadoop-common.version}"
@@ -122,5 +126,12 @@
name="paranamer"
rev="${paranamer.version}"
conf="common->default"/>
+
+ <!-- Exclusions for transitive dependencies pulled in by log4j -->
+ <exclude org="com.sun.jdmk"/>
+ <exclude org="com.sun.jmx"/>
+ <exclude org="javax.jms"/>
+ <exclude org="javax.mail"/>
+
</dependencies>
</ivy-module>
Propchange: hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/index/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug 4 17:09:14 2011
@@ -1,2 +1,3 @@
+/hadoop/common/trunk/mapreduce/src/contrib/index:1152502-1153927
/hadoop/core/branches/branch-0.19/mapred/src/contrib/index:713112
/hadoop/core/trunk/src/contrib/index:776175-786373
Modified: hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/index/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/index/ivy.xml?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/index/ivy.xml (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/index/ivy.xml Thu Aug 4 17:09:14 2011
@@ -41,6 +41,7 @@
<artifact conf="master"/>
</publications>
<dependencies>
+ <dependency org="org.apache.hadoop" name="hadoop-annotations" rev="${hadoop-common.version}" conf="common->default"/>
<dependency org="org.apache.hadoop" name="hadoop-common" rev="${hadoop-common.version}" conf="common->default"/>
<dependency org="org.apache.hadoop" name="hadoop-common-test" rev="${hadoop-common.version}" conf="common->default"/>
<dependency org="org.apache.hadoop" name="hadoop-hdfs" rev="${hadoop-hdfs.version}" conf="common->default"/>
@@ -48,5 +49,13 @@
<dependency org="commons-logging" name="commons-logging" rev="${commons-logging.version}" conf="common->default"/>
<dependency org="log4j" name="log4j" rev="${log4j.version}" conf="common->master"/>
<dependency org="org.apache.lucene" name="lucene-core" rev="${lucene-core.version}" conf="common->default"/>
+ <dependency org="junit" name="junit" rev="${junit.version}" conf="common->default"/>
+
+ <!-- Exclusions for transitive dependencies pulled in by log4j -->
+ <exclude org="com.sun.jdmk"/>
+ <exclude org="com.sun.jmx"/>
+ <exclude org="javax.jms"/>
+ <exclude org="javax.mail"/>
+
</dependencies>
</ivy-module>
Modified: hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/mumak/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/mumak/ivy.xml?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/mumak/ivy.xml (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/mumak/ivy.xml Thu Aug 4 17:09:14 2011
@@ -39,6 +39,10 @@
<artifact conf="master"/>
</publications>
<dependencies>
+ <dependency org="org.apache.hadoop"
+ name="hadoop-annotations"
+ rev="${hadoop-common.version}"
+ conf="common->default"/>
<dependency org="org.apache.hadoop" name="hadoop-common"
rev="${hadoop-common.version}" conf="common->default"/>
<dependency org="org.apache.hadoop" name="hadoop-common-test"
@@ -122,5 +126,12 @@
<exclude module="jetty"/>
<exclude module="slf4j-simple"/>
</dependency>
+
+ <!-- Exclusions for transitive dependencies pulled in by log4j -->
+ <exclude org="com.sun.jdmk"/>
+ <exclude org="com.sun.jmx"/>
+ <exclude org="javax.jms"/>
+ <exclude org="javax.mail"/>
+
</dependencies>
</ivy-module>
Modified: hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/raid/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/raid/ivy.xml?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/raid/ivy.xml (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/raid/ivy.xml Thu Aug 4 17:09:14 2011
@@ -36,6 +36,7 @@
<artifact conf="master"/>
</publications>
<dependencies>
+ <dependency org="org.apache.hadoop" name="hadoop-annotations" rev="${hadoop-common.version}" conf="common->default"/>
<dependency org="org.apache.hadoop"
name="hadoop-common"
rev="${hadoop-common.version}"
@@ -122,5 +123,12 @@
name="paranamer"
rev="${paranamer.version}"
conf="common->default"/>
+
+ <!-- Exclusions for transitive dependencies pulled in by log4j -->
+ <exclude org="com.sun.jdmk"/>
+ <exclude org="com.sun.jmx"/>
+ <exclude org="javax.jms"/>
+ <exclude org="javax.mail"/>
+
</dependencies>
</ivy-module>
Propchange: hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/streaming/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug 4 17:09:14 2011
@@ -1,2 +1,3 @@
+/hadoop/common/trunk/mapreduce/src/contrib/streaming:1152502-1153927
/hadoop/core/branches/branch-0.19/mapred/src/contrib/streaming:713112
/hadoop/core/trunk/src/contrib/streaming:776175-786373
Modified: hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/streaming/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/streaming/ivy.xml?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/streaming/ivy.xml (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/streaming/ivy.xml Thu Aug 4 17:09:14 2011
@@ -40,6 +40,7 @@
<artifact conf="master"/>
</publications>
<dependencies>
+ <dependency org="org.apache.hadoop" name="hadoop-annotations" rev="${hadoop-common.version}" conf="common->default"/>
<dependency org="org.apache.hadoop" name="hadoop-common"
rev="${hadoop-common.version}" conf="common->default"/>
<dependency org="org.apache.hadoop" name="hadoop-common-test"
@@ -76,5 +77,12 @@
</dependency>
<dependency org="org.slf4j" name="slf4j-api"
rev="${slf4j-api.version}" conf="common->master"/>
+
+ <!-- Exclusions for transitive dependencies pulled in by log4j -->
+ <exclude org="com.sun.jdmk"/>
+ <exclude org="com.sun.jmx"/>
+ <exclude org="javax.jms"/>
+ <exclude org="javax.mail"/>
+
</dependencies>
</ivy-module>
Propchange: hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/vaidya/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug 4 17:09:14 2011
@@ -1,2 +1,3 @@
+/hadoop/common/trunk/mapreduce/src/contrib/vaidya:1152502-1153927
/hadoop/core/branches/branch-0.19/mapred/src/contrib/vaidya:713112
/hadoop/core/trunk/src/contrib/vaidya:776175-786373
Modified: hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/vaidya/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/vaidya/ivy.xml?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/vaidya/ivy.xml (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/vaidya/ivy.xml Thu Aug 4 17:09:14 2011
@@ -41,8 +41,16 @@
<artifact conf="master"/>
</publications>
<dependencies>
+ <dependency org="org.apache.hadoop" name="hadoop-annotations" rev="${hadoop-common.version}" conf="common->default"/>
<dependency org="org.apache.hadoop" name="hadoop-common" rev="${hadoop-common.version}" conf="common->default"/>
<dependency org="log4j" name="log4j" rev="${log4j.version}" conf="common->master"/>
<dependency org="commons-logging" name="commons-logging" rev="${commons-logging.version}" conf="common->default"/>
+
+ <!-- Exclusions for transitive dependencies pulled in by log4j -->
+ <exclude org="com.sun.jdmk"/>
+ <exclude org="com.sun.jmx"/>
+ <exclude org="javax.jms"/>
+ <exclude org="javax.mail"/>
+
</dependencies>
</ivy-module>
Modified: hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/vertica/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/vertica/ivy.xml?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/vertica/ivy.xml (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/src/contrib/vertica/ivy.xml Thu Aug 4 17:09:14 2011
@@ -40,6 +40,7 @@
<artifact conf="master"/>
</publications>
<dependencies>
+ <dependency org="org.apache.hadoop" name="hadoop-annotations" rev="${hadoop-common.version}" conf="common->default"/>
<dependency org="org.apache.hadoop"
name="hadoop-common"
rev="${hadoop-common.version}"
@@ -64,5 +65,12 @@
name="log4j"
rev="${log4j.version}"
conf="common->master"/>
+
+ <!-- Exclusions for transitive dependencies pulled in by log4j -->
+ <exclude org="com.sun.jdmk"/>
+ <exclude org="com.sun.jmx"/>
+ <exclude org="javax.jms"/>
+ <exclude org="javax.mail"/>
+
</dependencies>
</ivy-module>
Propchange: hadoop/common/branches/HDFS-1623/mapreduce/src/examples/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug 4 17:09:14 2011
@@ -1,2 +1,3 @@
+/hadoop/common/trunk/mapreduce/src/examples:1152502-1153927
/hadoop/core/branches/branch-0.19/mapred/src/examples:713112
/hadoop/core/trunk/src/examples:776175-784663
Propchange: hadoop/common/branches/HDFS-1623/mapreduce/src/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug 4 17:09:14 2011
@@ -1,2 +1,3 @@
+/hadoop/common/trunk/mapreduce/src/java:1152502-1153927
/hadoop/core/branches/branch-0.19/mapred/src/java:713112
/hadoop/core/trunk/src/mapred:776175-785643
Modified: hadoop/common/branches/HDFS-1623/mapreduce/src/java/mapred-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/src/java/mapred-default.xml?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/src/java/mapred-default.xml (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/src/java/mapred-default.xml Thu Aug 4 17:09:14 2011
@@ -256,7 +256,7 @@
<property>
- <name>mapreduce.job.split.metainfo.maxsize</name>
+ <name>mapreduce.jobtracker.split.metainfo.maxsize</name>
<value>10000000</value>
<description>The maximum permissible size of the split metainfo file.
The JobTracker won't attempt to read split metainfo files bigger than
@@ -1042,6 +1042,14 @@
<!-- End of TaskTracker DistributedCache configuration -->
<property>
+ <name>mapreduce.task.combine.progress.records</name>
+ <value>10000</value>
+ <description> The number of records to process during combine output collection
+ before sending a progress notification to the TaskTracker.
+ </description>
+</property>
+
+<property>
<name>mapreduce.task.merge.progress.records</name>
<value>10000</value>
<description> The number of records to process during merge before
Modified: hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/CompletedJobStatusStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/CompletedJobStatusStore.java?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/CompletedJobStatusStore.java (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/CompletedJobStatusStore.java Thu Aug 4 17:09:14 2011
@@ -32,6 +32,7 @@ import org.apache.hadoop.security.UserGr
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.DiskChecker.DiskErrorException;
/**
@@ -172,8 +173,9 @@ class CompletedJobStatusStore implements
if (active && retainTime > 0) {
JobID jobId = job.getStatus().getJobID();
Path jobStatusFile = getInfoFilePath(jobId);
+ FSDataOutputStream dataOut = null;
try {
- FSDataOutputStream dataOut = fs.create(jobStatusFile);
+ dataOut = fs.create(jobStatusFile);
job.getStatus().write(dataOut);
@@ -189,6 +191,7 @@ class CompletedJobStatusStore implements
}
dataOut.close();
+ dataOut = null; // set dataOut to null explicitly so that close in finally will not be executed again.
} catch (IOException ex) {
LOG.warn("Could not store [" + jobId + "] job info : " +
ex.getMessage(), ex);
@@ -198,6 +201,8 @@ class CompletedJobStatusStore implements
catch (IOException ex1) {
//ignore
}
+ } finally {
+ IOUtils.cleanup(LOG, dataOut);
}
}
}
Modified: hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/MapTask.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/MapTask.java?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/MapTask.java (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/MapTask.java Thu Aug 4 17:09:14 2011
@@ -946,7 +946,7 @@ class MapTask extends Task {
if (combinerRunner != null) {
final Counters.Counter combineOutputCounter =
reporter.getCounter(TaskCounter.COMBINE_OUTPUT_RECORDS);
- combineCollector= new CombineOutputCollector<K,V>(combineOutputCounter);
+ combineCollector= new CombineOutputCollector<K,V>(combineOutputCounter, reporter, conf);
} else {
combineCollector = null;
}
Modified: hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/ReduceTask.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/ReduceTask.java?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/ReduceTask.java (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/ReduceTask.java Thu Aug 4 17:09:14 2011
@@ -352,7 +352,7 @@ public class ReduceTask extends Task {
Class combinerClass = conf.getCombinerClass();
CombineOutputCollector combineCollector =
(null != combinerClass) ?
- new CombineOutputCollector(reduceCombineOutputCounter) : null;
+ new CombineOutputCollector(reduceCombineOutputCounter, reporter, conf) : null;
Shuffle shuffle =
new Shuffle(getTaskID(), job, FileSystem.getLocal(job), umbilical,
Modified: hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/Task.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/Task.java?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/Task.java (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/Task.java Thu Aug 4 17:09:14 2011
@@ -58,6 +58,7 @@ import org.apache.hadoop.mapreduce.Outpu
import org.apache.hadoop.mapreduce.TaskCounter;
import org.apache.hadoop.mapreduce.JobStatus;
import org.apache.hadoop.mapreduce.MRConfig;
+import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer;
import org.apache.hadoop.mapreduce.task.ReduceContextImpl;
import org.apache.hadoop.mapreduce.util.ResourceCalculatorPlugin;
@@ -79,6 +80,7 @@ abstract public class Task implements Wr
LogFactory.getLog(Task.class);
public static String MERGED_OUTPUT_PREFIX = ".merged";
+ public static final long DEFAULT_COMBINE_RECORDS_BEFORE_PROGRESS = 10000;
/**
* Counters to measure the usage of the different file systems.
@@ -1176,16 +1178,26 @@ abstract public class Task implements Wr
implements OutputCollector<K, V> {
private Writer<K, V> writer;
private Counters.Counter outCounter;
- public CombineOutputCollector(Counters.Counter outCounter) {
+ private Progressable progressable;
+ private long progressBar;
+
+ public CombineOutputCollector(Counters.Counter outCounter, Progressable progressable, Configuration conf) {
this.outCounter = outCounter;
+ this.progressable=progressable;
+ progressBar = conf.getLong(MRJobConfig.COMBINE_RECORDS_BEFORE_PROGRESS, DEFAULT_COMBINE_RECORDS_BEFORE_PROGRESS);
}
+
public synchronized void setWriter(Writer<K, V> writer) {
this.writer = writer;
}
+
public synchronized void collect(K key, V value)
throws IOException {
outCounter.increment(1);
writer.append(key, value);
+ if ((outCounter.getValue() % progressBar) == 0) {
+ progressable.progress();
+ }
}
}
Modified: hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/TaskLog.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/TaskLog.java?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/TaskLog.java (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/TaskLog.java Thu Aug 4 17:09:14 2011
@@ -41,6 +41,7 @@ import org.apache.hadoop.fs.LocalFileSys
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.SecureIOUtils;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.mapreduce.util.ProcessTree;
@@ -111,34 +112,42 @@ public class TaskLog {
//stderr:<start-offset in the stderr file> <length>
//syslog:<start-offset in the syslog file> <length>
LogFileDetail l = new LogFileDetail();
- String str = fis.readLine();
- if (str == null) { //the file doesn't have anything
- throw new IOException ("Index file for the log of " + taskid+" doesn't exist.");
- }
- l.location = str.substring(str.indexOf(LogFileDetail.LOCATION)+
- LogFileDetail.LOCATION.length());
- //special cases are the debugout and profile.out files. They are guaranteed
- //to be associated with each task attempt since jvm reuse is disabled
- //when profiling/debugging is enabled
- if (filter.equals(LogName.DEBUGOUT) || filter.equals(LogName.PROFILE)) {
- l.length = new File(l.location, filter.toString()).length();
- l.start = 0;
- fis.close();
- return l;
- }
- str = fis.readLine();
- while (str != null) {
- //look for the exact line containing the logname
- if (str.contains(filter.toString())) {
- str = str.substring(filter.toString().length()+1);
- String[] startAndLen = str.split(" ");
- l.start = Long.parseLong(startAndLen[0]);
- l.length = Long.parseLong(startAndLen[1]);
- break;
+ String str = null;
+ try {
+ str = fis.readLine();
+ if (str == null) { // the file doesn't have anything
+ throw new IOException("Index file for the log of " + taskid
+ + " doesn't exist.");
+ }
+ l.location = str.substring(str.indexOf(LogFileDetail.LOCATION)
+ + LogFileDetail.LOCATION.length());
+ // special cases are the debugout and profile.out files. They are
+ // guaranteed
+ // to be associated with each task attempt since jvm reuse is disabled
+ // when profiling/debugging is enabled
+ if (filter.equals(LogName.DEBUGOUT) || filter.equals(LogName.PROFILE)) {
+ l.length = new File(l.location, filter.toString()).length();
+ l.start = 0;
+ fis.close();
+ return l;
}
str = fis.readLine();
+ while (str != null) {
+ // look for the exact line containing the logname
+ if (str.contains(filter.toString())) {
+ str = str.substring(filter.toString().length() + 1);
+ String[] startAndLen = str.split(" ");
+ l.start = Long.parseLong(startAndLen[0]);
+ l.length = Long.parseLong(startAndLen[1]);
+ break;
+ }
+ str = fis.readLine();
+ }
+ fis.close();
+ fis = null;
+ } finally {
+ IOUtils.cleanup(LOG, fis);
}
- fis.close();
return l;
}
@@ -189,22 +198,27 @@ public class TaskLog {
//LOG_DIR: <the dir where the task logs are really stored>
//STDOUT: <start-offset in the stdout file> <length>
//STDERR: <start-offset in the stderr file> <length>
- //SYSLOG: <start-offset in the syslog file> <length>
- dos.writeBytes(LogFileDetail.LOCATION + logLocation + "\n"
- + LogName.STDOUT.toString() + ":");
- dos.writeBytes(Long.toString(prevOutLength) + " ");
- dos.writeBytes(Long.toString(new File(logLocation, LogName.STDOUT
- .toString()).length() - prevOutLength)
- + "\n" + LogName.STDERR + ":");
- dos.writeBytes(Long.toString(prevErrLength) + " ");
- dos.writeBytes(Long.toString(new File(logLocation, LogName.STDERR
- .toString()).length() - prevErrLength)
- + "\n" + LogName.SYSLOG.toString() + ":");
- dos.writeBytes(Long.toString(prevLogLength) + " ");
- dos.writeBytes(Long.toString(new File(logLocation, LogName.SYSLOG
- .toString()).length() - prevLogLength)
- + "\n");
- dos.close();
+ //SYSLOG: <start-offset in the syslog file> <length>
+ try{
+ dos.writeBytes(LogFileDetail.LOCATION + logLocation + "\n"
+ + LogName.STDOUT.toString() + ":");
+ dos.writeBytes(Long.toString(prevOutLength) + " ");
+ dos.writeBytes(Long.toString(new File(logLocation, LogName.STDOUT
+ .toString()).length() - prevOutLength)
+ + "\n" + LogName.STDERR + ":");
+ dos.writeBytes(Long.toString(prevErrLength) + " ");
+ dos.writeBytes(Long.toString(new File(logLocation, LogName.STDERR
+ .toString()).length() - prevErrLength)
+ + "\n" + LogName.SYSLOG.toString() + ":");
+ dos.writeBytes(Long.toString(prevLogLength) + " ");
+ dos.writeBytes(Long.toString(new File(logLocation, LogName.SYSLOG
+ .toString()).length() - prevLogLength)
+ + "\n");
+ dos.close();
+ dos = null;
+ } finally {
+ IOUtils.cleanup(LOG, dos);
+ }
File indexFile = getIndexFile(currentTaskid, isCleanup);
Path indexFilePath = new Path(indexFile.getAbsolutePath());
Modified: hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/TaskTracker.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/TaskTracker.java?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/TaskTracker.java (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapred/TaskTracker.java Thu Aug 4 17:09:14 2011
@@ -1253,7 +1253,7 @@ public class TaskTracker
}
}
- private void launchTaskForJob(TaskInProgress tip, JobConf jobConf,
+ protected void launchTaskForJob(TaskInProgress tip, JobConf jobConf,
UserGroupInformation ugi) throws IOException {
synchronized (tip) {
tip.setJobConf(jobConf);
@@ -2351,30 +2351,35 @@ public class TaskTracker
* All exceptions are handled locally, so that we don't mess up the
* task tracker.
*/
- void startNewTask(TaskInProgress tip) {
- try {
- RunningJob rjob = localizeJob(tip);
- // Localization is done. Neither rjob.jobConf nor rjob.ugi can be null
- launchTaskForJob(tip, new JobConf(rjob.jobConf), rjob.ugi);
- } catch (Throwable e) {
- String msg = ("Error initializing " + tip.getTask().getTaskID() +
- ":\n" + StringUtils.stringifyException(e));
- LOG.warn(msg);
- tip.reportDiagnosticInfo(msg);
- try {
- tip.kill(true);
- tip.cleanup(true);
- } catch (IOException ie2) {
- LOG.info("Error cleaning up " + tip.getTask().getTaskID() + ":\n" +
- StringUtils.stringifyException(ie2));
+ void startNewTask(final TaskInProgress tip) {
+ Thread launchThread = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ RunningJob rjob = localizeJob(tip);
+ // Localization is done. Neither rjob.jobConf nor rjob.ugi can be null
+ launchTaskForJob(tip, new JobConf(rjob.getJobConf()), rjob.ugi);
+ } catch (Throwable e) {
+ String msg = ("Error initializing " + tip.getTask().getTaskID() +
+ ":\n" + StringUtils.stringifyException(e));
+ LOG.warn(msg);
+ tip.reportDiagnosticInfo(msg);
+ try {
+ tip.kill(true);
+ tip.cleanup(true);
+ } catch (IOException ie2) {
+ LOG.info("Error cleaning up " + tip.getTask().getTaskID() + ":\n" +
+ StringUtils.stringifyException(ie2));
+ }
+ if (e instanceof Error) {
+ LOG.error("TaskLauncher error " +
+ StringUtils.stringifyException(e));
+ }
+ }
}
+ });
+ launchThread.start();
- // Careful!
- // This might not be an 'Exception' - don't handle 'Error' here!
- if (e instanceof Error) {
- throw ((Error) e);
- }
- }
}
void addToMemoryManager(TaskAttemptID attemptId, boolean isMap,
Modified: hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapreduce/MRJobConfig.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapreduce/MRJobConfig.java?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapreduce/MRJobConfig.java (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapreduce/MRJobConfig.java Thu Aug 4 17:09:14 2011
@@ -260,6 +260,8 @@ public interface MRJobConfig {
public static final String REDUCE_MEMTOMEM_ENABLED = "mapreduce.reduce.merge.memtomem.enabled";
+ public static final String COMBINE_RECORDS_BEFORE_PROGRESS = "mapreduce.task.combine.progress.records";
+
public static final String JOB_NAMENODES = "mapreduce.job.hdfs-servers";
public static final String JOB_JOBTRACKER_ID = "mapreduce.job.kerberos.jtprinicipal";
Modified: hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java Thu Aug 4 17:09:14 2011
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.util.Iterator;
import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.mapreduce.Counter;
import org.apache.hadoop.mapreduce.CounterGroup;
import org.apache.hadoop.mapreduce.Counters;
@@ -33,6 +34,8 @@ import org.apache.avro.io.DatumWriter;
import org.apache.avro.specific.SpecificDatumWriter;
import org.apache.avro.generic.GenericData;
import org.apache.avro.util.Utf8;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
/**
* Event Writer is an utility class used to write events to the underlying
@@ -47,6 +50,7 @@ class EventWriter {
private DatumWriter<Event> writer =
new SpecificDatumWriter<Event>(Event.class);
private Encoder encoder;
+ private static final Log LOG = LogFactory.getLog(EventWriter.class);
EventWriter(FSDataOutputStream out) throws IOException {
this.out = out;
@@ -72,8 +76,13 @@ class EventWriter {
}
void close() throws IOException {
- encoder.flush();
- out.close();
+ try {
+ encoder.flush();
+ out.close();
+ out = null;
+ } finally {
+ IOUtils.cleanup(LOG, out);
+ }
}
private static final Schema GROUPS =
Modified: hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapreduce/lib/output/MultipleOutputs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapreduce/lib/output/MultipleOutputs.java?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapreduce/lib/output/MultipleOutputs.java (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapreduce/lib/output/MultipleOutputs.java Thu Aug 4 17:09:14 2011
@@ -132,6 +132,10 @@ public class MultipleOutputs<KEYOUT, VAL
* Cache for the taskContexts
*/
private Map<String, TaskAttemptContext> taskContexts = new HashMap<String, TaskAttemptContext>();
+ /**
+ * Cached TaskAttemptContext which uses the job's configured settings
+ */
+ private TaskAttemptContext jobOutputFormatContext;
/**
* Checks if a named output name is valid token.
@@ -382,11 +386,13 @@ public class MultipleOutputs<KEYOUT, VAL
public void write(KEYOUT key, VALUEOUT value, String baseOutputPath)
throws IOException, InterruptedException {
checkBaseOutputPath(baseOutputPath);
- TaskAttemptContext taskContext =
- new TaskAttemptContextImpl(context.getConfiguration(),
- context.getTaskAttemptID(),
- new WrappedStatusReporter(context));
- getRecordWriter(taskContext, baseOutputPath).write(key, value);
+ if (jobOutputFormatContext == null) {
+ jobOutputFormatContext =
+ new TaskAttemptContextImpl(context.getConfiguration(),
+ context.getTaskAttemptID(),
+ new WrappedStatusReporter(context));
+ }
+ getRecordWriter(jobOutputFormatContext, baseOutputPath).write(key, value);
}
// by being synchronized MultipleOutputTask can be use with a
Modified: hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapreduce/lib/partition/TotalOrderPartitioner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapreduce/lib/partition/TotalOrderPartitioner.java?rev=1153931&r1=1153930&r2=1153931&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapreduce/lib/partition/TotalOrderPartitioner.java (original)
+++ hadoop/common/branches/HDFS-1623/mapreduce/src/java/org/apache/hadoop/mapreduce/lib/partition/TotalOrderPartitioner.java Thu Aug 4 17:09:14 2011
@@ -23,6 +23,8 @@ import java.lang.reflect.Array;
import java.util.ArrayList;
import java.util.Arrays;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configurable;
@@ -30,6 +32,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BinaryComparable;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.RawComparator;
@@ -56,6 +59,7 @@ public class TotalOrderPartitioner<K ext
public static final String NATURAL_ORDER =
"mapreduce.totalorderpartitioner.naturalorder";
Configuration conf;
+ private static final Log LOG = LogFactory.getLog(TotalOrderPartitioner.class);
public TotalOrderPartitioner() { }
@@ -298,11 +302,16 @@ public class TotalOrderPartitioner<K ext
ArrayList<K> parts = new ArrayList<K>();
K key = ReflectionUtils.newInstance(keyClass, conf);
NullWritable value = NullWritable.get();
- while (reader.next(key, value)) {
- parts.add(key);
- key = ReflectionUtils.newInstance(keyClass, conf);
+ try {
+ while (reader.next(key, value)) {
+ parts.add(key);
+ key = ReflectionUtils.newInstance(keyClass, conf);
+ }
+ reader.close();
+ reader = null;
+ } finally {
+ IOUtils.cleanup(LOG, reader);
}
- reader.close();
return parts.toArray((K[])Array.newInstance(keyClass, parts.size()));
}
Propchange: hadoop/common/branches/HDFS-1623/mapreduce/src/test/mapred/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug 4 17:09:14 2011
@@ -1,2 +1,3 @@
+/hadoop/common/trunk/mapreduce/src/test/mapred:1152502-1153927
/hadoop/core/branches/branch-0.19/mapred/src/test/mapred:713112
/hadoop/core/trunk/src/test/mapred:776175-785643
Propchange: hadoop/common/branches/HDFS-1623/mapreduce/src/test/mapred/org/apache/hadoop/fs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug 4 17:09:14 2011
@@ -1,3 +1,4 @@
+/hadoop/common/trunk/mapreduce/src/test/mapred/org/apache/hadoop/fs:1152502-1153927
/hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/fs:713112
/hadoop/core/trunk/src/test/mapred/org/apache/hadoop/fs:776175-785643
/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/fs:817878-835934
Propchange: hadoop/common/branches/HDFS-1623/mapreduce/src/test/mapred/org/apache/hadoop/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug 4 17:09:14 2011
@@ -1,3 +1,4 @@
+/hadoop/common/trunk/mapreduce/src/test/mapred/org/apache/hadoop/hdfs:1152502-1153927
/hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/hdfs:713112
/hadoop/core/trunk/src/test/mapred/org/apache/hadoop/hdfs:776175-785643
/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/hdfs:817878-835934
Propchange: hadoop/common/branches/HDFS-1623/mapreduce/src/test/mapred/org/apache/hadoop/io/FileBench.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug 4 17:09:14 2011
@@ -1,3 +1,4 @@
+/hadoop/common/trunk/mapreduce/src/test/mapred/org/apache/hadoop/io/FileBench.java:1152502-1153927
/hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/io/FileBench.java:713112
/hadoop/core/trunk/src/test/mapred/org/apache/hadoop/io/FileBench.java:776175-785643
/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/io/FileBench.java:817878-835934
Propchange: hadoop/common/branches/HDFS-1623/mapreduce/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug 4 17:09:14 2011
@@ -1,3 +1,4 @@
+/hadoop/common/trunk/mapreduce/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java:1152502-1153927
/hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java:713112
/hadoop/core/trunk/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java:776175-785643
/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java:817878-835934
Propchange: hadoop/common/branches/HDFS-1623/mapreduce/src/test/mapred/org/apache/hadoop/ipc/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug 4 17:09:14 2011
@@ -1,3 +1,4 @@
+/hadoop/common/trunk/mapreduce/src/test/mapred/org/apache/hadoop/ipc:1152502-1153927
/hadoop/core/branches/branch-0.19/hdfs/src/test/hdfs-with-mr/org/apache/hadoop/ipc:713112
/hadoop/core/trunk/src/test/hdfs-with-mr/org/apache/hadoop/ipc:776175-784663
/hadoop/hdfs/branches/HDFS-265/src/test/hdfs-with-mr/org/apache/hadoop/ipc:796829-820463
Propchange: hadoop/common/branches/HDFS-1623/mapreduce/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug 4 17:09:14 2011
@@ -1,3 +1,4 @@
+/hadoop/common/trunk/mapreduce/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java:1152502-1153927
/hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java:713112
/hadoop/core/trunk/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java:776175-785643
/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java:817878-835934
Propchange: hadoop/common/branches/HDFS-1623/mapreduce/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug 4 17:09:14 2011
@@ -1,3 +1,4 @@
+/hadoop/common/trunk/mapreduce/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java:1152502-1153927
/hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java:713112
/hadoop/core/trunk/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java:776175-785643
/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java:817878-835934
Propchange: hadoop/common/branches/HDFS-1623/mapreduce/src/webapps/job/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Aug 4 17:09:14 2011
@@ -1,2 +1,3 @@
+/hadoop/common/trunk/mapreduce/src/webapps/job:1152502-1153927
/hadoop/core/branches/branch-0.19/mapred/src/webapps/job:713112
/hadoop/core/trunk/src/webapps/job:776175-785643
|