hadoop-mapreduce-commits mailing list archives

Author: amareshwari
Date: Wed Jun 16 11:23:27 2010
New Revision: 955199

URL: http://svn.apache.org/viewvc?rev=955199&view=rev
Log:
MAPREDUCE-1853. Merge -r 955197:955198 from trunk

Modified:
    hadoop/mapreduce/branches/branch-0.21/CHANGES.txt
    hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/lib/output/MultipleOutputs.java

Modified: hadoop/mapreduce/branches/branch-0.21/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/CHANGES.txt?rev=955199&r1=955198&r2=955199&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/branch-0.21/CHANGES.txt Wed Jun 16 11:23:27 2010
@@ -692,6 +692,9 @@ Release 0.21.0 - Unreleased
     MAPREDUCE-1610. Forrest documentation should be updated to reflect
     the changes in MAPREDUCE-856. (Ravi Gummadi via vinodkv)
 
+    MAPREDUCE-1853. Adds caching for TaskAttemptContext in MultipleOutputs.
+    (Torsten Curdt via amareshwari)
+
   BUG FIXES
 
     MAPREDUCE-878. Rename fair scheduler design doc to 

Modified: hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/lib/output/MultipleOutputs.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/lib/output/MultipleOutputs.java?rev=955199&r1=955198&r2=955199&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/lib/output/MultipleOutputs.java
(original)
+++ hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapreduce/lib/output/MultipleOutputs.java
Wed Jun 16 11:23:27 2010
@@ -131,6 +131,11 @@ public class MultipleOutputs<KEYOUT, VAL
   private static final String COUNTERS_GROUP = MultipleOutputs.class.getName();
 
   /**
+   * Cache for the taskContexts
+   */
+  private Map<String, TaskAttemptContext> taskContexts = new HashMap<String, TaskAttemptContext>();
+
+  /**
    * Checks if a named output name is valid token.
    *
    * @param namedOutput named output Name
@@ -422,15 +427,25 @@ public class MultipleOutputs<KEYOUT, VAL
    // Create a taskAttemptContext for the named output with 
    // output format and output key/value types put in the context
   private TaskAttemptContext getContext(String nameOutput) throws IOException {
+      
+    TaskAttemptContext taskContext = taskContexts.get(nameOutput);
+    
+    if (taskContext != null) {
+        return taskContext;
+    }
+    
     // The following trick leverages the instantiation of a record writer via
     // the job thus supporting arbitrary output formats.
     Job job = new Job(context.getConfiguration());
     job.setOutputFormatClass(getNamedOutputFormatClass(context, nameOutput));
     job.setOutputKeyClass(getNamedOutputKeyClass(context, nameOutput));
     job.setOutputValueClass(getNamedOutputValueClass(context, nameOutput));
-    TaskAttemptContext taskContext = 
+    taskContext = 
       new TaskAttemptContextImpl(job.getConfiguration(), 
                                  context.getTaskAttemptID());
+
+    taskContexts.put(nameOutput, taskContext);
+
     return taskContext;
   }