hama-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From edwardy...@apache.org
Subject svn commit: r712658 - in /incubator/hama/trunk: ./ src/java/org/apache/hama/ src/java/org/apache/hama/algebra/ src/test/org/apache/hama/
Date Mon, 10 Nov 2008 13:13:43 GMT
Author: edwardyoon
Date: Mon Nov 10 05:13:42 2008
New Revision: 712658

URL: http://svn.apache.org/viewvc?rev=712658&view=rev
Log:
Reduce an rows of intermediate to improve the multiplication performance

Modified:
    incubator/hama/trunk/CHANGES.txt
    incubator/hama/trunk/src/java/org/apache/hama/AbstractVector.java
    incubator/hama/trunk/src/java/org/apache/hama/DenseVector.java
    incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyMap.java
    incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyReduce.java
    incubator/hama/trunk/src/test/org/apache/hama/TestDenseVector.java

Modified: incubator/hama/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/CHANGES.txt?rev=712658&r1=712657&r2=712658&view=diff
==============================================================================
--- incubator/hama/trunk/CHANGES.txt (original)
+++ incubator/hama/trunk/CHANGES.txt Mon Nov 10 05:13:42 2008
@@ -31,6 +31,8 @@
     
   IMPROVEMENTS
     
+    HAMA-103: Reduce an rows of intermediate 
+                to improve the multiplication performance (edwardyoon)
     HAMA-88: DenseMatrix.close should not delete the table 
                that are aliased in HamaAdmin. (samuel via edwardyoon)
     HAMA-89: Exception Handling of a ParseException (samuel via edwardyoon)

Modified: incubator/hama/trunk/src/java/org/apache/hama/AbstractVector.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/AbstractVector.java?rev=712658&r1=712657&r2=712658&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/AbstractVector.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/AbstractVector.java Mon Nov 10 05:13:42
2008
@@ -54,7 +54,7 @@
    * @return a size of vector
    */
   public int size() {
-    return this.entries.size();
+    return (this.entries != null) ? this.entries.size() : 0;
   }
   
   public VectorMapWritable<Integer, VectorEntry> getEntries() {

Modified: incubator/hama/trunk/src/java/org/apache/hama/DenseVector.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/DenseVector.java?rev=712658&r1=712657&r2=712658&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/DenseVector.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/DenseVector.java Mon Nov 10 05:13:42 2008
@@ -151,4 +151,8 @@
 
     return res;
   }
+
+  public void clear() {
+    this.entries = null;
+  }
 }

Modified: incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyMap.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyMap.java?rev=712658&r1=712657&r2=712658&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyMap.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyMap.java Mon Nov 10
05:13:42 2008
@@ -41,7 +41,8 @@
   static final Logger LOG = Logger.getLogger(SIMDMultiplyMap.class);
   protected Matrix matrix_b;
   public static final String MATRIX_B = "hama.multiplication.matrix.b";
-
+  public static final DenseVector sum = new DenseVector();;
+  
   public void configure(JobConf job) {
     try {
       matrix_b = new DenseMatrix(new HamaConfiguration(), job.get(MATRIX_B, ""));
@@ -66,13 +67,14 @@
   public void map(IntWritable key, VectorWritable value,
       OutputCollector<IntWritable, VectorWritable> output, Reporter reporter)
       throws IOException {
+    sum.clear();
 
-    DenseVector v1 = value.getDenseVector();
-    
-    for(int i = 0; i < v1.size(); i++) {
+    for(int i = 0; i < value.size(); i++) {
       Vector v2 = matrix_b.getRow(i);
-      DenseVector sum = (DenseVector) v2.scale(v1.get(i));
-      output.collect(key, new VectorWritable(key.get(), sum));
+      DenseVector curr = (DenseVector) v2.scale(value.get(i).getValue());
+      sum.add(curr);
     }
+    
+    output.collect(key, new VectorWritable(key.get(), sum));
   }
 }

Modified: incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyReduce.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyReduce.java?rev=712658&r1=712657&r2=712658&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyReduce.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyReduce.java Mon Nov
10 05:13:42 2008
@@ -20,13 +20,13 @@
 package org.apache.hama.algebra;
 
 import java.io.IOException;
-import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
 
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.mapred.OutputCollector;
 import org.apache.hadoop.mapred.Reporter;
+import org.apache.hama.io.VectorEntry;
 import org.apache.hama.io.VectorUpdate;
 import org.apache.hama.io.VectorWritable;
 import org.apache.hama.mapred.RowCyclicReduce;
@@ -35,30 +35,20 @@
 public class SIMDMultiplyReduce extends
     RowCyclicReduce<IntWritable, VectorWritable> {
   static final Logger LOG = Logger.getLogger(SIMDMultiplyReduce.class);
-  public static final Map<Integer, Double> buffer = new HashMap<Integer, Double>();
   
   @Override
   public void reduce(IntWritable key, Iterator<VectorWritable> values,
       OutputCollector<IntWritable, VectorUpdate> output, Reporter reporter)
       throws IOException {
 
+
     VectorUpdate update = new VectorUpdate(key.get());
-    VectorWritable sum;
+    VectorWritable vector = values.next();
     
-    // Summation
-    while (values.hasNext()) {
-      sum = values.next();
-      for (int i = 0; i < sum.size(); i++) {
-        if (buffer.containsKey(i)) {
-          buffer.put(i, sum.get(i).getValue() + buffer.get(i));
-        } else {
-          buffer.put(i, sum.get(i).getValue());
-        }
-      }
+    for (Map.Entry<Integer, VectorEntry> f : vector.entrySet()) {
+      update.put(f.getKey(), f.getValue().getValue());
     }
-    
-    update.putAll(buffer);
-    buffer.clear();
+
     output.collect(key, update);
   }
 

Modified: incubator/hama/trunk/src/test/org/apache/hama/TestDenseVector.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/test/org/apache/hama/TestDenseVector.java?rev=712658&r1=712657&r2=712658&view=diff
==============================================================================
--- incubator/hama/trunk/src/test/org/apache/hama/TestDenseVector.java (original)
+++ incubator/hama/trunk/src/test/org/apache/hama/TestDenseVector.java Mon Nov 10 05:13:42
2008
@@ -144,4 +144,12 @@
     v1.add(0, norm1);
     assertEquals(v1.get(0), old + norm1);
   }
+  
+  /**
+   * Clear test
+   */
+  public void testClear() {
+    ((DenseVector) v1).clear();
+    assertEquals(v1.size(), 0);
+  }
 }



Mime
View raw message