hama-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From edwardy...@apache.org
Subject svn commit: r712931 - in /incubator/hama/trunk: ./ src/java/org/apache/hama/ src/java/org/apache/hama/algebra/ src/java/org/apache/hama/io/
Date Tue, 11 Nov 2008 03:56:26 GMT
Author: edwardyoon
Date: Mon Nov 10 19:56:26 2008
New Revision: 712931

URL: http://svn.apache.org/viewvc?rev=712931&view=rev
Log:
Add getNumMap/reduceTasks to HamaConfiguration

Modified:
    incubator/hama/trunk/CHANGES.txt
    incubator/hama/trunk/src/java/org/apache/hama/AbstractVector.java
    incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java
    incubator/hama/trunk/src/java/org/apache/hama/DenseVector.java
    incubator/hama/trunk/src/java/org/apache/hama/HamaConfiguration.java
    incubator/hama/trunk/src/java/org/apache/hama/algebra/RowCyclicAdditionMap.java
    incubator/hama/trunk/src/java/org/apache/hama/algebra/RowCyclicAdditionReduce.java
    incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyMap.java
    incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyReduce.java
    incubator/hama/trunk/src/java/org/apache/hama/io/VectorWritable.java

Modified: incubator/hama/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/CHANGES.txt?rev=712931&r1=712930&r2=712931&view=diff
==============================================================================
--- incubator/hama/trunk/CHANGES.txt (original)
+++ incubator/hama/trunk/CHANGES.txt Mon Nov 10 19:56:26 2008
@@ -3,7 +3,7 @@
 Trunk (unreleased changes)
 
   NEW FEATURES
-    
+    HAMA-104: Add getNumMap/reduceTasks to HamaConfiguration (edwardyoon)
     HAMA-92: Add subMatrix() to Matrix (edwardyoon)
     HAMA-83: Add a writable comparable for BlockID (edwardyoon) 
     HAMA-81: Add subVector(int i0, int i1) to Vector (edwardyoon)

Modified: incubator/hama/trunk/src/java/org/apache/hama/AbstractVector.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/AbstractVector.java?rev=712931&r1=712930&r2=712931&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/AbstractVector.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/AbstractVector.java Mon Nov 10 19:56:26
2008
@@ -32,7 +32,12 @@
   }
   
   public void set(int index, double value) {
-    entries.put(index, new VectorEntry(value));
+    // If entries are null, create new object 
+    if(this.entries == null) {
+      this.entries = new VectorMapWritable<Integer, VectorEntry>();
+    }
+    
+    this.entries.put(index, new VectorEntry(value));
   }
   
   public void add(int index, double value) {

Modified: incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java?rev=712931&r1=712930&r2=712931&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java Mon Nov 10 19:56:26 2008
@@ -204,8 +204,11 @@
   public static Matrix random(HamaConfiguration conf, int m, int n)
       throws IOException {
     Matrix rand = new DenseMatrix(conf);
+    DenseVector vector = new DenseVector();
+    LOG.info("Create the " + m + " * " + n + " random matrix : " + rand.getPath());
+
     for (int i = 0; i < m; i++) {
-      DenseVector vector = new DenseVector();
+      vector.clear();
       for (int j = 0; j < n; j++) {
         vector.set(j, RandomVariable.rand());
       }
@@ -213,7 +216,6 @@
     }
 
     rand.setDimension(m, n);
-    LOG.info("Create the " + m + " * " + n + " random matrix : " + rand.getPath());
     return rand;
   }
 
@@ -229,7 +231,8 @@
   public static Matrix identity(HamaConfiguration conf, int m, int n)
       throws IOException {
     Matrix identity = new DenseMatrix(conf);
-
+    LOG.info("Create the " + m + " * " + n + " identity matrix : " + identity.getPath());
+    
     for (int i = 0; i < m; i++) {
       DenseVector vector = new DenseVector();
       for (int j = 0; j < n; j++) {
@@ -239,7 +242,6 @@
     }
 
     identity.setDimension(m, n);
-    LOG.info("Create the " + m + " * " + n + " identity matrix : " + identity.getPath());
     return identity;
   }
 
@@ -249,9 +251,8 @@
     JobConf jobConf = new JobConf(config);
     jobConf.setJobName("addition MR job" + result.getPath());
 
-    jobConf.setNumMapTasks(Integer.parseInt(config.get("mapred.map.tasks")));
-    jobConf.setNumReduceTasks(Integer.parseInt(config
-        .get("mapred.reduce.tasks")));
+    jobConf.setNumMapTasks(config.getNumMapTasks());
+    jobConf.setNumReduceTasks(config.getNumReduceTasks());
 
     RowCyclicAdditionMap.initJob(this.getPath(), B.getPath(), RowCyclicAdditionMap.class,
         IntWritable.class, VectorWritable.class, jobConf);
@@ -267,15 +268,7 @@
   }
 
   public DenseVector getRow(int row) throws IOException {
-    VectorMapWritable<Integer, VectorEntry> values = new VectorMapWritable<Integer,
VectorEntry>();
-    RowResult rowResult = table.getRow(String.valueOf(row));
-
-    for (Map.Entry<byte[], Cell> f : rowResult.entrySet()) {
-      VectorEntry entry = new VectorEntry(f.getValue());
-      values.put(Numeric.getColumnIndex(f.getKey()), entry);
-    }
-
-    return new DenseVector(values);
+    return new DenseVector(table.getRow(Numeric.intToBytes(row)));
   }
 
   public Vector getColumn(int column) throws IOException {
@@ -286,8 +279,8 @@
     VectorMapWritable<Integer, VectorEntry> trunk = new VectorMapWritable<Integer,
VectorEntry>();
 
     for (RowResult row : scan) {
-      trunk.put(Numeric.bytesToInt(row.getRow()), new VectorEntry(row
-          .get(columnKey)));
+      trunk.put(Numeric.bytesToInt(row.getRow()), 
+          new VectorEntry(row.get(columnKey)));
     }
 
     return new DenseVector(trunk);
@@ -299,9 +292,8 @@
     JobConf jobConf = new JobConf(config);
     jobConf.setJobName("multiplication MR job : " + result.getPath());
 
-    jobConf.setNumMapTasks(Integer.parseInt(config.get("mapred.map.tasks")));
-    jobConf.setNumReduceTasks(Integer.parseInt(config
-        .get("mapred.reduce.tasks")));
+    jobConf.setNumMapTasks(config.getNumMapTasks());
+    jobConf.setNumReduceTasks(config.getNumReduceTasks());
 
     SIMDMultiplyMap.initJob(this.getPath(), B.getPath(), SIMDMultiplyMap.class,
         IntWritable.class, VectorWritable.class, jobConf);
@@ -352,8 +344,7 @@
       c[i] = Numeric.getColumnIndex(j0 + i);
     }
 
-    Scanner scan = table.getScanner(c, Numeric.intToBytes(i0), Numeric
-        .intToBytes(i1 + 1));
+    Scanner scan = table.getScanner(c, Numeric.intToBytes(i0), Numeric.intToBytes(i1 + 1));
 
     int rKey = 0, cKey = 0;
     for (RowResult row : scan) {

Modified: incubator/hama/trunk/src/java/org/apache/hama/DenseVector.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/DenseVector.java?rev=712931&r1=712930&r2=712931&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/DenseVector.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/DenseVector.java Mon Nov 10 19:56:26 2008
@@ -20,10 +20,14 @@
 package org.apache.hama;
 
 import java.util.Iterator;
+import java.util.Map;
 import java.util.Set;
 
+import org.apache.hadoop.hbase.io.Cell;
+import org.apache.hadoop.hbase.io.RowResult;
 import org.apache.hama.io.VectorEntry;
 import org.apache.hama.io.VectorMapWritable;
+import org.apache.hama.util.Numeric;
 import org.apache.log4j.Logger;
 
 public class DenseVector extends AbstractVector implements Vector {
@@ -37,6 +41,14 @@
     this.entries = m;
   }
 
+  public DenseVector(RowResult row) {
+    this.entries = new VectorMapWritable<Integer, VectorEntry>();
+    for (Map.Entry<byte[], Cell> f : row.entrySet()) {
+      this.entries.put(Numeric.getColumnIndex(f.getKey()), 
+          new VectorEntry(f.getValue()));
+    }
+  }
+
   public Vector add(double alpha, Vector v) {
     if (alpha == 0)
       return this;
@@ -75,19 +87,9 @@
   }
 
   public Vector scale(double alpha) {
-    Set<Integer> keySet = this.entries.keySet();
-    Iterator<Integer> it = keySet.iterator();
-
-    int i = 0;
-    while (it.hasNext()) {
-      int key = it.next();
-      double oValue = this.get(key);
-      double nValue = oValue * alpha;
-
-      this.entries.put(i, new VectorEntry(nValue));
-      i++;
+    for(Map.Entry<Integer, VectorEntry> e : this.entries.entrySet()) {
+      this.entries.put(e.getKey(), new VectorEntry(e.getValue().getValue() * alpha));
     }
-
     return this;
   }
 

Modified: incubator/hama/trunk/src/java/org/apache/hama/HamaConfiguration.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/HamaConfiguration.java?rev=712931&r1=712930&r2=712931&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/HamaConfiguration.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/HamaConfiguration.java Mon Nov 10 19:56:26
2008
@@ -54,6 +54,14 @@
     set("mapred.reduce.tasks",String.valueOf(reduce));
   }
   
+  public int getNumMapTasks() {
+    return Integer.parseInt(get("mapred.map.tasks"));
+  }
+
+  public int getNumReduceTasks() {
+    return Integer.parseInt(get("mapred.reduce.tasks"));
+  }
+  
   private void addHbaseResources() {
     addResource("hama-site.xml");
   }

Modified: incubator/hama/trunk/src/java/org/apache/hama/algebra/RowCyclicAdditionMap.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/algebra/RowCyclicAdditionMap.java?rev=712931&r1=712930&r2=712931&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/algebra/RowCyclicAdditionMap.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/algebra/RowCyclicAdditionMap.java Mon Nov
10 19:56:26 2008
@@ -29,12 +29,12 @@
 import org.apache.hama.DenseVector;
 import org.apache.hama.HamaConfiguration;
 import org.apache.hama.Matrix;
-import org.apache.hama.Vector;
 import org.apache.hama.io.VectorWritable;
 import org.apache.hama.mapred.RowCyclicMap;
 import org.apache.log4j.Logger;
 
-public class RowCyclicAdditionMap extends RowCyclicMap<IntWritable, VectorWritable>
{
+public class RowCyclicAdditionMap extends
+    RowCyclicMap<IntWritable, VectorWritable> {
   static final Logger LOG = Logger.getLogger(RowCyclicAdditionMap.class);
   protected Matrix matrix_b;
   public static final String MATRIX_B = "hama.addition.matrix.b";
@@ -64,9 +64,8 @@
       OutputCollector<IntWritable, VectorWritable> output, Reporter reporter)
       throws IOException {
 
-    Vector v1 = matrix_b.getRow(key.get());
-    output.collect(key, new VectorWritable(key.get(), (DenseVector) v1
-        .add(value.getDenseVector())));
+    output.collect(key, new VectorWritable(key.get(), 
+        (DenseVector) matrix_b.getRow(key.get()).add(value.getDenseVector())));
 
   }
 

Modified: incubator/hama/trunk/src/java/org/apache/hama/algebra/RowCyclicAdditionReduce.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/algebra/RowCyclicAdditionReduce.java?rev=712931&r1=712930&r2=712931&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/algebra/RowCyclicAdditionReduce.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/algebra/RowCyclicAdditionReduce.java Mon
Nov 10 19:56:26 2008
@@ -21,12 +21,10 @@
 
 import java.io.IOException;
 import java.util.Iterator;
-import java.util.Map;
 
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.mapred.OutputCollector;
 import org.apache.hadoop.mapred.Reporter;
-import org.apache.hama.io.VectorEntry;
 import org.apache.hama.io.VectorUpdate;
 import org.apache.hama.io.VectorWritable;
 import org.apache.hama.mapred.RowCyclicReduce;
@@ -39,11 +37,7 @@
       throws IOException {
 
     VectorUpdate update = new VectorUpdate(key.get());
-    VectorWritable vector = values.next();
-    
-    for (Map.Entry<Integer, VectorEntry> f : vector.entrySet()) {
-      update.put(f.getKey(), f.getValue().getValue());
-    }
+    update.putAll(values.next().entrySet());
 
     output.collect(key, update);
   }

Modified: incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyMap.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyMap.java?rev=712931&r1=712930&r2=712931&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyMap.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyMap.java Mon Nov 10
19:56:26 2008
@@ -29,7 +29,6 @@
 import org.apache.hama.DenseVector;
 import org.apache.hama.HamaConfiguration;
 import org.apache.hama.Matrix;
-import org.apache.hama.Vector;
 import org.apache.hama.io.VectorWritable;
 import org.apache.hama.mapred.RowCyclicMap;
 import org.apache.log4j.Logger;
@@ -70,9 +69,7 @@
     sum.clear();
 
     for(int i = 0; i < value.size(); i++) {
-      Vector v2 = matrix_b.getRow(i);
-      DenseVector curr = (DenseVector) v2.scale(value.get(i).getValue());
-      sum.add(curr);
+      sum.add(matrix_b.getRow(i).scale(value.get(i)));
     }
     
     output.collect(key, new VectorWritable(key.get(), sum));

Modified: incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyReduce.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyReduce.java?rev=712931&r1=712930&r2=712931&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyReduce.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyReduce.java Mon Nov
10 19:56:26 2008
@@ -21,12 +21,10 @@
 
 import java.io.IOException;
 import java.util.Iterator;
-import java.util.Map;
 
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.mapred.OutputCollector;
 import org.apache.hadoop.mapred.Reporter;
-import org.apache.hama.io.VectorEntry;
 import org.apache.hama.io.VectorUpdate;
 import org.apache.hama.io.VectorWritable;
 import org.apache.hama.mapred.RowCyclicReduce;
@@ -41,13 +39,8 @@
       OutputCollector<IntWritable, VectorUpdate> output, Reporter reporter)
       throws IOException {
 
-
     VectorUpdate update = new VectorUpdate(key.get());
-    VectorWritable vector = values.next();
-    
-    for (Map.Entry<Integer, VectorEntry> f : vector.entrySet()) {
-      update.put(f.getKey(), f.getValue().getValue());
-    }
+    update.putAll(values.next().entrySet());
 
     output.collect(key, update);
   }

Modified: incubator/hama/trunk/src/java/org/apache/hama/io/VectorWritable.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/io/VectorWritable.java?rev=712931&r1=712930&r2=712931&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/io/VectorWritable.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/io/VectorWritable.java Mon Nov 10 19:56:26
2008
@@ -57,6 +57,14 @@
     return new DenseVector(entries);
   }
 
+  public Double get(Integer column) {
+    return this.entries.get(column).getValue();
+  }
+
+  public int size() {
+    return this.entries.size();
+  }
+  
   public VectorEntry put(Integer key, VectorEntry value) {
     throw new UnsupportedOperationException("VectorWritable is read-only!");
   }
@@ -124,17 +132,6 @@
   }
 
   /**
-   * Get the VectorEntry that corresponds to column
-   */
-  public VectorEntry get(Integer column) {
-    return this.entries.get(column);
-  }
-
-  public int size() {
-    return this.entries.size();
-  }
-
-  /**
    * 
    * The inner class for an entry of row.
    * 



Mime
View raw message