hama-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From edwardy...@apache.org
Subject svn commit: r695149 - in /incubator/hama/trunk: ./ src/examples/org/apache/hama/examples/ src/java/org/apache/hama/ src/java/org/apache/hama/mapred/ src/test/org/apache/hama/
Date Sun, 14 Sep 2008 08:56:17 GMT
Author: edwardyoon
Date: Sun Sep 14 01:56:17 2008
New Revision: 695149

URL: http://svn.apache.org/viewvc?rev=695149&view=rev
Log:
Write dimension attributes when job completed

Modified:
    incubator/hama/trunk/CHANGES.txt
    incubator/hama/trunk/src/examples/org/apache/hama/examples/MatrixAddition.java
    incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java
    incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java
    incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixInputFormat.java
    incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixInputFormatBase.java
    incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixOutputFormat.java
    incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java

Modified: incubator/hama/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/CHANGES.txt?rev=695149&r1=695148&r2=695149&view=diff
==============================================================================
--- incubator/hama/trunk/CHANGES.txt (original)
+++ incubator/hama/trunk/CHANGES.txt Sun Sep 14 01:56:17 2008
@@ -48,6 +48,7 @@
 
   BUG FIXES
 
+    HAMA-55: Write dimension attributes when job completed (edwardyoon)
     HAMA-54: Split doesn't split by map task num (edwardyoon)
     HAMA-53: NullPointerException on distributed cluster (edwardyoon)
     HAMA-26: hama-formatter.xml should be removed (edwardyoon)  

Modified: incubator/hama/trunk/src/examples/org/apache/hama/examples/MatrixAddition.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/examples/org/apache/hama/examples/MatrixAddition.java?rev=695149&r1=695148&r2=695149&view=diff
==============================================================================
--- incubator/hama/trunk/src/examples/org/apache/hama/examples/MatrixAddition.java (original)
+++ incubator/hama/trunk/src/examples/org/apache/hama/examples/MatrixAddition.java Sun Sep
14 01:56:17 2008
@@ -43,16 +43,16 @@
 
     System.out.println("\nMatrix A");
     System.out.println("----------------------");
-    for(int i =  0; i < row; i++) {
-      for(int j =  0; j < row; j++) {
+    for(int i =  0; i < a.getRows(); i++) {
+      for(int j =  0; j < a.getColumns(); j++) {
         System.out.println(a.get(i, j));
       }
     }
     
     System.out.println("\nMatrix B");
     System.out.println("----------------------");
-    for(int i =  0; i < row; i++) {
-      for(int j =  0; j < row; j++) {
+    for(int i =  0; i < b.getRows(); i++) {
+      for(int j =  0; j < b.getColumns(); j++) {
         System.out.println(b.get(i, j));
       }
     }
@@ -62,8 +62,8 @@
     
     System.out.println("\nC = A + B");
     System.out.println("----------------------");
-    for(int i =  0; i < row; i++) {
-      for(int j =  0; j < row; j++) {
+    for(int i =  0; i < c.getRows(); i++) {
+      for(int j =  0; j < c.getColumns(); j++) {
         System.out.println(c.get(i, j));
       }
     }

Modified: incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java?rev=695149&r1=695148&r2=695149&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java Sun Sep 14 01:56:17
2008
@@ -28,6 +28,9 @@
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.io.BatchUpdate;
 import org.apache.hadoop.hbase.io.Cell;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.RunningJob;
 import org.apache.hama.util.Numeric;
 import org.apache.log4j.Logger;
 
@@ -75,6 +78,17 @@
     }
   }
 
+  public void execute(JobConf jobConf, Matrix result) throws IOException {
+    RunningJob rJob = JobClient.runJob(jobConf);
+    // TODO : When HADOOP-4043 done, we should change this.
+    long rows = rJob.getCounters().findCounter(
+        "org.apache.hadoop.mapred.Task$Counter", 8, "REDUCE_OUTPUT_RECORDS")
+        .getCounter();
+    // TODO : Thinking about more efficient method.
+    int columns = result.getRow(0).size();
+    result.setDimension((int)rows, columns);
+  }
+  
   /** {@inheritDoc} */
   public double get(int i, int j) throws IOException {
     double result = -1;

Modified: incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java?rev=695149&r1=695148&r2=695149&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java Sun Sep 14 01:56:17 2008
@@ -28,7 +28,6 @@
 import org.apache.hadoop.hbase.client.Scanner;
 import org.apache.hadoop.hbase.io.RowResult;
 import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.mapred.JobClient;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hama.algebra.AdditionMap;
 import org.apache.hama.algebra.AdditionReduce;
@@ -133,18 +132,17 @@
 
   public Matrix add(Matrix B) throws IOException {
     String output = RandomVariable.randMatrixName();
-    Matrix C = new DenseMatrix(config, output);
+    Matrix result = new DenseMatrix(config, output);
 
     JobConf jobConf = new JobConf(config);
-    jobConf.setJobName("addition MR job");
-    jobConf.setNumMapTasks(2);
+    jobConf.setJobName("addition MR job" + result.getName());
 
     AdditionMap.initJob(this.getName(), B.getName(), AdditionMap.class,
         IntWritable.class, DenseVector.class, jobConf);
-    MatrixReduce.initJob(C.getName(), AdditionReduce.class, jobConf);
+    MatrixReduce.initJob(result.getName(), AdditionReduce.class, jobConf);
 
-    JobClient.runJob(jobConf);
-    return C;
+    execute(jobConf, result);
+    return result;
   }
 
   public Matrix add(double alpha, Matrix B) throws IOException {
@@ -173,17 +171,16 @@
 
   public Matrix mult(Matrix B) throws IOException {
     String output = RandomVariable.randMatrixName();
-    Matrix C = new DenseMatrix(config, output);
+    Matrix result = new DenseMatrix(config, output);
 
     JobConf jobConf = new JobConf(config);
-    jobConf.setJobName("multiplication MR job");
+    jobConf.setJobName("multiplication MR job : " + result.getName());
 
-    MultiplicationMap.initJob(this.getName(), B.getName(), MultiplicationMap.class,
-        IntWritable.class, DenseVector.class, jobConf);
-    MatrixReduce.initJob(C.getName(), MultiplicationReduce.class, jobConf);
-
-    JobClient.runJob(jobConf);
-    return C;
+    MultiplicationMap.initJob(this.getName(), B.getName(),
+        MultiplicationMap.class, IntWritable.class, DenseVector.class, jobConf);
+    MatrixReduce.initJob(result.getName(), MultiplicationReduce.class, jobConf);
+    execute(jobConf, result);
+    return result;
   }
 
   public Matrix multAdd(double alpha, Matrix B, Matrix C) throws IOException {

Modified: incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixInputFormat.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixInputFormat.java?rev=695149&r1=695148&r2=695149&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixInputFormat.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixInputFormat.java Sun Sep 14
01:56:17 2008
@@ -37,11 +37,8 @@
 
   /**
    * space delimited list of columns
-   * 
-   * @see org.apache.hadoop.hbase.regionserver.HAbstractScanner for column name
-   *      wildcards
    */
-  public static final String COLUMN_LIST = "hbase.mapred.tablecolumns";
+  public static final String COLUMN_LIST = "hama.mapred.tablecolumns";
 
   /** {@inheritDoc} */
   public void configure(JobConf job) {

Modified: incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixInputFormatBase.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixInputFormatBase.java?rev=695149&r1=695148&r2=695149&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixInputFormatBase.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixInputFormatBase.java Sun Sep
14 01:56:17 2008
@@ -32,7 +32,6 @@
 import org.apache.hadoop.hbase.io.Cell;
 import org.apache.hadoop.hbase.io.RowResult;
 import org.apache.hadoop.hbase.mapred.TableSplit;
-import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.util.Writables;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.mapred.InputFormat;

Modified: incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixOutputFormat.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixOutputFormat.java?rev=695149&r1=695148&r2=695149&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixOutputFormat.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixOutputFormat.java Sun Sep 14
01:56:17 2008
@@ -27,7 +27,6 @@
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.io.BatchUpdate;
-import org.apache.hadoop.hbase.mapred.TableOutputFormat;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.mapred.FileAlreadyExistsException;
 import org.apache.hadoop.mapred.FileOutputFormat;
@@ -41,8 +40,8 @@
     FileOutputFormat<IntWritable, BatchUpdate> {
 
   /** JobConf parameter that specifies the output table */
-  public static final String OUTPUT_TABLE = "hbase.mapred.outputtable";
-  private final Log LOG = LogFactory.getLog(TableOutputFormat.class);
+  public static final String OUTPUT_TABLE = "hama.mapred.output";
+  private final static Log LOG = LogFactory.getLog(MatrixOutputFormat.class);
 
   /**
    * Convert Reduce output (key, value) to (HStoreKey, KeyedDataArrayWritable)
@@ -63,7 +62,6 @@
 
     /** {@inheritDoc} */
     public void close(Reporter reporter) {
-      // TODO : Sets the (row, column) dimension
     }
 
     /** {@inheritDoc} */

Modified: incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java?rev=695149&r1=695148&r2=695149&view=diff
==============================================================================
--- incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java (original)
+++ incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java Sun Sep 14 01:56:17
2008
@@ -35,7 +35,7 @@
  */
 public class TestDenseMatrix extends TestCase {
   static final Logger LOG = Logger.getLogger(TestDenseMatrix.class);
-  private static int SIZE = 5;
+  private static int SIZE = 10;
   private static Matrix m1;
   private static Matrix m2;
 
@@ -80,6 +80,9 @@
   public void testMatrixAdd() throws IOException {
     Matrix result = m1.add(m2);
 
+    assertEquals(result.getRows(), SIZE);
+    assertEquals(result.getColumns(), SIZE);
+
     for (int i = 0; i < SIZE; i++) {
       for (int j = 0; j < SIZE; j++) {
         assertEquals(result.get(i, j), m1.get(i, j) + m2.get(i, j));
@@ -95,31 +98,36 @@
   public void testMatrixMult() throws IOException {
     Matrix result = m1.mult(m2);
 
-    verifyMultResult(SIZE, m1, m2, result);
+    assertEquals(result.getRows(), SIZE);
+    assertEquals(result.getColumns(), SIZE);
+
+    verifyMultResult(m1, m2, result);
   }
 
   /**
    * Verifying multiplication result
    * 
-   * @param size
    * @param m1
    * @param m2
    * @param result
    * @throws IOException
    */
-  private void verifyMultResult(int size, Matrix m1, Matrix m2, Matrix result)
+  private void verifyMultResult(Matrix m1, Matrix m2, Matrix result)
       throws IOException {
     double[][] C = new double[SIZE][SIZE];
 
-    for (int i = 0; i < SIZE; i++)
-      for (int j = 0; j < SIZE; j++)
-        for (int k = 0; k < SIZE; k++)
+    for (int i = 0; i < SIZE; i++) {
+      for (int j = 0; j < SIZE; j++) {
+        for (int k = 0; k < SIZE; k++) {
           C[i][k] += m1.get(i, j) * m2.get(j, k);
+        }
+      }
+    }
 
-    for (int i = 0; i < 2; i++) {
-      for (int j = 0; j < 2; j++) {
-        LOG.info("result: " + result.get(i, j) + ", C: " + C[i][j]);
-        assertEquals(result.get(i, j), C[i][j]);
+    for (int i = 0; i < SIZE; i++) {
+      for (int j = 0; j < SIZE; j++) {
+        assertEquals(String.valueOf(result.get(i, j)).substring(0, 14), 
+            String.valueOf(C[i][j]).substring(0, 14));
       }
     }
   }



Mime
View raw message