hama-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From edwardy...@apache.org
Subject svn commit: r725509 - in /incubator/hama/trunk/src: examples/org/apache/hama/examples/MatrixMultiplication.java java/org/apache/hama/DenseMatrix.java test/org/apache/hama/TestDenseMatrix.java test/org/apache/hama/mapred/TestBlockMatrixMapReduce.java
Date Thu, 11 Dec 2008 00:59:08 GMT
Author: edwardyoon
Date: Wed Dec 10 16:59:05 2008
New Revision: 725509

URL: http://svn.apache.org/viewvc?rev=725509&view=rev
Log:
Add argument for the number of blocks

Modified:
    incubator/hama/trunk/src/examples/org/apache/hama/examples/MatrixMultiplication.java
    incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java
    incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java
    incubator/hama/trunk/src/test/org/apache/hama/mapred/TestBlockMatrixMapReduce.java

Modified: incubator/hama/trunk/src/examples/org/apache/hama/examples/MatrixMultiplication.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/examples/org/apache/hama/examples/MatrixMultiplication.java?rev=725509&r1=725508&r2=725509&view=diff
==============================================================================
--- incubator/hama/trunk/src/examples/org/apache/hama/examples/MatrixMultiplication.java (original)
+++ incubator/hama/trunk/src/examples/org/apache/hama/examples/MatrixMultiplication.java Wed
Dec 10 16:59:05 2008
@@ -27,7 +27,7 @@
 public class MatrixMultiplication extends AbstractExample {
   public static void main(String[] args) throws IOException {
     if (args.length < 2) {
-      System.out.println("mult  [-m maps] [-r reduces] <matrix_A> <matrix_B>");
+      System.out.println("mult  [-m maps] [-r reduces] <matrix_A> <matrix_B>
[blocks]");
       System.exit(-1);
     } else {
       parseArgs(args);
@@ -39,11 +39,11 @@
     DenseMatrix a = new DenseMatrix(conf, matrixA, false);
     DenseMatrix b = new DenseMatrix(conf, matrixB, false);
     
-    if (!a.isBlocked())
-      a.blocking_mapred(conf.getNumMapTasks());
-    if (!b.isBlocked())
-      b.blocking_mapred(conf.getNumMapTasks());
-
+    if(ARGS.size() > 2) {
+      a.blocking_mapred(Integer.parseInt(ARGS.get(2)));
+      b.blocking_mapred(Integer.parseInt(ARGS.get(2)));
+    }
+    
     Matrix c = a.mult(b);
     c.close();
   }

Modified: incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java?rev=725509&r1=725508&r2=725509&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java Wed Dec 10 16:59:05 2008
@@ -464,28 +464,6 @@
     return result;
   }
 
-  /**
-   * Using a map/reduce job to block a dense matrix.
-   * 
-   * @param blockNum
-   * @throws IOException
-   */
-  public void blocking_mapred(int blockNum) throws IOException {
-    setBlockPosition(blockNum);
-    setBlockSize(blockNum);
-    LOG.info("Convert to " + blockNum + " * " + blockNum + " blocked matrix");
-    
-    JobConf jobConf = new JobConf(config);
-    jobConf.setJobName("Blocking MR job" + getPath());
-
-    jobConf.setNumMapTasks(config.getNumMapTasks());
-    jobConf.setNumReduceTasks(config.getNumReduceTasks());
-
-    BlockingMapRed.initJob(getPath(), jobConf);
-
-    JobManager.execute(jobConf);
-  }
-
   public boolean isBlocked() throws IOException {
     return (table.get(Constants.METADATA, Constants.BLOCK_SIZE) == null) ? false
         : true;
@@ -557,6 +535,26 @@
   }
 
   /**
+   * Using a map/reduce job to block a dense matrix.
+   * 
+   * @param blockNum
+   * @throws IOException
+   */
+  public void blocking_mapred(int blockNum) throws IOException {
+    this.checkBlockNum(blockNum);
+    
+    JobConf jobConf = new JobConf(config);
+    jobConf.setJobName("Blocking MR job" + getPath());
+
+    jobConf.setNumMapTasks(config.getNumMapTasks());
+    jobConf.setNumReduceTasks(config.getNumReduceTasks());
+
+    BlockingMapRed.initJob(getPath(), jobConf);
+
+    JobManager.execute(jobConf);
+  }
+
+  /**
    * Using a scanner to block a dense matrix. If the matrix is large, use the
    * blocking_mapred()
    * 
@@ -564,9 +562,8 @@
    * @throws IOException
    */
   public void blocking(int blockNum) throws IOException {
-    setBlockPosition(blockNum);
-    setBlockSize(blockNum);
-
+    this.checkBlockNum(blockNum);
+    
     String[] columns = new String[] { Constants.BLOCK_STARTROW,
         Constants.BLOCK_ENDROW, Constants.BLOCK_STARTCOLUMN,
         Constants.BLOCK_ENDCOLUMN };
@@ -580,4 +577,16 @@
       setBlock(blockR, blockC, subMatrix(pos[0], pos[1], pos[2], pos[3]));
     }
   }
+  
+  private void checkBlockNum(int blockNum) throws IOException {
+    double blocks = Math.pow(blockNum, 0.5);
+    // TODO: Check also it is validation with matrix.
+    if(!String.valueOf(blocks).endsWith(".0"))
+      throw new IOException("can't divide.");
+    
+    int block_size = (int) blocks;  
+    setBlockPosition(block_size);
+    setBlockSize(block_size);
+    LOG.info("Create " + block_size + " * " + block_size + " blocked matrix");
+  }
 }

Modified: incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java?rev=725509&r1=725508&r2=725509&view=diff
==============================================================================
--- incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java (original)
+++ incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java Wed Dec 10 16:59:05
2008
@@ -84,7 +84,7 @@
   
   public void testBlocking() throws IOException, ClassNotFoundException {
     assertEquals(((DenseMatrix) m1).isBlocked(), false);
-    ((DenseMatrix) m1).blocking(2);
+    ((DenseMatrix) m1).blocking(4);
     assertEquals(((DenseMatrix) m1).isBlocked(), true);
     int[] pos = ((DenseMatrix) m1).getBlockPosition(1, 0);
     double[][] b = ((DenseMatrix) m1).subMatrix(pos[0], pos[1], pos[2], pos[3]).getDoubleArray();
@@ -107,7 +107,7 @@
    */
   public void testMRBlocking() throws IOException, ClassNotFoundException {
     assertEquals(((DenseMatrix) m2).isBlocked(), false);
-    ((DenseMatrix) m2).blocking_mapred(2);
+    ((DenseMatrix) m2).blocking_mapred(4);
     assertEquals(((DenseMatrix) m2).isBlocked(), true);
     int[] pos = ((DenseMatrix) m2).getBlockPosition(1, 0);
     double[][] b = ((DenseMatrix) m2).subMatrix(pos[0], pos[1], pos[2], pos[3]).getDoubleArray();

Modified: incubator/hama/trunk/src/test/org/apache/hama/mapred/TestBlockMatrixMapReduce.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/test/org/apache/hama/mapred/TestBlockMatrixMapReduce.java?rev=725509&r1=725508&r2=725509&view=diff
==============================================================================
--- incubator/hama/trunk/src/test/org/apache/hama/mapred/TestBlockMatrixMapReduce.java (original)
+++ incubator/hama/trunk/src/test/org/apache/hama/mapred/TestBlockMatrixMapReduce.java Wed
Dec 10 16:59:05 2008
@@ -25,8 +25,8 @@
   public void testBlockMatrixMapReduce() throws IOException, ClassNotFoundException {
     Matrix m1 = DenseMatrix.random(conf, SIZE, SIZE);
     Matrix m2 = DenseMatrix.random(conf, SIZE, SIZE);
-    ((DenseMatrix) m1).blocking_mapred(2);
-    ((DenseMatrix) m2).blocking_mapred(2);
+    ((DenseMatrix) m1).blocking_mapred(4);
+    ((DenseMatrix) m2).blocking_mapred(4);
 
     miniMRJob(m1.getPath(), m2.getPath());
     



Mime
View raw message