hama-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From edwardy...@apache.org
Subject svn commit: r781687 - in /incubator/hama/trunk: ./ src/examples/org/apache/hama/examples/ src/java/org/apache/hama/ src/java/org/apache/hama/mapred/ src/java/org/apache/hama/util/ src/test/org/apache/hama/mapred/ src/test/org/apache/hama/util/
Date Thu, 04 Jun 2009 08:53:00 GMT
Author: edwardyoon
Date: Thu Jun  4 08:53:00 2009
New Revision: 781687

URL: http://svn.apache.org/viewvc?rev=781687&view=rev
Log:
Add density option for random matrix

Modified:
    incubator/hama/trunk/CHANGES.txt
    incubator/hama/trunk/src/examples/org/apache/hama/examples/RandomMatrix.java
    incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java
    incubator/hama/trunk/src/java/org/apache/hama/SparseMatrix.java
    incubator/hama/trunk/src/java/org/apache/hama/mapred/RandomMatrixMap.java
    incubator/hama/trunk/src/java/org/apache/hama/util/RandomVariable.java
    incubator/hama/trunk/src/test/org/apache/hama/mapred/TestRandomMatrixMapReduce.java
    incubator/hama/trunk/src/test/org/apache/hama/util/TestRandomVariable.java

Modified: incubator/hama/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/CHANGES.txt?rev=781687&r1=781686&r2=781687&view=diff
==============================================================================
--- incubator/hama/trunk/CHANGES.txt (original)
+++ incubator/hama/trunk/CHANGES.txt Thu Jun  4 08:53:00 2009
@@ -38,6 +38,7 @@
 
   IMPROVEMENTS
 
+    HAMA-186: Add density option for random matrix (edwardyoon)
     HAMA-171: Implementation of Matrix.Norm (samuel)
     HAMA-178: Add example of norm (edwardyoon)
     HAMA-158: Implementation of random sparse matrix (edwardyoon)

Modified: incubator/hama/trunk/src/examples/org/apache/hama/examples/RandomMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/examples/org/apache/hama/examples/RandomMatrix.java?rev=781687&r1=781686&r2=781687&view=diff
==============================================================================
--- incubator/hama/trunk/src/examples/org/apache/hama/examples/RandomMatrix.java (original)
+++ incubator/hama/trunk/src/examples/org/apache/hama/examples/RandomMatrix.java Thu Jun 
4 08:53:00 2009
@@ -30,7 +30,9 @@
   public static void main(String[] args) throws IOException {
     if (args.length < 3) {
       System.out
-          .println("rand [-m maps] [-r reduces] <rows> <columns> <sparse |
dense> <matrix_name>");
+          .println("rand [-m maps] [-r reduces] <rows> <columns> <density>
<matrix_name>");
+      System.out
+      .println("ex) rand -m 10 -r 10 2000 2000 30% matrixA");
       System.exit(-1);
     } else {
       parseArgs(args);
@@ -38,12 +40,13 @@
 
     int row = Integer.parseInt(ARGS.get(0));
     int column = Integer.parseInt(ARGS.get(1));
-
+    int percent = Integer.parseInt(ARGS.get(2).substring(0, ARGS.get(2).length()-1));
+    
     Matrix a;
-    if(ARGS.get(2).equals("sparse"))
-      a = SparseMatrix.random_mapred(conf, row, column);
-    else
+    if(percent == 100)
       a = DenseMatrix.random_mapred(conf, row, column);
+    else
+      a = SparseMatrix.random_mapred(conf, row, column, percent);
     
     a.save(ARGS.get(3));
   }

Modified: incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java?rev=781687&r1=781686&r2=781687&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java Thu Jun  4 08:53:00 2009
@@ -243,9 +243,10 @@
     RandomMatrixReduce.initJob(rand.getPath(), RandomMatrixReduce.class,
         jobConf);
     jobConf.setSpeculativeExecution(false);
-    jobConf.set("matrix.column", String.valueOf(n));
+    jobConf.setInt("matrix.column", n);
     jobConf.set("matrix.type", TABLE_PREFIX);
-
+    jobConf.setInt("matrix.density", 100);
+    
     jobConf.setInputFormat(SequenceFileInputFormat.class);
     final FileSystem fs = FileSystem.get(jobConf);
     int interval = m / conf.getNumMapTasks();

Modified: incubator/hama/trunk/src/java/org/apache/hama/SparseMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/SparseMatrix.java?rev=781687&r1=781686&r2=781687&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/SparseMatrix.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/SparseMatrix.java Thu Jun  4 08:53:00 2009
@@ -109,7 +109,7 @@
     return rand;
   }
   
-  public static SparseMatrix random_mapred(HamaConfiguration conf, int m, int n) throws IOException
{
+  public static SparseMatrix random_mapred(HamaConfiguration conf, int m, int n, int percent)
throws IOException {
     SparseMatrix rand = new SparseMatrix(conf, m, n);
     LOG.info("Create the " + m + " * " + n + " random matrix : "
         + rand.getPath());
@@ -129,8 +129,9 @@
     RandomMatrixReduce.initJob(rand.getPath(), RandomMatrixReduce.class,
         jobConf);
     jobConf.setSpeculativeExecution(false);
-    jobConf.set("matrix.column", String.valueOf(n));
+    jobConf.setInt("matrix.column", n);
     jobConf.set("matrix.type", TABLE_PREFIX);
+    jobConf.setInt("matrix.density", percent);
 
     jobConf.setInputFormat(SequenceFileInputFormat.class);
     final FileSystem fs = FileSystem.get(jobConf);

Modified: incubator/hama/trunk/src/java/org/apache/hama/mapred/RandomMatrixMap.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/mapred/RandomMatrixMap.java?rev=781687&r1=781686&r2=781687&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/mapred/RandomMatrixMap.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/mapred/RandomMatrixMap.java Thu Jun  4 08:53:00
2009
@@ -20,7 +20,6 @@
 package org.apache.hama.mapred;
 
 import java.io.IOException;
-import java.util.Random;
 
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.MapWritable;
@@ -41,7 +40,7 @@
 public class RandomMatrixMap extends MapReduceBase implements
     Mapper<IntWritable, IntWritable, IntWritable, MapWritable> {
   static final Logger LOG = Logger.getLogger(RandomMatrixMap.class);
-  protected int column;
+  protected int column, density;
   protected String type;
   protected Vector vector = new DenseVector();
 
@@ -53,9 +52,7 @@
       ((SparseVector) vector).clear();
       for (int i = key.get(); i <= value.get(); i++) {
         for (int j = 0; j < column; j++) {
-          Random r = new Random(); 
-          if(r.nextInt(2) != 0)
-            ((SparseVector) vector).set(j, RandomVariable.rand());
+            ((SparseVector) vector).set(j, RandomVariable.rand(density));
         }
         output.collect(new IntWritable(i), vector.getEntries());
       }
@@ -71,7 +68,8 @@
   }
 
   public void configure(JobConf job) {
-    column = Integer.parseInt(job.get("matrix.column"));
+    column = job.getInt("matrix.column", 0);
+    density = job.getInt("matrix.density", 100);
     type = job.get("matrix.type");
     if (type.equals("SparseMatrix"))
       vector = new SparseVector();

Modified: incubator/hama/trunk/src/java/org/apache/hama/util/RandomVariable.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/util/RandomVariable.java?rev=781687&r1=781686&r2=781687&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/util/RandomVariable.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/util/RandomVariable.java Thu Jun  4 08:53:00
2009
@@ -26,7 +26,7 @@
  * numbers.
  */
 public class RandomVariable {
-
+  
   /**
    * Generate a random number between 0 and 1.
    * 
@@ -38,6 +38,23 @@
   }
 
   /**
+   * Generate a random number
+   * 
+   * @param density percentage of the non-zero
+   * @return a double 0 or between 0 and 1
+   */
+  public static double rand(int density) {
+    int[] prob = new int[100];
+    double x = Math.random();
+    
+    for(int i = 0; i < density; i++) {
+      prob[i] = 1;
+    }
+
+    return (prob[randInt(0, 99)] == 0) ? 0 : x;
+  }
+
+  /**
    * Generate a random integer.
    * 
    * @param i0 min of the random variable.

Modified: incubator/hama/trunk/src/test/org/apache/hama/mapred/TestRandomMatrixMapReduce.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/test/org/apache/hama/mapred/TestRandomMatrixMapReduce.java?rev=781687&r1=781686&r2=781687&view=diff
==============================================================================
--- incubator/hama/trunk/src/test/org/apache/hama/mapred/TestRandomMatrixMapReduce.java (original)
+++ incubator/hama/trunk/src/test/org/apache/hama/mapred/TestRandomMatrixMapReduce.java Thu
Jun  4 08:53:00 2009
@@ -42,7 +42,7 @@
     
     rand.close();
     
-    SparseMatrix rand2 = SparseMatrix.random_mapred(conf, 20, 20);
+    SparseMatrix rand2 = SparseMatrix.random_mapred(conf, 20, 20, 30);
     assertEquals(20, rand2.getRows());
     assertEquals(20, rand2.getColumns());
     boolean zeroAppear = false;

Modified: incubator/hama/trunk/src/test/org/apache/hama/util/TestRandomVariable.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/test/org/apache/hama/util/TestRandomVariable.java?rev=781687&r1=781686&r2=781687&view=diff
==============================================================================
--- incubator/hama/trunk/src/test/org/apache/hama/util/TestRandomVariable.java (original)
+++ incubator/hama/trunk/src/test/org/apache/hama/util/TestRandomVariable.java Thu Jun  4
08:53:00 2009
@@ -19,12 +19,15 @@
  */
 package org.apache.hama.util;
 
+import org.apache.log4j.Logger;
+
 import junit.framework.TestCase;
 
 /**
  * Random variable generation test
  */
 public class TestRandomVariable extends TestCase {
+  static final Logger LOG = Logger.getLogger(TestRandomVariable.class);
   final static int COUNT = 50;
 
   /**
@@ -37,6 +40,13 @@
       double result = RandomVariable.rand();
       assertTrue(result >= 0.0d && result <= 1.0);
     }
+    
+    int nonZero = 0;
+    for (int i = 0; i < COUNT; i++) {
+      if(RandomVariable.rand(70) > 0)
+        nonZero++;
+    }
+    assertTrue((COUNT/2) < nonZero);
   }
 
   /**



Mime
View raw message