hama-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From edwardy...@apache.org
Subject svn commit: r696609 - in /incubator/hama/trunk: ./ src/java/org/apache/hama/ src/test/org/apache/hama/
Date Thu, 18 Sep 2008 09:15:16 GMT
Author: edwardyoon
Date: Thu Sep 18 02:15:15 2008
New Revision: 696609

URL: http://svn.apache.org/viewvc?rev=696609&view=rev
Log:
Load/Save methods

Added:
    incubator/hama/trunk/src/java/org/apache/hama/HamaAdmin.java
Modified:
    incubator/hama/trunk/CHANGES.txt
    incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java
    incubator/hama/trunk/src/java/org/apache/hama/Constants.java
    incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java
    incubator/hama/trunk/src/java/org/apache/hama/Matrix.java
    incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java

Modified: incubator/hama/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/CHANGES.txt?rev=696609&r1=696608&r2=696609&view=diff
==============================================================================
--- incubator/hama/trunk/CHANGES.txt (original)
+++ incubator/hama/trunk/CHANGES.txt Thu Sep 18 02:15:15 2008
@@ -4,6 +4,7 @@
 
   NEW FEATURES
     
+    HAMA-61: Load / save matrices from HTable (edwardyoon)
     HAMA-51: Add get/setRowAttribute() method (edwardyoon)
     HAMA-13: Matrix multiplication (edwardyoon)
     HAMA-48: Add getColumn(int column) method which returns column vector (edwardyoon)

Modified: incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java?rev=696609&r1=696608&r2=696609&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java Thu Sep 18 02:15:15
2008
@@ -51,7 +51,8 @@
   protected HTable table;
   /** Matrix attribute description */
   protected HTableDescriptor tableDesc;
-
+  public HamaAdmin hAdmin;
+  
   /**
    * Sets the job configuration
    * 
@@ -64,13 +65,17 @@
     } catch (MasterNotRunningException e) {
       LOG.error(e, e);
     }
+    
+    hAdmin = new HamaAdmin(conf, admin);
   }
 
   /**
    * Create matrix space
    */
   protected void create() throws IOException {
+    this.tableDesc.addFamily(new HColumnDescriptor(Constants.COLUMN));
     this.tableDesc.addFamily(new HColumnDescriptor(Constants.ATTRIBUTE));
+    
     LOG.info("Initializing the matrix storage.");
     this.admin.createTable(this.tableDesc);
   }
@@ -161,4 +166,12 @@
   public String getName() {
     return (matrixName != null) ? matrixName : null;
   }
+  
+  public void clear() throws IOException {
+    admin.deleteTable(matrixName);
+  }
+
+  public boolean save(String path) throws IOException {
+    return hAdmin.put(this.matrixName, path);
+  }
 }

Modified: incubator/hama/trunk/src/java/org/apache/hama/Constants.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/Constants.java?rev=696609&r1=696608&r2=696609&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/Constants.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/Constants.java Thu Sep 18 02:15:15 2008
@@ -63,4 +63,14 @@
    * Temporary random matrices name prefix 
    */
   public final static String RANDOM = "rand";
+  
+  /**
+   * Admin table name
+   */
+  public final static String ADMINTABLE = "admin.table";
+
+  /**
+   * Matrix path columnFamily
+   */
+  public static final String PATHCOLUMN = "path:";
 }

Modified: incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java?rev=696609&r1=696608&r2=696609&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java Thu Sep 18 02:15:15 2008
@@ -64,7 +64,6 @@
 
       if (!admin.tableExists(matrixName)) {
         tableDesc = new HTableDescriptor(matrixName);
-        tableDesc.addFamily(new HColumnDescriptor(Constants.COLUMN));
         create();
       }
 
@@ -215,4 +214,9 @@
   public void setColumn(int column, Vector vector) throws IOException {
     // TODO Auto-generated method stub
   }
+
+  public void load(String path) throws IOException {
+    matrixName = hAdmin.get(path);
+    table = new HTable(matrixName);
+  }
 }

Added: incubator/hama/trunk/src/java/org/apache/hama/HamaAdmin.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/HamaAdmin.java?rev=696609&view=auto
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/HamaAdmin.java (added)
+++ incubator/hama/trunk/src/java/org/apache/hama/HamaAdmin.java Thu Sep 18 02:15:15 2008
@@ -0,0 +1,88 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.io.BatchUpdate;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Admin tool
+ */
+public class HamaAdmin {
+  public HamaConfiguration conf;
+  public HBaseAdmin admin;
+  public HTable table;
+
+  public HamaAdmin(HamaConfiguration conf) {
+    this.conf = conf;
+    initialJob();
+  }
+
+  public HamaAdmin(HamaConfiguration conf, HBaseAdmin admin) {
+    this.conf = conf;
+    this.admin = admin;
+    initialJob();
+  }
+
+  public void initialJob() {
+    try {
+      if (!admin.tableExists(Constants.ADMINTABLE)) {
+        HTableDescriptor tableDesc = new HTableDescriptor(Constants.ADMINTABLE);
+        tableDesc.addFamily(new HColumnDescriptor(Constants.PATHCOLUMN));
+        admin.createTable(tableDesc);
+      }
+
+      table = new HTable(conf, Constants.ADMINTABLE);
+    } catch (Exception e) {
+      e.printStackTrace();
+    }
+  }
+
+  public boolean put(String matrixName, String path) {
+    boolean result = false;
+
+    BatchUpdate update = new BatchUpdate(path);
+    update.put(Constants.PATHCOLUMN, Bytes.toBytes(matrixName));
+    try {
+      table.commit(update);
+      result = true;
+    } catch (IOException e) {
+      e.printStackTrace();
+    }
+
+    return result;
+  }
+
+  public String get(String path) {
+    try {
+      byte[] result = table.get(path, Constants.PATHCOLUMN).getValue();
+      return Bytes.toString(result);
+    } catch (IOException e) {
+      e.printStackTrace();
+      return null;
+    }
+  }
+}

Modified: incubator/hama/trunk/src/java/org/apache/hama/Matrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/Matrix.java?rev=696609&r1=696608&r2=696609&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/Matrix.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/Matrix.java Thu Sep 18 02:15:15 2008
@@ -32,7 +32,7 @@
    * @param i ith row of the matrix
    * @param j jth column of the matrix
    * @return the value of entry
-   * @throws IOException 
+   * @throws IOException
    */
   public double get(int i, int j) throws IOException;
 
@@ -58,7 +58,7 @@
    * Get the number of row of the matrix from the meta-data column
    * 
    * @return a number of rows of the matrix
-   * @throws IOException 
+   * @throws IOException
    */
   public int getRows() throws IOException;
 
@@ -66,7 +66,7 @@
    * Get the number of column of the matrix from the meta-data column
    * 
    * @return a number of columns of the matrix
-   * @throws IOException 
+   * @throws IOException
    */
   public int getColumns() throws IOException;
 
@@ -76,23 +76,30 @@
    * @throws IOException
    */
   public String getRowAttribute(int i) throws IOException;
-  
+
   /**
-   * Sets the attribute of the row
+   * Gets the attribute of the column
    * 
-   * @param i
-   * @param name
    * @throws IOException
    */
-  public void setRowAttribute(int i, String name) throws IOException;
-  
+  public String getColumnAttribute(int j) throws IOException;
+
   /**
-   * Gets the attribute of the column
+   * Return the matrix name
    * 
+   * @return the name of the matrix
+   */
+  public String getName();
+
+  /**
+   * Sets the attribute of the row
+   * 
+   * @param i
+   * @param name
    * @throws IOException
    */
-  public String getColumnAttribute(int j) throws IOException;
-  
+  public void setRowAttribute(int i, String name) throws IOException;
+
   /**
    * Sets the attribute of the column
    * 
@@ -101,14 +108,14 @@
    * @throws IOException
    */
   public void setColumnAttribute(int j, String name) throws IOException;
-  
+
   /**
    * Sets the double value of (i, j)
    * 
    * @param i ith row of the matrix
    * @param j jth column of the matrix
    * @param value the value of entry
-   * @throws IOException 
+   * @throws IOException
    */
   public void set(int i, int j, double value) throws IOException;
 
@@ -118,7 +125,7 @@
    * @param alpha
    * @param B
    * @return A
-   * @throws IOException 
+   * @throws IOException
    */
   public Matrix set(double alpha, Matrix B) throws IOException;
 
@@ -127,7 +134,7 @@
    * 
    * @param B
    * @return A
-   * @throws IOException 
+   * @throws IOException
    */
   public Matrix set(Matrix B) throws IOException;
 
@@ -139,7 +146,7 @@
    * @throws IOException
    */
   public void setRow(int row, Vector vector) throws IOException;
-  
+
   /**
    * Set the column of a matrix to a given vector
    * 
@@ -148,7 +155,7 @@
    * @throws IOException
    */
   public void setColumn(int column, Vector vector) throws IOException;
-  
+
   /**
    * Sets the dimension of matrix
    * 
@@ -164,7 +171,7 @@
    * @param i
    * @param j
    * @param value
-   * @throws IOException 
+   * @throws IOException
    */
   public void add(int i, int j, double value) throws IOException;
 
@@ -173,7 +180,7 @@
    * 
    * @param B
    * @return A
-   * @throws IOException 
+   * @throws IOException
    */
   public Matrix add(Matrix B) throws IOException;
 
@@ -192,7 +199,7 @@
    * 
    * @param B
    * @return C
-   * @throws IOException 
+   * @throws IOException
    */
   public Matrix mult(Matrix B) throws IOException;
 
@@ -203,7 +210,7 @@
    * @param B
    * @param C
    * @return C
-   * @throws IOException 
+   * @throws IOException
    */
   public Matrix multAdd(double alpha, Matrix B, Matrix C) throws IOException;
 
@@ -212,7 +219,7 @@
    * 
    * @param type
    * @return norm of the matrix
-   * @throws IOException 
+   * @throws IOException
    */
   public double norm(Norm type) throws IOException;
 
@@ -223,11 +230,28 @@
     /** Largest entry in absolute value */
     Infinity
   }
-  
+
   /**
-   * Return the matrix name
+   * Save to a table or file
    * 
-   * @return the name of the matrix
+   * @param path
+   * @return true if saved
+   * @throws IOException
    */
-  public String getName();
+  public boolean save(String path) throws IOException;
+
+  /**
+   * Load the matrix from a table or file
+   * 
+   * @param path
+   * @throws IOException
+   */
+  public void load(String path) throws IOException;
+
+  /**
+   * Clear current matrix.
+   * 
+   * @throws Exception
+   */
+  public void clear() throws IOException;
 }

Modified: incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java?rev=696609&r1=696608&r2=696609&view=diff
==============================================================================
--- incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java (original)
+++ incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java Thu Sep 18 02:15:15
2008
@@ -50,12 +50,21 @@
       }
 
       protected void tearDown() {
-        LOG.info("tearDown()");
+        try {
+          clearTest();
+        } catch (IOException e) {
+          e.printStackTrace();
+        }
       }
     };
     return setup;
   }
 
+  public static void clearTest() throws IOException {
+    m1.clear();
+    m2.clear();
+  }
+
   /**
    * Column vector test.
    * 
@@ -76,12 +85,12 @@
     m1.setRowAttribute(0, "row1");
     assertEquals(m1.getRowAttribute(0), "row1");
     assertEquals(m1.getRowAttribute(1), null);
-    
+
     m1.setColumnAttribute(0, "column1");
     assertEquals(m1.getColumnAttribute(0), "column1");
     assertEquals(m1.getColumnAttribute(1), null);
   }
-  
+
   /**
    * Test matrices addition
    * 
@@ -133,6 +142,19 @@
     }
   }
 
+  public void testLoadSave() throws IOException {
+    m1.save("udanax");
+    HCluster hCluster = new HCluster();
+    DenseMatrix loadTest = new DenseMatrix(hCluster.conf);
+    loadTest.load("udanax");
+
+    for (int i = 0; i < loadTest.getRows(); i++) {
+      for (int j = 0; j < loadTest.getColumns(); j++) {
+        assertEquals(m1.get(i, j), loadTest.get(i, j));
+      }
+    }
+  }
+
   /**
    * Verifying multiplication result
    * 



Mime
View raw message