hama-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From edwardy...@apache.org
Subject svn commit: r727641 - in /incubator/hama/trunk: lib/ src/java/org/apache/hama/mapred/
Date Thu, 18 Dec 2008 04:57:25 GMT
Author: edwardyoon
Date: Wed Dec 17 20:57:24 2008
New Revision: 727641

URL: http://svn.apache.org/viewvc?rev=727641&view=rev
Log:
Update hadoop/hbase 0.19 in lib folder

Added:
    incubator/hama/trunk/lib/hadoop-0.19.0-core.jar   (with props)
    incubator/hama/trunk/lib/hadoop-0.19.0-test.jar   (with props)
    incubator/hama/trunk/lib/hbase-0.19.0-dev-2008-12-18.jar   (with props)
    incubator/hama/trunk/lib/hbase-0.19.0-dev-test--2008-12-18.jar   (with props)
Removed:
    incubator/hama/trunk/lib/hadoop-0.18.2-core.jar
    incubator/hama/trunk/lib/hadoop-0.18.2-test.jar
    incubator/hama/trunk/lib/hbase-0.18.1-test.jar
    incubator/hama/trunk/lib/hbase-0.18.1.jar
Modified:
    incubator/hama/trunk/src/java/org/apache/hama/mapred/TableInputFormatBase.java

Added: incubator/hama/trunk/lib/hadoop-0.19.0-core.jar
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/lib/hadoop-0.19.0-core.jar?rev=727641&view=auto
==============================================================================
Binary file - no diff available.

Propchange: incubator/hama/trunk/lib/hadoop-0.19.0-core.jar
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Added: incubator/hama/trunk/lib/hadoop-0.19.0-test.jar
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/lib/hadoop-0.19.0-test.jar?rev=727641&view=auto
==============================================================================
Binary file - no diff available.

Propchange: incubator/hama/trunk/lib/hadoop-0.19.0-test.jar
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Added: incubator/hama/trunk/lib/hbase-0.19.0-dev-2008-12-18.jar
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/lib/hbase-0.19.0-dev-2008-12-18.jar?rev=727641&view=auto
==============================================================================
Binary file - no diff available.

Propchange: incubator/hama/trunk/lib/hbase-0.19.0-dev-2008-12-18.jar
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Added: incubator/hama/trunk/lib/hbase-0.19.0-dev-test--2008-12-18.jar
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/lib/hbase-0.19.0-dev-test--2008-12-18.jar?rev=727641&view=auto
==============================================================================
Binary file - no diff available.

Propchange: incubator/hama/trunk/lib/hbase-0.19.0-dev-test--2008-12-18.jar
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Modified: incubator/hama/trunk/src/java/org/apache/hama/mapred/TableInputFormatBase.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/mapred/TableInputFormatBase.java?rev=727641&r1=727640&r2=727641&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/mapred/TableInputFormatBase.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/mapred/TableInputFormatBase.java Wed Dec
17 20:57:24 2008
@@ -28,14 +28,11 @@
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.filter.RowFilterInterface;
-import org.apache.hadoop.hbase.io.Cell;
 import org.apache.hadoop.hbase.mapred.TableSplit;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.mapred.FileInputFormat;
 import org.apache.hadoop.mapred.InputSplit;
 import org.apache.hadoop.mapred.JobConf;
-import org.apache.hama.Constants;
-import org.apache.hama.util.BytesUtil;
 
 public abstract class TableInputFormatBase {
   private static final Log LOG = LogFactory.getLog(TableInputFormatBase.class);
@@ -79,24 +76,31 @@
   }
 
   public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException {
-    Cell meta = this.table.get(Constants.METADATA, Constants.METADATA_ROWS);
-
-    if (BytesUtil.bytesToInt(meta.getValue()) < numSplits) {
-      numSplits = BytesUtil.bytesToInt(meta.getValue());
-    }
-
-    int[] startKeys = new int[numSplits];
-    int interval = BytesUtil.bytesToInt(meta.getValue()) / numSplits;
-
-    for (int i = 0; i < numSplits; i++) {
-      startKeys[i] = (i * interval);
-    }
-
-    InputSplit[] splits = new InputSplit[startKeys.length];
-    for (int i = 0; i < startKeys.length; i++) {
-      splits[i] = new TableSplit(this.table.getTableName(), BytesUtil
-          .intToBytes(startKeys[i]), ((i + 1) < startKeys.length) ? BytesUtil
-          .intToBytes(startKeys[i + 1]) : HConstants.EMPTY_START_ROW);
+    byte [][] startKeys = this.table.getStartKeys();
+    if (startKeys == null || startKeys.length == 0) {
+      throw new IOException("Expecting at least one region");
+    }
+    if (this.table == null) {
+      throw new IOException("No table was provided");
+    }
+    if (this.inputColumns == null || this.inputColumns.length == 0) {
+      throw new IOException("Expecting at least one column");
+    }
+    int realNumSplits = numSplits > startKeys.length? startKeys.length:
+      numSplits;
+    InputSplit[] splits = new InputSplit[realNumSplits];
+    int middle = startKeys.length / realNumSplits;
+    int startPos = 0;
+    for (int i = 0; i < realNumSplits; i++) {
+      int lastPos = startPos + middle;
+      lastPos = startKeys.length % realNumSplits > i ? lastPos + 1 : lastPos;
+      String regionLocation = table.getRegionLocation(startKeys[startPos]).
+        getServerAddress().getHostname(); 
+      splits[i] = new TableSplit(this.table.getTableName(),
+        startKeys[startPos], ((i + 1) < realNumSplits) ? startKeys[lastPos]:
+          HConstants.EMPTY_START_ROW, regionLocation);
+      LOG.info("split: " + i + "->" + splits[i]);
+      startPos = lastPos;
     }
     return splits;
   }



Mime
View raw message