pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From knogu...@apache.org
Subject svn commit: r1842134 - in /pig/trunk: CHANGES.txt src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigRecordReader.java src/org/apache/pig/backend/hadoop/hbase/HBaseTableInputFormat.java
Date Thu, 27 Sep 2018 18:10:45 GMT
Author: knoguchi
Date: Thu Sep 27 18:10:44 2018
New Revision: 1842134

URL: http://svn.apache.org/viewvc?rev=1842134&view=rev
Log:
PIG-5355: Negative progress report by HBaseTableRecordReader (satishsaley via knoguchi)

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigRecordReader.java
    pig/trunk/src/org/apache/pig/backend/hadoop/hbase/HBaseTableInputFormat.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1842134&r1=1842133&r2=1842134&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Thu Sep 27 18:10:44 2018
@@ -78,6 +78,8 @@ OPTIMIZATIONS
  
 BUG FIXES
 
+PIG-5355: Negative progress report by HBaseTableRecordReader (satishsaley via knoguchi)
+
 PIG-5341: PigStorage with -tagFile/-tagPath produces incorrect results with column pruning
(knoguchi)
 
 PIG-5335: Error message from range projection completely misleading (knoguchi)

Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigRecordReader.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigRecordReader.java?rev=1842134&r1=1842133&r2=1842134&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigRecordReader.java
(original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigRecordReader.java
Thu Sep 27 18:10:44 2018
@@ -162,7 +162,7 @@ public class PigRecordReader extends Rec
             // idx is always one past the current subsplit's true index.
             subprogress = (long)(curReader.getProgress() * pigSplit.getLength(idx - 1));
         }
-        return Math.min(1.0f,  (progress + subprogress)/(float)(pigSplit.getLength()));
+        return Math.max(0.0f, Math.min(1.0f,  (progress + subprogress)/(float)(pigSplit.getLength())));
     }
 
     @Override

Modified: pig/trunk/src/org/apache/pig/backend/hadoop/hbase/HBaseTableInputFormat.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/hbase/HBaseTableInputFormat.java?rev=1842134&r1=1842133&r2=1842134&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/hbase/HBaseTableInputFormat.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/hbase/HBaseTableInputFormat.java Thu Sep 27
18:10:44 2018
@@ -118,7 +118,7 @@ public class HBaseTableInputFormat exten
         private byte[] startRow_;
         private byte[] endRow_;
         private transient byte[] currRow_;
-
+        private int maxRowLength;
         private BigInteger bigStart_;
         private BigInteger bigEnd_;
         private BigDecimal bigRange_;
@@ -151,6 +151,7 @@ public class HBaseTableInputFormat exten
             bigStart_ = new BigInteger(Bytes.add(prependHeader, startPadded));
             bigEnd_ = new BigInteger(Bytes.add(prependHeader, endPadded));
             bigRange_ = new BigDecimal(bigEnd_.subtract(bigStart_));
+            maxRowLength = endRow_.length > startRow_.length ? endRow_.length : startRow_.length;
             LOG.info("setScan with ranges: " + bigStart_ + " - " + bigEnd_ + " ( " + bigRange_
+ ")");
         }
 
@@ -173,11 +174,8 @@ public class HBaseTableInputFormat exten
                 return 0;
             }
             byte[] lastPadded = currRow_;
-            if (currRow_.length < endRow_.length) {
-                lastPadded = Bytes.padTail(currRow_, endRow_.length - currRow_.length);
-            }
-            if (currRow_.length < startRow_.length) {
-                lastPadded = Bytes.padTail(currRow_, startRow_.length - currRow_.length);
+            if(maxRowLength > currRow_.length) {
+                lastPadded = Bytes.padTail(currRow_, maxRowLength - currRow_.length);
             }
             byte [] prependHeader = {1, 0};
             BigInteger bigLastRow = new BigInteger(Bytes.add(prependHeader, lastPadded));



Mime
View raw message