carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject [carbondata] 11/22: [CARBONDATA-3359]Fix data mismatch issue for decimal column after delete operation
Date Thu, 16 May 2019 19:05:44 GMT
This is an automated email from the ASF dual-hosted git repository.

ravipesala pushed a commit to branch branch-1.5
in repository https://gitbox.apache.org/repos/asf/carbondata.git

commit f7cdb47e7535c7543147bf96f42cfbc14b36b082
Author: akashrn5 <akashnilugal@gmail.com>
AuthorDate: Thu Apr 25 15:16:35 2019 +0530

    [CARBONDATA-3359]Fix data mismatch issue for decimal column after delete operation
    
    Problem:
    after delete operation is performed, the decimal column data is wrong. This is because,
during filling vector for decimal column,
     we were not considering the deleted rows if present any, we were filling all the row
data for decimal.
    
    Solution
    in case of decimal, get the vector from ColumnarVectorWrapperDirectFactory and then put
data, which will take care of the deleted rows
    
    This closes #3189
---
 .../metadata/datatype/DecimalConverterFactory.java | 55 +++++++++++++---------
 .../src/test/resources/decimalData.csv             |  4 ++
 .../testsuite/iud/DeleteCarbonTableTestCase.scala  | 17 +++++++
 3 files changed, 54 insertions(+), 22 deletions(-)

diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java
b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java
index 9793c38..2e155f4 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java
@@ -23,6 +23,7 @@ import java.util.BitSet;
 
 import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
 import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
+import org.apache.carbondata.core.scan.result.vector.impl.directread.ColumnarVectorWrapperDirectFactory;
 import org.apache.carbondata.core.util.ByteUtil;
 import org.apache.carbondata.core.util.DataTypeUtil;
 
@@ -102,13 +103,13 @@ public final class DecimalConverterFactory {
       return BigDecimal.valueOf((Long) valueToBeConverted, scale);
     }
 
-    @Override public void fillVector(Object valuesToBeConverted, int size, ColumnVectorInfo
info,
-        BitSet nullBitset, DataType pageType) {
+    @Override public void fillVector(Object valuesToBeConverted, int size,
+        ColumnVectorInfo vectorInfo, BitSet nullBitSet, DataType pageType) {
       // TODO we need to find way to directly set to vector with out conversion. This way
is very
       // inefficient.
-      CarbonColumnVector vector = info.vector;
-      int precision = info.measure.getMeasure().getPrecision();
-      int newMeasureScale = info.measure.getMeasure().getScale();
+      CarbonColumnVector vector = getCarbonColumnVector(vectorInfo, nullBitSet);
+      int precision = vectorInfo.measure.getMeasure().getPrecision();
+      int newMeasureScale = vectorInfo.measure.getMeasure().getScale();
       if (!(valuesToBeConverted instanceof byte[])) {
         throw new UnsupportedOperationException("This object type " + valuesToBeConverted.getClass()
             + " is not supported in this method");
@@ -116,7 +117,7 @@ public final class DecimalConverterFactory {
       byte[] data = (byte[]) valuesToBeConverted;
       if (pageType == DataTypes.BYTE) {
         for (int i = 0; i < size; i++) {
-          if (nullBitset.get(i)) {
+          if (nullBitSet.get(i)) {
             vector.putNull(i);
           } else {
             BigDecimal value = BigDecimal.valueOf(data[i], scale);
@@ -128,7 +129,7 @@ public final class DecimalConverterFactory {
         }
       } else if (pageType == DataTypes.SHORT) {
         for (int i = 0; i < size; i++) {
-          if (nullBitset.get(i)) {
+          if (nullBitSet.get(i)) {
             vector.putNull(i);
           } else {
             BigDecimal value = BigDecimal
@@ -142,7 +143,7 @@ public final class DecimalConverterFactory {
         }
       } else if (pageType == DataTypes.SHORT_INT) {
         for (int i = 0; i < size; i++) {
-          if (nullBitset.get(i)) {
+          if (nullBitSet.get(i)) {
             vector.putNull(i);
           } else {
             BigDecimal value = BigDecimal
@@ -156,7 +157,7 @@ public final class DecimalConverterFactory {
         }
       } else if (pageType == DataTypes.INT) {
         for (int i = 0; i < size; i++) {
-          if (nullBitset.get(i)) {
+          if (nullBitSet.get(i)) {
             vector.putNull(i);
           } else {
             BigDecimal value = BigDecimal
@@ -170,7 +171,7 @@ public final class DecimalConverterFactory {
         }
       } else if (pageType == DataTypes.LONG) {
         for (int i = 0; i < size; i++) {
-          if (nullBitset.get(i)) {
+          if (nullBitSet.get(i)) {
             vector.putNull(i);
           } else {
             BigDecimal value = BigDecimal
@@ -261,18 +262,18 @@ public final class DecimalConverterFactory {
       return new BigDecimal(bigInteger, scale);
     }
 
-    @Override public void fillVector(Object valuesToBeConverted, int size, ColumnVectorInfo
info,
-        BitSet nullBitset, DataType pageType) {
-      CarbonColumnVector vector = info.vector;
-      int precision = info.measure.getMeasure().getPrecision();
-      int newMeasureScale = info.measure.getMeasure().getScale();
+    @Override public void fillVector(Object valuesToBeConverted, int size,
+        ColumnVectorInfo vectorInfo, BitSet nullBitSet, DataType pageType) {
+      CarbonColumnVector vector = getCarbonColumnVector(vectorInfo, nullBitSet);
+      int precision = vectorInfo.measure.getMeasure().getPrecision();
+      int newMeasureScale = vectorInfo.measure.getMeasure().getScale();
       if (scale < newMeasureScale) {
         scale = newMeasureScale;
       }
       if (valuesToBeConverted instanceof byte[][]) {
         byte[][] data = (byte[][]) valuesToBeConverted;
         for (int i = 0; i < size; i++) {
-          if (nullBitset.get(i)) {
+          if (nullBitSet.get(i)) {
             vector.putNull(i);
           } else {
             BigInteger bigInteger = new BigInteger(data[i]);
@@ -307,15 +308,15 @@ public final class DecimalConverterFactory {
       return DataTypeUtil.byteToBigDecimal((byte[]) valueToBeConverted);
     }
 
-    @Override public void fillVector(Object valuesToBeConverted, int size, ColumnVectorInfo
info,
-        BitSet nullBitset, DataType pageType) {
-      CarbonColumnVector vector = info.vector;
-      int precision = info.measure.getMeasure().getPrecision();
-      int newMeasureScale = info.measure.getMeasure().getScale();
+    @Override public void fillVector(Object valuesToBeConverted, int size,
+        ColumnVectorInfo vectorInfo, BitSet nullBitSet, DataType pageType) {
+      CarbonColumnVector vector = getCarbonColumnVector(vectorInfo, nullBitSet);
+      int precision = vectorInfo.measure.getMeasure().getPrecision();
+      int newMeasureScale = vectorInfo.measure.getMeasure().getScale();
       if (valuesToBeConverted instanceof byte[][]) {
         byte[][] data = (byte[][]) valuesToBeConverted;
         for (int i = 0; i < size; i++) {
-          if (nullBitset.get(i)) {
+          if (nullBitSet.get(i)) {
             vector.putNull(i);
           } else {
             BigDecimal value = DataTypeUtil.byteToBigDecimal(data[i]);
@@ -337,6 +338,16 @@ public final class DecimalConverterFactory {
     }
   }
 
+  private static CarbonColumnVector getCarbonColumnVector(ColumnVectorInfo vectorInfo,
+      BitSet nullBitSet) {
+    CarbonColumnVector vector = vectorInfo.vector;
+    BitSet deletedRows = vectorInfo.deletedRows;
+    vector = ColumnarVectorWrapperDirectFactory
+        .getDirectVectorWrapperFactory(vector, vectorInfo.invertedIndex, nullBitSet, deletedRows,
+            true, false);
+    return vector;
+  }
+
   public DecimalConverter getDecimalConverter(int precision, int scale) {
     if (precision < 0) {
       return new LVBytesDecimalConverter();
diff --git a/integration/spark-common-test/src/test/resources/decimalData.csv b/integration/spark-common-test/src/test/resources/decimalData.csv
new file mode 100644
index 0000000..8cbbd21
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/decimalData.csv
@@ -0,0 +1,4 @@
+smallIntField,intField,bigIntField,floatField,doubleField,decimalField,timestampField,dateField,stringField,varcharField,charField,arrayField,structField
+-1,-1,-1,-1.1,-1.1,-1.1234,2017-06-11 00:00:01,2017-06-11,abc1,abcd1,abcde1,a$b$c$1,a$b$1
+2,2,2,2.1,2.1,2.1234,2017-06-12 23:59:02,2017-06-12,abc2,abcd2,abcde2,a$b$c$2,a$b$2
+3,3,3,3.1,3.1,3.1234,2017-06-13 23:59:03,2017-06-13,abc3,abcd3,abcde3,a$b$c$3,a$b$3
\ No newline at end of file
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala
index 2f95133..f26283b 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala
@@ -344,6 +344,23 @@ class DeleteCarbonTableTestCase extends QueryTest with BeforeAndAfterAll
{
     sql("drop table if exists test_dm_index")
   }
 
+  test("test delete on table with decimal column") {
+    sql("drop table if exists decimal_table")
+    sql(
+      s"""create table decimal_table(smallIntField smallInt,intField int,bigIntField bigint,floatField
float,
+          doubleField double,decimalField decimal(25, 4),timestampField timestamp,dateField
date,stringField string,
+          varcharField varchar(10),charField char(10))stored as carbondata
+      """.stripMargin)
+    sql(s"load data local inpath '$resourcesPath/decimalData.csv' into table decimal_table")
+    val frame = sql("select decimalfield from decimal_table where smallIntField = -1 or smallIntField
= 3")
+    sql(s"delete from decimal_table where smallIntField = 2")
+    checkAnswer(frame, Seq(
+      Row(-1.1234),
+      Row(3.1234)
+    ))
+    sql("drop table if exists decimal_table")
+  }
+
   override def afterAll {
     sql("use default")
     sql("drop database  if exists iud_db cascade")


Mime
View raw message