carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From chenliang...@apache.org
Subject carbondata git commit: [CARBONDATA-1774] [PrestoIntegration] Not able to fetch data from a table with Boolean data type in presto
Date Thu, 04 Jan 2018 08:50:56 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master deabeffd9 -> e2f19979b


[CARBONDATA-1774] [PrestoIntegration] Not able to fetch data from a table with Boolean data
type in presto

Not able to fetch data from a Carbon table with Boolean data type in presto

This closes #1709


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/e2f19979
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/e2f19979
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/e2f19979

Branch: refs/heads/master
Commit: e2f19979b6be2a38de936da316ed91d9b9f6bfee
Parents: deabeff
Author: anubhav100 <anubhav.tarar@knoldus.in>
Authored: Tue Dec 19 15:33:06 2017 +0530
Committer: chenliang613 <chenliang613@huawei.com>
Committed: Thu Jan 4 16:50:42 2018 +0800

----------------------------------------------------------------------
 .../presto/CarbonVectorizedRecordReader.java    |  17 ++-
 .../presto/readers/BooleanStreamReader.java     | 107 +++++++++++++++++++
 .../presto/readers/StreamReaders.java           |   4 +
 .../presto/src/test/resources/alldatatype.csv   |  22 ++--
 .../integrationtest/PrestoAllDataTypeTest.scala |  15 +++
 .../presto/util/CarbonDataStoreCreator.scala    |  13 ++-
 6 files changed, 157 insertions(+), 21 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/e2f19979/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonVectorizedRecordReader.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonVectorizedRecordReader.java
b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonVectorizedRecordReader.java
index 910ccf4..9a8f8c5 100644
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonVectorizedRecordReader.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonVectorizedRecordReader.java
@@ -186,18 +186,17 @@ class CarbonVectorizedRecordReader extends AbstractRecordReader<Object>
{
       }
     }
 
-    for (int i = 0; i < queryMeasures.size(); i++) {
-      QueryMeasure msr = queryMeasures.get(i);
+    for (QueryMeasure msr : queryMeasures) {
       DataType dataType = msr.getMeasure().getDataType();
-      if (dataType == DataTypes.SHORT || dataType == DataTypes.INT || dataType == DataTypes.LONG)
{
-        fields[msr.getQueryOrder()] = new StructField(msr.getColumnName(),
-            msr.getMeasure().getDataType());
+      if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.SHORT || dataType == DataTypes.INT
+          || dataType == DataTypes.LONG) {
+        fields[msr.getQueryOrder()] =
+            new StructField(msr.getColumnName(), msr.getMeasure().getDataType());
       } else if (DataTypes.isDecimal(dataType)) {
-        fields[msr.getQueryOrder()] = new StructField(msr.getColumnName(),
-           msr.getMeasure().getDataType());
+        fields[msr.getQueryOrder()] =
+            new StructField(msr.getColumnName(), msr.getMeasure().getDataType());
       } else {
-        fields[msr.getQueryOrder()] = new StructField(msr.getColumnName(),
-            DataTypes.DOUBLE);
+        fields[msr.getQueryOrder()] = new StructField(msr.getColumnName(), DataTypes.DOUBLE);
       }
     }
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/e2f19979/integration/presto/src/main/java/org/apache/carbondata/presto/readers/BooleanStreamReader.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/readers/BooleanStreamReader.java
b/integration/presto/src/main/java/org/apache/carbondata/presto/readers/BooleanStreamReader.java
new file mode 100644
index 0000000..2a45e72
--- /dev/null
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/readers/BooleanStreamReader.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.presto.readers;
+
+import java.io.IOException;
+
+import org.apache.carbondata.core.cache.dictionary.Dictionary;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.util.DataTypeUtil;
+
+import com.facebook.presto.spi.block.Block;
+import com.facebook.presto.spi.block.BlockBuilder;
+import com.facebook.presto.spi.block.BlockBuilderStatus;
+import com.facebook.presto.spi.type.Type;
+
+public class BooleanStreamReader extends AbstractStreamReader {
+
+  private boolean isDictionary;
+  private Dictionary dictionary;
+
+  public BooleanStreamReader() {
+
+  }
+
+  public BooleanStreamReader(boolean isDictionary, Dictionary dictionary) {
+    this.isDictionary = isDictionary;
+    this.dictionary = dictionary;
+  }
+
+  public Block readBlock(Type type) throws IOException {
+    int numberOfRows = 0;
+    BlockBuilder builder = null;
+    if (isVectorReader) {
+      numberOfRows = batchSize;
+      builder = type.createBlockBuilder(new BlockBuilderStatus(), numberOfRows);
+      if (columnVector != null) {
+        if(columnVector.anyNullsSet()) {
+          handleNullInVector(type, numberOfRows, builder);
+        }
+        else {
+          populateVector(type, numberOfRows, builder);
+        }
+      }
+
+    } else {
+      numberOfRows = streamData.length;
+      builder = type.createBlockBuilder(new BlockBuilderStatus(), numberOfRows);
+      if (streamData != null) {
+        for (int i = 0; i < numberOfRows; i++) {
+          type.writeBoolean(builder, byteToBoolean(streamData[i]));
+        }
+      }
+    }
+
+    return builder.build();
+  }
+
+  private void handleNullInVector(Type type, int numberOfRows, BlockBuilder builder) {
+    for (int i = 0; i < numberOfRows; i++) {
+      if (columnVector.isNullAt(i)) {
+        builder.appendNull();
+      } else {
+        type.writeBoolean(builder, byteToBoolean(columnVector.getData(i)));
+      }
+    }
+  }
+
+  private void populateVector(Type type, int numberOfRows, BlockBuilder builder) {
+    if(isDictionary) {
+      for (int i = 0; i < numberOfRows; i++) {
+        int value = (int) columnVector.getData(i);
+        Object data = DataTypeUtil
+            .getDataBasedOnDataType(dictionary.getDictionaryValueForKey(value), DataTypes.BOOLEAN);
+        if (data != null) {
+          type.writeBoolean(builder,(boolean) data);
+        } else {
+          builder.appendNull();
+        }
+      }
+    }
+    else {
+      for (int i = 0; i < numberOfRows; i++) {
+        type.writeBoolean(builder, byteToBoolean(columnVector.getData(i)));
+      }
+    }
+  }
+  private Boolean byteToBoolean(Object value){
+    byte byteValue = (byte)value;
+    return byteValue == 1;
+  }
+}
+

http://git-wip-us.apache.org/repos/asf/carbondata/blob/e2f19979/integration/presto/src/main/java/org/apache/carbondata/presto/readers/StreamReaders.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/readers/StreamReaders.java
b/integration/presto/src/main/java/org/apache/carbondata/presto/readers/StreamReaders.java
index b9b8b7e..1ad3d28 100644
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/readers/StreamReaders.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/readers/StreamReaders.java
@@ -61,6 +61,8 @@ public final class StreamReaders {
         } else {
           return new SliceStreamReader(true, dictionarySliceArrayBlock);
         }
+      }else if (javaType == boolean.class) {
+              return new BooleanStreamReader(true,dictionary);
       } else {
         return new ObjectStreamReader();
       }
@@ -85,6 +87,8 @@ public final class StreamReaders {
         } else {
           return new SliceStreamReader();
         }
+      }else if (javaType == boolean.class) {
+        return new BooleanStreamReader();
       } else {
         return new ObjectStreamReader();
       }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/e2f19979/integration/presto/src/test/resources/alldatatype.csv
----------------------------------------------------------------------
diff --git a/integration/presto/src/test/resources/alldatatype.csv b/integration/presto/src/test/resources/alldatatype.csv
index ba6156e..7201542 100644
--- a/integration/presto/src/test/resources/alldatatype.csv
+++ b/integration/presto/src/test/resources/alldatatype.csv
@@ -1,11 +1,11 @@
-ID,date,country,name,phonetype,serialname,salary,bonus,monthlyBonus,dob,shortfield
-1,2015-07-23,china,anubhav,phone197,ASD69643,5000000.00,1234.444,12.1234,2016-04-14 15/00/09,10
-2,2015-07-24,china,jatin,phone756,ASD42892,150010.999,1234.5555,15.13,2016-04-14 15:00:09,10
-3,2015-07-25,china,liang,phone1904,ASD37014,15002.110,600.777,16.181,2016-01-14 15:07:09,8
-4,2015-07-26,china,prince,phone2435,ASD66902,15003.00,9999.999,17.3654,1992-04-14 13:00:09,4
-5,2015-07-27,china,bhavya,phone2441,ASD90633,15004.00,5000.999,12.11,2010-06-19 14:10:06,11
-6,2015-07-28,china,akash,phone294,ASD59961,15005.00,500.59,18.65,2013-07-19 12:10:08,18
-7,2015-07-29,china,sahil,phone610,ASD14875,15006.00,500.99,,19.65,2007-04-19 11:10:06,17
-8,2015-07-30,china,geetika,phone1848,ASD57308,15007.500,500.88,200.97,2008-09-21 11:10:06,10
-9,2015-07-18,china,ravindra,phone706,ASD86717,15008.00,700.999,45.25,2009-06-19 15:10:06,1
-9,2015/07/18,china,jitesh,phone706,ASD86717,15008.00,500.414,11.655,2001-08-29 13:09:03,12
+ID,date,country,name,phonetype,serialname,salary,bonus,monthlyBonus,dob,shortfield,isCurrentEmployee
+1,2015-07-23,china,anubhav,phone197,ASD69643,5000000.00,1234.444,12.1234,2016-04-14 15/00/09,10,true
+2,2015-07-24,china,jatin,phone756,ASD42892,150010.999,1234.5555,15.13,2016-04-14 15:00:09,10,null
+3,2015-07-25,china,liang,phone1904,ASD37014,15002.110,600.777,16.181,2016-01-14 15:07:09,8,true
+4,2015-07-26,china,prince,phone2435,ASD66902,15003.00,9999.999,17.3654,1992-04-14 13:00:09,4,true
+5,2015-07-27,china,bhavya,phone2441,ASD90633,15004.00,5000.999,12.11,2010-06-19 14:10:06,11,true
+6,2015-07-28,china,akash,phone294,ASD59961,15005.00,500.59,18.65,2013-07-19 12:10:08,18,false
+7,2015-07-29,china,sahil,phone610,ASD14875,15006.00,500.99,,19.65,2007-04-19 11:10:06,17,false
+8,2015-07-30,china,geetika,phone1848,ASD57308,15007.500,500.88,200.97,2008-09-21 11:10:06,10,true
+9,2015-07-18,china,ravindra,phone706,ASD86717,15008.00,700.999,45.25,2009-06-19 15:10:06,1,true
+9,2015/07/18,china,jitesh,phone706,ASD86717,15008.00,500.414,11.655,2001-08-29 13:09:03,12,true

http://git-wip-us.apache.org/repos/asf/carbondata/blob/e2f19979/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala
----------------------------------------------------------------------
diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala
b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala
index 153ca5e..49da227 100644
--- a/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala
+++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala
@@ -435,4 +435,19 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll
{
     val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> "2"))
     assert(actualResult.toString() equals expectedResult.toString())
   }
+  test("test the boolean data type") {
+    val actualResult: List[Map[String, Any]] = PrestoServer
+      .executeQuery("SELECT isCurrentEmployee AS RESULT FROM TESTDB.TESTTABLE WHERE ID=1")
+    assert(actualResult.head("RESULT").toString.toBoolean)
+  }
+  test("test the boolean data type for null value") {
+    val actualResult: List[Map[String, Any]] = PrestoServer
+      .executeQuery("SELECT id AS RESULT FROM TESTDB.TESTTABLE WHERE isCurrentEmployee is
null")
+    assert(actualResult.head("RESULT").toString.toInt==2)
+  }
+  test("test the boolean data type for not null value with filter ") {
+    val actualResult: List[Map[String, Any]] = PrestoServer
+      .executeQuery("SELECT id AS RESULT FROM TESTDB.TESTTABLE WHERE isCurrentEmployee is
NOT null AND ID>8")
+    assert(actualResult.head("RESULT").toString.toInt==9)
+  }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/carbondata/blob/e2f19979/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
----------------------------------------------------------------------
diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
b/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
index 5a0adf5..7b5c311 100644
--- a/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
+++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
@@ -131,7 +131,7 @@ object CarbonDataStoreCreator {
         "true")
       loadModel.setMaxColumns("15")
       loadModel.setCsvHeader(
-        "ID,date,country,name,phonetype,serialname,salary,bonus,monthlyBonus,dob,shortField")
+        "ID,date,country,name,phonetype,serialname,salary,bonus,monthlyBonus,dob,shortField,isCurrentEmployee")
       loadModel.setCsvHeaderColumns(loadModel.getCsvHeader.split(","))
       loadModel.setTaskNo("0")
       loadModel.setSegmentId("0")
@@ -302,6 +302,17 @@ object CarbonDataStoreCreator {
     shortField.setColumnReferenceId(shortField.getColumnUniqueId)
     columnSchemas.add(shortField)
 
+    val isCurrentEmployee: ColumnSchema = new ColumnSchema()
+    isCurrentEmployee.setColumnName("isCurrentEmployee")
+    isCurrentEmployee.setColumnar(true)
+    isCurrentEmployee.setDataType(DataTypes.BOOLEAN)
+    isCurrentEmployee.setEncodingList(invertedIndexEncoding)
+    isCurrentEmployee.setColumnUniqueId(UUID.randomUUID().toString)
+    isCurrentEmployee.setDimensionColumn(false)
+    isCurrentEmployee.setColumnGroup(11)
+    isCurrentEmployee.setColumnReferenceId(isCurrentEmployee.getColumnUniqueId)
+    columnSchemas.add(isCurrentEmployee)
+
     tableSchema.setListOfColumns(columnSchemas)
     val schemaEvol: SchemaEvolution = new SchemaEvolution()
     schemaEvol.setSchemaEvolutionEntryList(


Mime
View raw message