carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject [07/47] incubator-carbondata git commit: Struct of array is failing because of complex dimension ordinal (#848)
Date Mon, 01 Aug 2016 10:05:05 GMT
Struct of array is failing because of complex dimension ordinal (#848)

Problem:
Assignment of complex dimension ordinal was not correct as it was not increment and two children
was getting same complex dimension ordinal
Second issue was selecting all the block indexes of complex dimension children.

Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/07116d24
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/07116d24
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/07116d24

Branch: refs/heads/master
Commit: 07116d242db652ad8edf6e80526f6a377295343d
Parents: 4cff504
Author: Kumar Vishal <kumarvishal.1802@gmail.com>
Authored: Fri Jul 22 21:13:31 2016 +0530
Committer: sujith71955 <sujithchacko.2010@gmail.com>
Committed: Fri Jul 22 21:13:31 2016 +0530

----------------------------------------------------------------------
 .../metadata/schema/table/CarbonTable.java      | 48 ++++++++++++++------
 .../schema/table/column/CarbonDimension.java    |  4 ++
 .../query/carbon/executor/util/QueryUtil.java   | 34 +++++++-------
 .../spark/src/test/resources/struct_all.csv     |  4 ++
 .../src/test/resources/structusingstruct.csv    |  2 +
 .../complexType/TestComplexTypeQuery.scala      | 33 +++++++++++++-
 6 files changed, 91 insertions(+), 34 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/07116d24/core/src/main/java/org/carbondata/core/carbon/metadata/schema/table/CarbonTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/carbon/metadata/schema/table/CarbonTable.java
b/core/src/main/java/org/carbondata/core/carbon/metadata/schema/table/CarbonTable.java
index 6023e7a..4d16659 100644
--- a/core/src/main/java/org/carbondata/core/carbon/metadata/schema/table/CarbonTable.java
+++ b/core/src/main/java/org/carbondata/core/carbon/metadata/schema/table/CarbonTable.java
@@ -121,23 +121,20 @@ public class CarbonTable implements Serializable {
     int columnGroupOrdinal = -1;
     int previousColumnGroupId = -1;
     List<ColumnSchema> listOfColumns = tableSchema.getListOfColumns();
-    int complexTypeOrdinal = 0;
-    ;
+    int complexTypeOrdinal = -1;
     for (int i = 0; i < listOfColumns.size(); i++) {
       ColumnSchema columnSchema = listOfColumns.get(i);
       if (columnSchema.isDimensionColumn()) {
         if (columnSchema.getNumberOfChild() > 0) {
           CarbonDimension complexDimension =
-              new CarbonDimension(columnSchema, dimensionOrdinal++, -1, -1, complexTypeOrdinal++);
+              new CarbonDimension(columnSchema, dimensionOrdinal++, -1, -1, ++complexTypeOrdinal);
           complexDimension.initializeChildDimensionsList(columnSchema.getNumberOfChild());
           dimensions.add(complexDimension);
-          int lastDimensionOrdianl=dimensionOrdinal;
           dimensionOrdinal =
               readAllComplexTypeChildrens(dimensionOrdinal, columnSchema.getNumberOfChild(),
-                  listOfColumns, complexDimension, complexTypeOrdinal);
+                  listOfColumns, complexDimension);
           i = dimensionOrdinal - 1;
-          int incrementInOrdinal=dimensionOrdinal-lastDimensionOrdianl;
-          complexTypeOrdinal+=incrementInOrdinal;
+          complexTypeOrdinal = assignComplexOrdinal(complexDimension, complexTypeOrdinal);
         } else {
           if (!columnSchema.getEncodingList().contains(Encoding.DICTIONARY)) {
             dimensions.add(new CarbonDimension(columnSchema, dimensionOrdinal++, -1, -1,
-1));
@@ -171,24 +168,21 @@ public class CarbonTable implements Serializable {
    * @return
    */
   private int readAllComplexTypeChildrens(int dimensionOrdinal, int childCount,
-      List<ColumnSchema> listOfColumns, CarbonDimension parentDimension,
-      int complexDimensionOrdianl) {
+      List<ColumnSchema> listOfColumns, CarbonDimension parentDimension) {
     for (int i = 0; i < childCount; i++) {
       ColumnSchema columnSchema = listOfColumns.get(dimensionOrdinal);
       if (columnSchema.isDimensionColumn()) {
         if (columnSchema.getNumberOfChild() > 0) {
           CarbonDimension complexDimension =
-              new CarbonDimension(columnSchema, dimensionOrdinal++, -1, -1,
-                  complexDimensionOrdianl++);
+              new CarbonDimension(columnSchema, dimensionOrdinal++, -1, -1, -1);
           complexDimension.initializeChildDimensionsList(columnSchema.getNumberOfChild());
           parentDimension.getListOfChildDimensions().add(complexDimension);
           dimensionOrdinal =
               readAllComplexTypeChildrens(dimensionOrdinal, columnSchema.getNumberOfChild(),
-                  listOfColumns, complexDimension, complexDimensionOrdianl);
+                  listOfColumns, complexDimension);
         } else {
-          parentDimension.getListOfChildDimensions().add(
-              new CarbonDimension(columnSchema, dimensionOrdinal++, -1, -1,
-                  complexDimensionOrdianl++));
+          parentDimension.getListOfChildDimensions()
+              .add(new CarbonDimension(columnSchema, dimensionOrdinal++, -1, -1, -1));
         }
       }
     }
@@ -196,6 +190,30 @@ public class CarbonTable implements Serializable {
   }
 
   /**
+   * Read all primitive/complex children and set it as list of child carbon dimension to
parent
+   * dimension
+   *
+   * @param dimensionOrdinal
+   * @param childCount
+   * @param listOfColumns
+   * @param parentDimension
+   * @return
+   */
+  private int assignComplexOrdinal(CarbonDimension parentDimension, int complexDimensionOrdianl)
{
+    for (int i = 0; i < parentDimension.getNumberOfChild(); i++) {
+      CarbonDimension dimension = parentDimension.getListOfChildDimensions().get(i);
+      if (dimension.getNumberOfChild() > 0) {
+        dimension.setComplexTypeOridnal(++complexDimensionOrdianl);
+        complexDimensionOrdianl = assignComplexOrdinal(dimension, complexDimensionOrdianl);
+      } else {
+        parentDimension.getListOfChildDimensions().get(i)
+            .setComplexTypeOridnal(++complexDimensionOrdianl);
+      }
+    }
+    return complexDimensionOrdianl;
+  }
+
+  /**
    * @return the databaseName
    */
   public String getDatabaseName() {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/07116d24/core/src/main/java/org/carbondata/core/carbon/metadata/schema/table/column/CarbonDimension.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/carbon/metadata/schema/table/column/CarbonDimension.java
b/core/src/main/java/org/carbondata/core/carbon/metadata/schema/table/column/CarbonDimension.java
index 3a4dd11..d3df181 100644
--- a/core/src/main/java/org/carbondata/core/carbon/metadata/schema/table/column/CarbonDimension.java
+++ b/core/src/main/java/org/carbondata/core/carbon/metadata/schema/table/column/CarbonDimension.java
@@ -114,6 +114,10 @@ public class CarbonDimension extends CarbonColumn {
     return complexTypeOrdinal;
   }
 
+  public void setComplexTypeOridnal(int complexTypeOrdinal) {
+    this.complexTypeOrdinal = complexTypeOrdinal;
+  }
+
   /**
    * to generate the hash code for this class
    */

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/07116d24/core/src/main/java/org/carbondata/query/carbon/executor/util/QueryUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/carbon/executor/util/QueryUtil.java b/core/src/main/java/org/carbondata/query/carbon/executor/util/QueryUtil.java
index 241f5b4..2433cbc 100644
--- a/core/src/main/java/org/carbondata/query/carbon/executor/util/QueryUtil.java
+++ b/core/src/main/java/org/carbondata/query/carbon/executor/util/QueryUtil.java
@@ -252,19 +252,22 @@ public class QueryUtil {
       blockIndex =
           dimensionOrdinalToBlockMapping.get(queryDimensions.get(i).getDimension().getOrdinal());
       dimensionBlockIndex.add(blockIndex);
-      addChildrenBlockIndex(blockIndex, dimensionBlockIndex, queryDimensions.get(i).getDimension());
+      if (queryDimensions.get(i).getDimension().numberOfChild() > 0) {
+        addChildrenBlockIndex(dimensionBlockIndex, queryDimensions.get(i).getDimension());
+      }
     }
     for (int i = 0; i < dimAggInfo.size(); i++) {
       blockIndex = dimensionOrdinalToBlockMapping.get(dimAggInfo.get(i).getDim().getOrdinal());
+      // not adding the children dimension as dimension aggregation
+      // is not push down in case of complex dimension
       dimensionBlockIndex.add(blockIndex);
-      addChildrenBlockIndex(blockIndex, dimensionBlockIndex, dimAggInfo.get(i).getDim());
-
     }
     for (int i = 0; i < customAggregationDimension.size(); i++) {
       blockIndex =
           dimensionOrdinalToBlockMapping.get(customAggregationDimension.get(i).getOrdinal());
+      // not adding the children dimension as dimension aggregation
+      // is not push down in case of complex dimension
       dimensionBlockIndex.add(blockIndex);
-      addChildrenBlockIndex(blockIndex, dimensionBlockIndex, customAggregationDimension.get(i));
     }
     return ArrayUtils
         .toPrimitive(dimensionBlockIndex.toArray(new Integer[dimensionBlockIndex.size()]));
@@ -274,16 +277,13 @@ public class QueryUtil {
    * Below method will be used to add the children block index
    * this will be basically for complex dimension which will have children
    *
-   * @param startBlockIndex start block index
-   * @param blockIndexList  block index list
-   * @param dimension       parent dimension
+   * @param blockIndexes block indexes
+   * @param dimension    parent dimension
    */
-  private static void addChildrenBlockIndex(int startBlockIndex, Set<Integer> blockIndexList,
-      CarbonDimension dimension) {
+  private static void addChildrenBlockIndex(Set<Integer> blockIndexes, CarbonDimension
dimension) {
     for (int i = 0; i < dimension.numberOfChild(); i++) {
-      blockIndexList.add(++startBlockIndex);
-      addChildrenBlockIndex(startBlockIndex, blockIndexList,
-          dimension.getListOfChildDimensions().get(i));
+      addChildrenBlockIndex(blockIndexes, dimension.getListOfChildDimensions().get(i));
+      blockIndexes.add(dimension.getListOfChildDimensions().get(i).getOrdinal());
     }
   }
 
@@ -379,7 +379,7 @@ public class QueryUtil {
       if (queryDimensions.getListOfChildDimensions().get(j).numberOfChild() > 0) {
         getChildDimensionDictionaryDetail(queryDimensions.getListOfChildDimensions().get(j),
             dictionaryDimensionFromQuery);
-      } else if(!CarbonUtil.hasEncoding(encodingList, Encoding.DIRECT_DICTIONARY)) {
+      } else if (!CarbonUtil.hasEncoding(encodingList, Encoding.DIRECT_DICTIONARY)) {
         dictionaryDimensionFromQuery
             .add(queryDimensions.getListOfChildDimensions().get(j).getColumnId());
       }
@@ -873,7 +873,7 @@ public class QueryUtil {
           && queryDimension.getDimension().numberOfChild() == 0) {
         dictionaryDimensionBlockIndex
             .add(columnOrdinalToBlockIndexMapping.get(queryDimension.getDimension().getOrdinal()));
-      } else if(queryDimension.getDimension().numberOfChild() == 0){
+      } else if (queryDimension.getDimension().numberOfChild() == 0) {
         noDictionaryDimensionBlockIndex
             .add(columnOrdinalToBlockIndexMapping.get(queryDimension.getDimension().getOrdinal()));
       }
@@ -1078,9 +1078,9 @@ public class QueryUtil {
                   dimension.getColName(), ++parentBlockIndex));
           break;
         default:
-          boolean isDirectDictionary = CarbonUtil.hasEncoding(
-              dimension.getListOfChildDimensions().get(i).getEncoder(),
-              Encoding.DIRECT_DICTIONARY);
+          boolean isDirectDictionary = CarbonUtil
+              .hasEncoding(dimension.getListOfChildDimensions().get(i).getEncoder(),
+                  Encoding.DIRECT_DICTIONARY);
           parentQueryType.addChildren(
               new PrimitiveQueryType(dimension.getListOfChildDimensions().get(i).getColName(),
                   dimension.getColName(), ++parentBlockIndex,

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/07116d24/integration/spark/src/test/resources/struct_all.csv
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/resources/struct_all.csv b/integration/spark/src/test/resources/struct_all.csv
new file mode 100644
index 0000000..02b6387
--- /dev/null
+++ b/integration/spark/src/test/resources/struct_all.csv
@@ -0,0 +1,4 @@
+10&10$10&10
+20&20$20&20
+30&30$30&30
+40&40$40&40

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/07116d24/integration/spark/src/test/resources/structusingstruct.csv
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/resources/structusingstruct.csv b/integration/spark/src/test/resources/structusingstruct.csv
new file mode 100644
index 0000000..5a2bae1
--- /dev/null
+++ b/integration/spark/src/test/resources/structusingstruct.csv
@@ -0,0 +1,2 @@
+def$klm&abc$12
+pri$sac&pra$18
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/07116d24/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/complexType/TestComplexTypeQuery.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/complexType/TestComplexTypeQuery.scala
b/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/complexType/TestComplexTypeQuery.scala
index d17840f..ab48133 100644
--- a/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/complexType/TestComplexTypeQuery.scala
+++ b/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/complexType/TestComplexTypeQuery.scala
@@ -37,12 +37,37 @@ class TestComplexTypeQuery extends QueryTest with BeforeAndAfterAll {
   override def beforeAll: Unit = {
      sql("drop table if exists complexcarbontable").show
      sql("drop table if exists complexhivetable").show
+     sql("drop table if exists structusingstructCarbon").show
+     sql("drop table if exists structusingstructHive").show
+     sql("drop table if exists structusingarraycarbon").show
+     sql("drop table if exists structusingarrayhive").show
      sql("create table complexcarbontable(deviceInformationId int, channelsId string, ROMSize
string, ROMName String, purchasedate string, mobile struct<imei:string, imsi:string>,
MAC array<string>, locationinfo array<struct<ActiveAreaId:int, ActiveCountry:string,
ActiveProvince:string, Activecity:string, ActiveDistrict:string, ActiveStreet:string>>,
proddate struct<productionDate:string,activeDeactivedate:array<string>>, gamePointId
double,contractNumber double)  STORED BY 'org.apache.carbondata.format'  TBLPROPERTIES ('DICTIONARY_INCLUDE'='deviceInformationId',
'DICTIONARY_EXCLUDE'='channelsId','COLUMN_GROUP'='(ROMSize,ROMName)')");
      sql("LOAD DATA local inpath './src/test/resources/complextypesample.csv' INTO table
complexcarbontable  OPTIONS('DELIMITER'=',', 'QUOTECHAR'='\"', 'FILEHEADER'='deviceInformationId,channelsId,ROMSize,ROMName,purchasedate,mobile,MAC,locationinfo,proddate,gamePointId,contractNumber',
'COMPLEX_DELIMITER_LEVEL_1'='$', 'COMPLEX_DELIMITER_LEVEL_2'=':')");
      sql("create table complexhivetable(deviceInformationId int, channelsId string, ROMSize
string, ROMName String, purchasedate string, mobile struct<imei:string, imsi:string>,
MAC array<string>, locationinfo array<struct<ActiveAreaId:int, ActiveCountry:string,
ActiveProvince:string, Activecity:string, ActiveDistrict:string, ActiveStreet:string>>,
proddate struct<productionDate:string,activeDeactivedate:array<string>>, gamePointId
double,contractNumber double)row format delimited fields terminated by ',' collection items
terminated by '$' map keys terminated by ':'")
      sql("LOAD DATA local inpath './src/test/resources/complextypesample.csv' INTO table
complexhivetable");
+     
+     sql("create table structusingarraycarbon (MAC struct<MAC1:array<string>,ActiveCountry:array<string>>)
STORED BY 'org.apache.carbondata.format'");
+     sql("LOAD DATA local INPATH './src/test/resources/struct_all.csv' INTO table structusingarraycarbon
options ('DELIMITER'=',', 'QUOTECHAR'='\"', 'FILEHEADER'='MAC','COMPLEX_DELIMITER_LEVEL_1'='$','COMPLEX_DELIMITER_LEVEL_2'='&')")
+     sql("create table structusingarrayhive (MAC struct<MAC1:array<string>,ActiveCountry:array<string>>)row
format delimited fields terminated by ',' collection items terminated by '$' map keys terminated
by '&'");
+     sql("LOAD DATA local INPATH './src/test/resources/struct_all.csv' INTO table structusingarrayhive")

+     
+     sql("create table structusingstructCarbon(name struct<middlename:string, othernames:struct<firstname:string,lastname:string>,age:int>
) STORED BY 'org.apache.carbondata.format'")
+     sql("LOAD DATA local INPATH './src/test/resources/structusingstruct.csv' INTO table
structusingstructCarbon options ('DELIMITER'=',', 'QUOTECHAR'='\"', 'FILEHEADER'='name','COMPLEX_DELIMITER_LEVEL_1'='$','COMPLEX_DELIMITER_LEVEL_2'='&')")
+      sql("create table structusingstructhive(name struct<middlename:string, othernames:struct<firstname:string,lastname:string>,age:int>
)row format delimited fields terminated by ',' collection items terminated by '$' map keys
terminated by '&'")
+     sql("LOAD DATA local INPATH './src/test/resources/structusingstruct.csv' INTO table
structusingstructhive")
+     
   }
 
+  test("select * from structusingarraycarbon") {
+     checkAnswer(sql("select * from structusingarraycarbon"),
+     sql("select * from structusingarrayhive"))
+  }
+  
+   test("select * from structusingstructCarbon") {
+     checkAnswer(sql("select * from structusingstructCarbon"),
+     sql("select * from structusingstructhive"))
+  }
+  
   test("select * from complexcarbontable") {
      checkAnswer(sql("select * from complexcarbontable"),
      sql("select * from complexhivetable"))
@@ -112,8 +137,12 @@ class TestComplexTypeQuery extends QueryTest with BeforeAndAfterAll {
   }
 
   override def afterAll {
-	  sql("drop table if exists complexcarbontable")
-	  sql("drop table if exists complexhivetable")
+	 sql("drop table if exists complexcarbontable").show
+     sql("drop table if exists complexhivetable").show
+     sql("drop table if exists structusingstructCarbon").show
+     sql("drop table if exists structusingstructHive").show
+     sql("drop table if exists structusingarraycarbon").show
+     sql("drop table if exists structusingarrayhive").show
 
   }
 }


Mime
View raw message