carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zhan...@apache.org
Subject [carbondata] branch master updated: fix start/end key of no-dict sort columns
Date Sun, 01 Dec 2019 08:28:52 GMT
This is an automated email from the ASF dual-hosted git repository.

zhangzc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git


The following commit(s) were added to refs/heads/master by this push:
     new bacda1a  fix start/end key of no-dict sort columns
bacda1a is described below

commit bacda1a6b57b3e515919584eeb85b290c021ec0a
Author: QiangCai <qiangcai@qq.com>
AuthorDate: Sat Nov 30 22:57:39 2019 +0800

    fix start/end key of no-dict sort columns
    
    This closes #3491
---
 .../core/datastore/page/key/TablePageKey.java      | 49 +++++++++++++++-------
 .../core/metadata/schema/table/CarbonTable.java    | 12 ++++++
 2 files changed, 45 insertions(+), 16 deletions(-)

diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/key/TablePageKey.java
b/core/src/main/java/org/apache/carbondata/core/datastore/page/key/TablePageKey.java
index 609f17e..3786ef1 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/key/TablePageKey.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/key/TablePageKey.java
@@ -18,18 +18,20 @@
 package org.apache.carbondata.core.datastore.page.key;
 
 import java.nio.ByteBuffer;
+import java.util.List;
 
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import org.apache.carbondata.core.datastore.block.SegmentProperties;
 import org.apache.carbondata.core.datastore.row.CarbonRow;
 import org.apache.carbondata.core.datastore.row.WriteStepRowUtil;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
+import org.apache.carbondata.core.util.DataTypeUtil;
 import org.apache.carbondata.core.util.NonDictionaryUtil;
 
 public class TablePageKey {
   private int pageSize;
 
-  private byte[][] currentNoDictionaryKey;
-
   // MDK start key
   private byte[] startKey;
 
@@ -37,10 +39,10 @@ public class TablePageKey {
   private byte[] endKey;
 
   // startkey for no dictionary columns
-  private byte[][] noDictStartKey;
+  private Object[] noDictStartKey;
 
   // endkey for no diciotn
-  private byte[][] noDictEndKey;
+  private Object[] noDictEndKey;
 
   // startkey for no dictionary columns after packing into one column
   private byte[] packedNoDictStartKey;
@@ -60,17 +62,17 @@ public class TablePageKey {
 
   /** update all keys based on the input row */
   public void update(int rowId, CarbonRow row, byte[] mdk) {
-    if (hasNoDictionary) {
-      Object[] noDictAndComplexDimension = WriteStepRowUtil.getNoDictAndComplexDimension(row);
-      currentNoDictionaryKey = new byte[noDictAndComplexDimension.length][0];
-    }
     if (rowId == 0) {
       startKey = mdk;
-      noDictStartKey = currentNoDictionaryKey;
+      if (hasNoDictionary) {
+        noDictStartKey = WriteStepRowUtil.getNoDictAndComplexDimension(row);
+      }
     }
-    noDictEndKey = currentNoDictionaryKey;
     if (rowId == pageSize - 1) {
       endKey = mdk;
+      if (hasNoDictionary) {
+        noDictEndKey = WriteStepRowUtil.getNoDictAndComplexDimension(row);
+      }
       finalizeKeys();
     }
   }
@@ -107,8 +109,8 @@ public class TablePageKey {
     if (numberOfNoDictSortColumns > 0) {
       // if sort_columns contain no-dictionary columns
       if (noDictStartKey.length > numberOfNoDictSortColumns) {
-        byte[][] newNoDictionaryStartKey = new byte[numberOfNoDictSortColumns][];
-        byte[][] newNoDictionaryEndKey = new byte[numberOfNoDictSortColumns][];
+        Object[] newNoDictionaryStartKey = new Object[numberOfNoDictSortColumns];
+        Object[] newNoDictionaryEndKey = new Object[numberOfNoDictSortColumns];
         System.arraycopy(
             noDictStartKey, 0, newNoDictionaryStartKey, 0, numberOfNoDictSortColumns);
         System.arraycopy(
@@ -116,10 +118,12 @@ public class TablePageKey {
         noDictStartKey = newNoDictionaryStartKey;
         noDictEndKey = newNoDictionaryEndKey;
       }
-      packedNoDictStartKey =
-          NonDictionaryUtil.packByteBufferIntoSingleByteArray(noDictStartKey);
-      packedNoDictEndKey =
-          NonDictionaryUtil.packByteBufferIntoSingleByteArray(noDictEndKey);
+      List<CarbonDimension> noDictSortColumns =
+          CarbonTable.getNoDictSortColumns(segmentProperties.getDimensions());
+      packedNoDictStartKey = NonDictionaryUtil.packByteBufferIntoSingleByteArray(
+          convertKeys(noDictStartKey, noDictSortColumns));
+      packedNoDictEndKey = NonDictionaryUtil.packByteBufferIntoSingleByteArray(
+          convertKeys(noDictEndKey, noDictSortColumns));
     } else {
       noDictStartKey = new byte[0][];
       noDictEndKey = new byte[0][];
@@ -128,6 +132,19 @@ public class TablePageKey {
     }
   }
 
+  private byte[][] convertKeys(Object[] keys, List<CarbonDimension> noDictSortColumns)
{
+    byte[][] finalKeys = new byte[keys.length][];
+    for (int i = 0; i < keys.length; i++) {
+      if (keys[i] instanceof byte[]) {
+        finalKeys[i] = (byte[]) keys[i];
+      } else {
+        finalKeys[i] = DataTypeUtil.getBytesDataDataTypeForNoDictionaryColumn(keys[i],
+            noDictSortColumns.get(i).getDataType());
+      }
+    }
+    return finalKeys;
+  }
+
   public byte[] getNoDictStartKey() {
     return packedNoDictStartKey;
   }
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index bb0b28d..9b9b756 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -818,6 +818,18 @@ public class CarbonTable implements Serializable, Writable {
     return numberOfNoDictSortColumns;
   }
 
+  public static List<CarbonDimension> getNoDictSortColumns(List<CarbonDimension>
dimensions) {
+    List<CarbonDimension> noDictSortColumns = new ArrayList<>(dimensions.size());
+    for (int i = 0; i < dimensions.size(); i++) {
+      CarbonDimension dimension = dimensions.get(i);
+      if (dimension.isSortColumn() &&
+          !dimension.getColumnSchema().hasEncoding(Encoding.DICTIONARY)) {
+        noDictSortColumns.add(dimension);
+      }
+    }
+    return noDictSortColumns;
+  }
+
   public CarbonColumn getRangeColumn() {
     String rangeColumn =
         tableInfo.getFactTable().getTableProperties().get(CarbonCommonConstants.RANGE_COLUMN);


Mime
View raw message