carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From chenliang...@apache.org
Subject [1/2] incubator-carbondata git commit: FixedTestcasefailureIssue
Date Tue, 28 Feb 2017 13:58:38 GMT
Repository: incubator-carbondata
Updated Branches:
  refs/heads/master 891fa4346 -> f4e3ec819


FixedTestcasefailureIssue

Fixed failure for V1 format


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/b16c3088
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/b16c3088
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/b16c3088

Branch: refs/heads/master
Commit: b16c30885e706764bbcd87adaf850a88b1411698
Parents: 891fa43
Author: kumarvishal <kumarvishal.1802@gmail.com>
Authored: Tue Feb 28 19:21:50 2017 +0800
Committer: chenliang613 <chenliang613@huawei.com>
Committed: Tue Feb 28 21:57:05 2017 +0800

----------------------------------------------------------------------
 .../BlockIndexerStorageForNoInvertedIndex.java  | 135 ++++++-------------
 ...ndexerStorageForNoInvertedIndexForShort.java | 104 ++++++++++++++
 .../store/CarbonFactDataHandlerColumnar.java    |   8 +-
 3 files changed, 151 insertions(+), 96 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b16c3088/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndex.java
b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndex.java
index d9875a0..0ef2518 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndex.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndex.java
@@ -14,128 +14,76 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
 package org.apache.carbondata.core.datastore.columnar;
 
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.List;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.columnar.IndexStorage;
 import org.apache.carbondata.core.util.ByteUtil;
 
+/**
+ * Below class will be used to for no inverted index
+ */
 public class BlockIndexerStorageForNoInvertedIndex implements IndexStorage<int[]> {
+
+  /**
+   * column data
+   */
   private byte[][] keyBlock;
-  private byte[][] sortedBlock;
+
+  /**
+   * total number of rows
+   */
   private int totalSize;
-  private int[] dataIndexMap;
 
-  public BlockIndexerStorageForNoInvertedIndex(byte[][] keyBlockInput, boolean compressData,
-      boolean isNoDictionary) {
-    // without invertedindex but can be RLE
-    if (compressData) {
-      // with RLE
-      byte[] prvKey = keyBlockInput[0];
-      List<byte[]> list = new ArrayList<byte[]>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
-      list.add(keyBlockInput[0]);
-      int counter = 1;
-      int start = 0;
-      List<Integer> map = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
-      int length = keyBlockInput.length;
-      for (int i = 1; i < length; i++) {
-        if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(prvKey, keyBlockInput[i]) != 0) {
-          prvKey = keyBlockInput[i];
-          list.add(keyBlockInput[i]);
-          map.add(start);
-          map.add(counter);
-          start += counter;
-          counter = 1;
-          continue;
-        }
-        counter++;
+  private byte[] min;
+  private byte[] max;
+
+  public BlockIndexerStorageForNoInvertedIndex(byte[][] keyBlockInput, boolean isNoDictionary)
{
+    this.keyBlock = keyBlockInput;
+    min = keyBlock[0];
+    max = keyBlock[0];
+    totalSize += keyBlock[0].length;
+    int minCompare = 0;
+    int maxCompare = 0;
+    for (int i = 1; i < keyBlock.length; i++) {
+      totalSize += keyBlock[i].length;
+      minCompare = ByteUtil.compare(min, keyBlock[i]);
+      maxCompare = ByteUtil.compare(max, keyBlock[i]);
+      if (minCompare > 0) {
+        min = keyBlock[i];
       }
-      map.add(start);
-      map.add(counter);
-      this.keyBlock = convertToKeyArray(list);
-      if (keyBlockInput.length == this.keyBlock.length) {
-        dataIndexMap = new int[0];
-      } else {
-        dataIndexMap = convertToArray(map);
+      if (maxCompare < 0) {
+        max = keyBlock[i];
       }
-    } else {
-      this.keyBlock = keyBlockInput;
-      dataIndexMap = new int[0];
-    }
-
-    this.sortedBlock = new byte[keyBlock.length][];
-    System.arraycopy(keyBlock, 0, sortedBlock, 0, keyBlock.length);
-    if (isNoDictionary) {
-      Arrays.sort(sortedBlock, new Comparator<byte[]>() {
-        @Override
-        public int compare(byte[] col1, byte[] col2) {
-          return ByteUtil.UnsafeComparer.INSTANCE
-              .compareTo(col1, 2, col1.length - 2, col2, 2, col2.length - 2);
-        }
-      });
-    } else {
-      Arrays.sort(sortedBlock, new Comparator<byte[]>() {
-        @Override
-        public int compare(byte[] col1, byte[] col2) {
-          return ByteUtil.UnsafeComparer.INSTANCE.compareTo(col1, col2);
-        }
-      });
-    }
-
-  }
-
-  private int[] convertToArray(List<Integer> list) {
-    int[] shortArray = new int[list.size()];
-    for (int i = 0; i < shortArray.length; i++) {
-      shortArray[i] = list.get(i);
     }
-    return shortArray;
   }
 
-  private byte[][] convertToKeyArray(List<byte[]> list) {
-    byte[][] shortArray = new byte[list.size()][];
-    for (int i = 0; i < shortArray.length; i++) {
-      shortArray[i] = list.get(i);
-      totalSize += shortArray[i].length;
-    }
-    return shortArray;
-  }
-
-  @Override
-  public int[] getDataIndexMap() {
-    return dataIndexMap;
+  @Override public int[] getDataIndexMap() {
+    return new int[0];
   }
 
-  @Override
-  public int getTotalSize() {
+  @Override public int getTotalSize() {
     return totalSize;
   }
 
-  @Override
-  public boolean isAlreadySorted() {
+  @Override public boolean isAlreadySorted() {
     return true;
   }
 
   /**
    * no use
+   *
    * @return
    */
-  @Override
-  public int[] getDataAfterComp() {
+  @Override public int[] getDataAfterComp() {
     return new int[0];
   }
 
   /**
    * no use
+   *
    * @return
    */
-  @Override
-  public int[] getIndexMap() {
+  @Override public int[] getIndexMap() {
     return new int[0];
   }
 
@@ -147,11 +95,10 @@ public class BlockIndexerStorageForNoInvertedIndex implements IndexStorage<int[]
   }
 
   @Override public byte[] getMin() {
-    return sortedBlock[0];
+    return min;
   }
 
   @Override public byte[] getMax() {
-    return sortedBlock[sortedBlock.length - 1];
+    return max;
   }
-
-}
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b16c3088/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndexForShort.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndexForShort.java
b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndexForShort.java
new file mode 100644
index 0000000..731df96
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndexForShort.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datastore.columnar;
+
+import org.apache.carbondata.core.util.ByteUtil;
+
+/**
+ * Below class will be used to for no inverted index
+ */
+public class BlockIndexerStorageForNoInvertedIndexForShort implements IndexStorage<short[]>
{
+
+  /**
+   * column data
+   */
+  private byte[][] keyBlock;
+
+  /**
+   * total number of rows
+   */
+  private int totalSize;
+
+  private byte[] min;
+  private byte[] max;
+
+  public BlockIndexerStorageForNoInvertedIndexForShort(byte[][] keyBlockInput,
+      boolean isNoDictionary) {
+    this.keyBlock = keyBlockInput;
+    min = keyBlock[0];
+    max = keyBlock[0];
+    totalSize += keyBlock[0].length;
+    int minCompare = 0;
+    int maxCompare = 0;
+    for (int i = 1; i < keyBlock.length; i++) {
+      totalSize += keyBlock[i].length;
+      minCompare = ByteUtil.compare(min, keyBlock[i]);
+      maxCompare = ByteUtil.compare(max, keyBlock[i]);
+      if (minCompare > 0) {
+        min = keyBlock[i];
+      }
+      if (maxCompare < 0) {
+        max = keyBlock[i];
+      }
+    }
+  }
+
+  @Override public short[] getDataIndexMap() {
+    return new short[0];
+  }
+
+  @Override public int getTotalSize() {
+    return totalSize;
+  }
+
+  @Override public boolean isAlreadySorted() {
+    return true;
+  }
+
+  /**
+   * no use
+   *
+   * @return
+   */
+  @Override public short[] getDataAfterComp() {
+    return new short[0];
+  }
+
+  /**
+   * no use
+   *
+   * @return
+   */
+  @Override public short[] getIndexMap() {
+    return new short[0];
+  }
+
+  /**
+   * @return the keyBlock
+   */
+  public byte[][] getKeyBlock() {
+    return keyBlock;
+  }
+
+  @Override public byte[] getMin() {
+    return min;
+  }
+
+  @Override public byte[] getMax() {
+    return max;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b16c3088/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
index 0699167..ed461f9 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
@@ -44,6 +44,7 @@ import org.apache.carbondata.core.constants.CarbonV3DataFormatConstants;
 import org.apache.carbondata.core.datastore.block.SegmentProperties;
 import org.apache.carbondata.core.datastore.columnar.BlockIndexerStorageForInt;
 import org.apache.carbondata.core.datastore.columnar.BlockIndexerStorageForNoInvertedIndex;
+import org.apache.carbondata.core.datastore.columnar.BlockIndexerStorageForNoInvertedIndexForShort;
 import org.apache.carbondata.core.datastore.columnar.BlockIndexerStorageForShort;
 import org.apache.carbondata.core.datastore.columnar.ColumnGroupModel;
 import org.apache.carbondata.core.datastore.columnar.IndexStorage;
@@ -1637,8 +1638,11 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler
{
               isSortRequired);
         }
       } else {
-        return new BlockIndexerStorageForNoInvertedIndex(this.data, isCompressionReq,
-            isNoDictionary);
+        if (version == ColumnarFormatVersion.V3) {
+          return new BlockIndexerStorageForNoInvertedIndexForShort(this.data, isNoDictionary);
+        } else {
+          return new BlockIndexerStorageForNoInvertedIndex(this.data, isNoDictionary);
+        }
       }
 
     }


Mime
View raw message