carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [03/24] carbondata git commit: [CARBONDATA-1653] Rename aggType to measureDataType
Date Wed, 08 Nov 2017 04:15:37 GMT
[CARBONDATA-1653] Rename aggType to measureDataType

This closes #1444


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/4d70a211
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/4d70a211
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/4d70a211

Branch: refs/heads/pre-aggregate
Commit: 4d70a2118fb26936e0b7eee9a0ce2147cb0501c9
Parents: 93a604d
Author: Jacky Li <jacky.likun@qq.com>
Authored: Sat Oct 28 16:26:38 2017 +0530
Committer: chenliang613 <chenliang613@apache.org>
Committed: Sat Oct 28 23:39:26 2017 +0530

----------------------------------------------------------------------
 .../scan/executor/impl/AbstractQueryExecutor.java  |  4 ++--
 .../core/scan/executor/util/QueryUtil.java         | 17 -----------------
 .../core/scan/executor/util/QueryUtilTest.java     |  9 ---------
 .../merger/CompactionResultSortProcessor.java      |  8 +-------
 .../sort/sortdata/IntermediateFileMerger.java      |  6 +++---
 .../sort/sortdata/SortTempFileChunkHolder.java     | 12 ++++++------
 .../store/CarbonFactDataHandlerModel.java          |  6 +++---
 7 files changed, 15 insertions(+), 47 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/4d70a211/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
index b2c1a2b..56ed575 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
@@ -161,10 +161,10 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E>
{
         .addStatistics(QueryStatisticsConstants.LOAD_BLOCKS_EXECUTOR, System.currentTimeMillis());
     queryProperties.queryStatisticsRecorder.recordStatistics(queryStatistic);
     // calculating the total number of aggeragted columns
-    int aggTypeCount = queryModel.getQueryMeasures().size();
+    int measureCount = queryModel.getQueryMeasures().size();
 
     int currentIndex = 0;
-    DataType[] dataTypes = new DataType[aggTypeCount];
+    DataType[] dataTypes = new DataType[measureCount];
 
     for (QueryMeasure carbonMeasure : queryModel.getQueryMeasures()) {
       // adding the data type and aggregation type of all the measure this

http://git-wip-us.apache.org/repos/asf/carbondata/blob/4d70a211/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
index 59387b3..6aab8a0 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
@@ -678,23 +678,6 @@ public class QueryUtil {
   }
 
   /**
-   * below method will be used to get the actual type aggregator
-   *
-   * @param aggType
-   * @return index in aggrgetor
-   */
-  public static int[] getActualTypeIndex(List<String> aggType) {
-    List<Integer> indexList = new ArrayList<Integer>();
-    for (int i = 0; i < aggType.size(); i++) {
-      if (!CarbonCommonConstants.SUM.equals(aggType.get(i)) && !CarbonCommonConstants.AVERAGE
-          .equals(aggType.get(i))) {
-        indexList.add(i);
-      }
-    }
-    return ArrayUtils.toPrimitive(indexList.toArray(new Integer[indexList.size()]));
-  }
-
-  /**
    * Below method will be used to get the key structure for the column group
    *
    * @param segmentProperties      segment properties

http://git-wip-us.apache.org/repos/asf/carbondata/blob/4d70a211/core/src/test/java/org/apache/carbondata/core/scan/executor/util/QueryUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/executor/util/QueryUtilTest.java
b/core/src/test/java/org/apache/carbondata/core/scan/executor/util/QueryUtilTest.java
index 23504d3..b764569 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/executor/util/QueryUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/executor/util/QueryUtilTest.java
@@ -221,15 +221,6 @@ public class QueryUtilTest extends TestCase {
     assertArrayEquals(expectedValue, actualValue);
   }
 
-  @Test public void testGetActualTypeIndex() {
-    List<String> dummyList = new ArrayList<>();
-    dummyList.add("test1");
-    dummyList.add("test2");
-    int[] actualValue = QueryUtil.getActualTypeIndex(dummyList);
-    int[] expectedValue = { 0, 1 };
-    assertArrayEquals(expectedValue, actualValue);
-  }
-
   @AfterClass public void tearDown() {
     segmentProperties = null;
   }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/4d70a211/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
index 0e20ef5..a361f3a 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
@@ -155,7 +155,7 @@ public class CompactionResultSortProcessor extends AbstractResultProcessor
{
     try {
       initTempStoreLocation();
       initSortDataRows();
-      initAggType();
+      dataTypes = CarbonDataProcessorUtil.initDataType(carbonTable, tableName, measureCount);
       processResult(resultIteratorList);
       // After delete command, if no records are fetched from one split,
       // below steps are not required to be initialized.
@@ -406,10 +406,4 @@ public class CompactionResultSortProcessor extends AbstractResultProcessor
{
             true, false);
   }
 
-  /**
-   * initialise aggregation type for measures for their storage format
-   */
-  private void initAggType() {
-    dataTypes = CarbonDataProcessorUtil.initDataType(carbonTable, tableName, measureCount);
-  }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/4d70a211/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateFileMerger.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateFileMerger.java
b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateFileMerger.java
index fa50f1c..ebc811c 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateFileMerger.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateFileMerger.java
@@ -326,7 +326,7 @@ public class IntermediateFileMerger implements Callable<Void> {
       return;
     }
     try {
-      DataType[] aggType = mergerParameters.getMeasureDataType();
+      DataType[] measureDataType = mergerParameters.getMeasureDataType();
       int[] mdkArray = (int[]) row[0];
       byte[][] nonDictArray = (byte[][]) row[1];
       int mdkIndex = 0;
@@ -346,7 +346,7 @@ public class IntermediateFileMerger implements Callable<Void> {
       for (int counter = 0; counter < mergerParameters.getMeasureColCount(); counter++)
{
         if (null != NonDictionaryUtil.getMeasure(fieldIndex, row)) {
           stream.write((byte) 1);
-          DataType dataType = aggType[counter];
+          DataType dataType = measureDataType[counter];
           if (dataType == DataTypes.BOOLEAN) {
             stream.writeBoolean((boolean)NonDictionaryUtil.getMeasure(fieldIndex, row));
           } else if (dataType == DataTypes.SHORT) {
@@ -362,7 +362,7 @@ public class IntermediateFileMerger implements Callable<Void> {
             stream.writeInt(bigDecimalInBytes.length);
             stream.write(bigDecimalInBytes);
           } else {
-            throw new IllegalArgumentException("unsupported data type:" + aggType[counter]);
+            throw new IllegalArgumentException("unsupported data type:" + measureDataType[counter]);
           }
         } else {
           stream.write((byte) 0);

http://git-wip-us.apache.org/repos/asf/carbondata/blob/4d70a211/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortTempFileChunkHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortTempFileChunkHolder.java
b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortTempFileChunkHolder.java
index afa4cd7..91bc83c 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortTempFileChunkHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortTempFileChunkHolder.java
@@ -129,7 +129,7 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
 
   private int noDictionaryCount;
 
-  private DataType[] aggType;
+  private DataType[] measureDataTypes;
 
   /**
    * to store whether dimension is of dictionary type or not
@@ -150,11 +150,11 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
    * @param measureCount
    * @param fileBufferSize
    * @param noDictionaryCount
-   * @param aggType
+   * @param measureDataTypes
    * @param isNoDictionaryDimensionColumn
    */
   public SortTempFileChunkHolder(File tempFile, int dimensionCount, int complexDimensionCount,
-      int measureCount, int fileBufferSize, int noDictionaryCount, DataType[] aggType,
+      int measureCount, int fileBufferSize, int noDictionaryCount, DataType[] measureDataTypes,
       boolean[] isNoDictionaryDimensionColumn, boolean[] isNoDictionarySortColumn,
       String tableName) {
     // set temp file
@@ -170,7 +170,7 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
     this.fileBufferSize = fileBufferSize;
     this.executorService = Executors
         .newFixedThreadPool(1, new CarbonThreadFactory("SafeSortTempChunkHolderPool:" + tableName));
-    this.aggType = aggType;
+    this.measureDataTypes = measureDataTypes;
 
     this.isNoDictionaryDimensionColumn = isNoDictionaryDimensionColumn;
     this.isNoDictionarySortColumn = isNoDictionarySortColumn;
@@ -344,7 +344,7 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
       // read measure values
       for (int i = 0; i < this.measureCount; i++) {
         if (stream.readByte() == 1) {
-          DataType dataType = aggType[i];
+          DataType dataType = measureDataTypes[i];
           if (dataType == DataTypes.BOOLEAN) {
             measures[index++] = stream.readBoolean();
           } else if (dataType == DataTypes.SHORT) {
@@ -361,7 +361,7 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
             stream.readFully(buff);
             measures[index++] = DataTypeUtil.byteToBigDecimal(buff);
           } else {
-            throw new IllegalArgumentException("unsupported data type:" + aggType[i]);
+            throw new IllegalArgumentException("unsupported data type:" + measureDataTypes[i]);
           }
         } else {
           measures[index++] = null;

http://git-wip-us.apache.org/repos/asf/carbondata/blob/4d70a211/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
index 2c346b2..b6ac19d 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
@@ -299,12 +299,12 @@ public class CarbonFactDataHandlerModel {
     Map<Integer, GenericDataType> complexIndexMap =
         new HashMap<Integer, GenericDataType>(segmentProperties.getComplexDimensions().size());
     carbonFactDataHandlerModel.setComplexIndexMap(complexIndexMap);
-    DataType[] aggType = new DataType[segmentProperties.getMeasures().size()];
+    DataType[] measureDataTypes = new DataType[segmentProperties.getMeasures().size()];
     int i = 0;
     for (CarbonMeasure msr : segmentProperties.getMeasures()) {
-      aggType[i++] = msr.getDataType();
+      measureDataTypes[i++] = msr.getDataType();
     }
-    carbonFactDataHandlerModel.setMeasureDataType(aggType);
+    carbonFactDataHandlerModel.setMeasureDataType(measureDataTypes);
     String carbonDataDirectoryPath = CarbonDataProcessorUtil
         .checkAndCreateCarbonStoreLocation(loadModel.getStorePath(), loadModel.getDatabaseName(),
             tableName, loadModel.getPartitionId(), loadModel.getSegmentId());


Mime
View raw message