carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kumarvisha...@apache.org
Subject carbondata git commit: [CARBONDATA-1907] Avoid unnecessary logging to improve query performance for no dictionary non string columns
Date Wed, 20 Dec 2017 08:36:25 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master 694ee774c -> a86c54b3e


[CARBONDATA-1907] Avoid unnecessary logging to improve query performance for no dictionary
non string columns

Changes done to return null in case of no dictionary column for non string data types when
data is empty.This is done to avoid excessive logging which is impacting the query performance.

This closes #1679


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/a86c54b3
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/a86c54b3
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/a86c54b3

Branch: refs/heads/master
Commit: a86c54b3e83fef0b45ed70c50ee7483f5cc0894c
Parents: 694ee77
Author: manishgupta88 <tomanishgupta18@gmail.com>
Authored: Tue Dec 19 17:27:30 2017 +0530
Committer: kumarvishal <kumarvishal.1802@gmail.com>
Committed: Wed Dec 20 14:06:09 2017 +0530

----------------------------------------------------------------------
 .../executor/impl/AbstractQueryExecutor.java    | 11 ++++-----
 .../carbondata/core/util/DataTypeUtil.java      | 24 ++++++++++++++++++++
 .../carbondata/core/util/DataTypeUtilTest.java  | 18 +++++++++++++++
 3 files changed, 46 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/a86c54b3/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
index 88a8238..cb193e4 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
@@ -27,7 +27,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
 
 import org.apache.carbondata.common.CarbonIterator;
 import org.apache.carbondata.common.logging.LogService;
@@ -544,12 +543,10 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E>
{
     }
     UnsafeMemoryManager.INSTANCE.freeMemoryAll(ThreadLocalTaskInfo.getCarbonTaskInfo().getTaskId());
     if (null != queryProperties.executorService) {
-      queryProperties.executorService.shutdown();
-      try {
-        queryProperties.executorService.awaitTermination(1, TimeUnit.HOURS);
-      } catch (InterruptedException e) {
-        throw new QueryExecutionException(e);
-      }
+      // In case of limit query when number of limit records is already found so executors
+      // must stop all the running execution otherwise it will keep running and will hit
+      // the query performance.
+      queryProperties.executorService.shutdownNow();
     }
   }
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/a86c54b3/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
index d8c13a3..0e819c4 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
@@ -353,12 +353,26 @@ public final class DataTypeUtil {
       } else if (actualDataType == DataTypes.STRING) {
         return getDataTypeConverter().convertFromByteToUTF8String(dataInBytes);
       } else if (actualDataType == DataTypes.SHORT) {
+        // for non string type no dictionary column empty byte array is empty value
+        // so no need to parse
+        if (isEmptyByteArray(dataInBytes)) {
+          return null;
+        }
         return ByteUtil.toShort(dataInBytes, 0, dataInBytes.length);
       } else if (actualDataType == DataTypes.INT) {
+        if (isEmptyByteArray(dataInBytes)) {
+          return null;
+        }
         return ByteUtil.toInt(dataInBytes, 0, dataInBytes.length);
       } else if (actualDataType == DataTypes.LONG) {
+        if (isEmptyByteArray(dataInBytes)) {
+          return null;
+        }
         return ByteUtil.toLong(dataInBytes, 0, dataInBytes.length);
       } else if (actualDataType == DataTypes.TIMESTAMP) {
+        if (isEmptyByteArray(dataInBytes)) {
+          return null;
+        }
         return ByteUtil.toLong(dataInBytes, 0, dataInBytes.length) * 1000L;
       } else {
         return ByteUtil.toString(dataInBytes, 0, dataInBytes.length);
@@ -372,6 +386,16 @@ public final class DataTypeUtil {
     }
   }
 
+  /**
+   * Method to check if byte array is empty
+   *
+   * @param dataInBytes
+   * @return
+   */
+  private static boolean isEmptyByteArray(byte[] dataInBytes) {
+    return dataInBytes.length == 0;
+  }
+
 
   /**
    * Below method will be used to convert the data passed to its actual data

http://git-wip-us.apache.org/repos/asf/carbondata/blob/a86c54b3/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java
index c944d18..bb528b9 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java
@@ -98,6 +98,24 @@ public class DataTypeUtilTest {
 
   }
 
+  @Test public void testGetDataBasedOnDataTypeForNoDictionaryColumn() {
+    Object result = DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(new byte[0],
+        DataTypes.INT);
+    assert (result == null);
+    result = DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(new byte[0],
+        DataTypes.SHORT);
+    assert (result == null);
+    result = DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(new byte[0],
+        DataTypes.LONG);
+    assert (result == null);
+    result = DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(new byte[0],
+        DataTypes.TIMESTAMP);
+    assert (result == null);
+    result = DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(new byte[0],
+        DataTypes.STRING);
+    assert (result != null);
+  }
+
 }
 
 


Mime
View raw message