carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject [36/50] [abbrv] carbondata git commit: [CARBONDATA-2034]Fixed Query performance issue
Date Sun, 28 Jan 2018 06:46:05 GMT
[CARBONDATA-2034]Fixed Query performance issue

Problem: Dictionary loading is taking more time in executor side when number of nodes is high.

Solution: During query no need to load non complex dimension dictionary. Dictionary decoder
will take care of loading and decoding the dictionary column

This closes #1811


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/4f9aeaea
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/4f9aeaea
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/4f9aeaea

Branch: refs/heads/fgdatamap
Commit: 4f9aeaea8290764dc108e4e3c2df0c479f311659
Parents: 2f3f3b5
Author: kumarvishal <kumarvishal.1802@gmail.com>
Authored: Tue Jan 16 14:37:25 2018 +0530
Committer: ravipesala <ravi.pesala@gmail.com>
Committed: Wed Jan 17 18:22:44 2018 +0530

----------------------------------------------------------------------
 .../core/scan/executor/util/QueryUtil.java      | 29 ++++++++++----------
 1 file changed, 15 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/4f9aeaea/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
index b33ab8d..8eb0a0d 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
@@ -281,10 +281,8 @@ public class QueryUtil {
       List<QueryDimension> queryDimensions, Set<CarbonDimension> filterComplexDimensions,
       AbsoluteTableIdentifier absoluteTableIdentifier, TableProvider tableProvider)
       throws IOException {
-    // to store dimension unique column id list, this is required as
-    // dimension can be present in
-    // query dimension, as well as some aggregation function will be applied
-    // in the same dimension
+    // to store complex dimension and its child id unique column id list, this is required
as
+    // dimension can be present in  projection and filter
     // so we need to get only one instance of dictionary
     // direct dictionary skip is done only for the dictionary lookup
     Set<String> dictionaryDimensionFromQuery = new HashSet<String>();
@@ -294,20 +292,19 @@ public class QueryUtil {
       // write encoding dictionary
       if (CarbonUtil.hasEncoding(encodingList, Encoding.DICTIONARY) && !CarbonUtil
           .hasEncoding(encodingList, Encoding.DIRECT_DICTIONARY) && !CarbonUtil
-          .hasEncoding(encodingList, Encoding.IMPLICIT)) {
-
-        if (queryDimensions.get(i).getDimension().getNumberOfChild() == 0) {
-          dictionaryDimensionFromQuery.add(queryDimensions.get(i).getDimension().getColumnId());
-        }
-        if (queryDimensions.get(i).getDimension().getNumberOfChild() > 0) {
-          getChildDimensionDictionaryDetail(queryDimensions.get(i).getDimension(),
-              dictionaryDimensionFromQuery);
-        }
+          .hasEncoding(encodingList, Encoding.IMPLICIT)
+          && queryDimensions.get(i).getDimension().getNumberOfChild() > 0) {
+        getChildDimensionDictionaryDetail(queryDimensions.get(i).getDimension(),
+            dictionaryDimensionFromQuery);
       }
     }
     Iterator<CarbonDimension> iterator = filterComplexDimensions.iterator();
     while (iterator.hasNext()) {
-      getChildDimensionDictionaryDetail(iterator.next(), dictionaryDimensionFromQuery);
+      CarbonDimension filterDim = iterator.next();
+      // only to add complex dimension
+      if (filterDim.getNumberOfChild() > 0) {
+        getChildDimensionDictionaryDetail(filterDim, dictionaryDimensionFromQuery);
+      }
     }
     // converting to list as api exposed needed list which i think
     // is not correct
@@ -348,6 +345,10 @@ public class QueryUtil {
   private static Map<String, Dictionary> getDictionaryMap(List<String> dictionaryColumnIdList,
       AbsoluteTableIdentifier absoluteTableIdentifier, TableProvider tableProvider)
       throws IOException {
+    // if any complex dimension not present in query then return the empty map
+    if (dictionaryColumnIdList.size() == 0) {
+      return new HashMap<>();
+    }
     // this for dictionary unique identifier
     List<DictionaryColumnUniqueIdentifier> dictionaryColumnUniqueIdentifiers =
         getDictionaryColumnUniqueIdentifierList(dictionaryColumnIdList,


Mime
View raw message