carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject carbondata git commit: [CARBONDATA-2900] Add dynamic configuration for system properties
Date Mon, 03 Sep 2018 16:48:32 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master 3cbabcde0 -> 5a1496f3e


[CARBONDATA-2900] Add dynamic configuration for system properties

Following system level properties are added for dynamic configuration. So that user can SET
command to set it in JDBC environment like Beeline.

carbon.number.of.cores.while.loading
carbon.number.of.cores.while.compacting
carbon.blockletgroup.size.in.mb
carbon.major.compaction.size
carbon.enable.vector.reader
enable.unsafe.in.query.processing

This closes #2674


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/5a1496f3
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/5a1496f3
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/5a1496f3

Branch: refs/heads/master
Commit: 5a1496f3e21769795b48bc73c4625fea9b9cd134
Parents: 3cbabcd
Author: Jacky Li <jacky.likun@qq.com>
Authored: Wed Aug 29 17:36:12 2018 +0800
Committer: ravipesala <ravi.pesala@gmail.com>
Committed: Mon Sep 3 22:18:18 2018 +0530

----------------------------------------------------------------------
 .../carbondata/core/util/SessionParams.java     | 21 ++++++++++++++++++++
 1 file changed, 21 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/5a1496f3/core/src/main/java/org/apache/carbondata/core/util/SessionParams.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/SessionParams.java b/core/src/main/java/org/apache/carbondata/core/util/SessionParams.java
index 51b157f..2abf0e1 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/SessionParams.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/SessionParams.java
@@ -33,9 +33,15 @@ import org.apache.carbondata.core.datamap.DataMapStoreManager;
 import org.apache.carbondata.core.exception.InvalidConfigurationException;
 
 import static org.apache.carbondata.core.constants.CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION;
+import static org.apache.carbondata.core.constants.CarbonCommonConstants.CARBON_MAJOR_COMPACTION_SIZE;
 import static org.apache.carbondata.core.constants.CarbonCommonConstants.CARBON_SEARCH_MODE_ENABLE;
+import static org.apache.carbondata.core.constants.CarbonCommonConstants.COMPACTION_SEGMENT_LEVEL_THRESHOLD;
 import static org.apache.carbondata.core.constants.CarbonCommonConstants.ENABLE_OFFHEAP_SORT;
+import static org.apache.carbondata.core.constants.CarbonCommonConstants.ENABLE_UNSAFE_IN_QUERY_EXECUTION;
 import static org.apache.carbondata.core.constants.CarbonCommonConstants.ENABLE_UNSAFE_SORT;
+import static org.apache.carbondata.core.constants.CarbonCommonConstants.ENABLE_VECTOR_READER;
+import static org.apache.carbondata.core.constants.CarbonCommonConstants.NUM_CORES_COMPACTING;
+import static org.apache.carbondata.core.constants.CarbonCommonConstants.NUM_CORES_LOADING;
 import static org.apache.carbondata.core.constants.CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_ACTION;
 import static org.apache.carbondata.core.constants.CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE;
 import static org.apache.carbondata.core.constants.CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORD_PATH;
@@ -47,6 +53,7 @@ import static org.apache.carbondata.core.constants.CarbonLoadOptionConstants.CAR
 import static org.apache.carbondata.core.constants.CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS;
 import static org.apache.carbondata.core.constants.CarbonLoadOptionConstants.CARBON_OPTIONS_SORT_SCOPE;
 import static org.apache.carbondata.core.constants.CarbonLoadOptionConstants.CARBON_OPTIONS_TIMESTAMPFORMAT;
+import static org.apache.carbondata.core.constants.CarbonV3DataFormatConstants.BLOCKLET_SIZE_IN_MB;
 
 /**
  * This class maintains carbon session params
@@ -153,6 +160,8 @@ public class SessionParams implements Serializable, Cloneable {
       case CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD:
       case CARBON_OPTIONS_SINGLE_PASS:
       case CARBON_SEARCH_MODE_ENABLE:
+      case ENABLE_VECTOR_READER:
+      case ENABLE_UNSAFE_IN_QUERY_EXECUTION:
         isValid = CarbonUtil.validateBoolean(value);
         if (!isValid) {
           throw new InvalidConfigurationException("Invalid value " + value + " for key "
+ key);
@@ -176,6 +185,10 @@ public class SessionParams implements Serializable, Cloneable {
         break;
       case CARBON_OPTIONS_BATCH_SORT_SIZE_INMB:
       case CARBON_OPTIONS_GLOBAL_SORT_PARTITIONS:
+      case NUM_CORES_LOADING:
+      case NUM_CORES_COMPACTING:
+      case BLOCKLET_SIZE_IN_MB:
+      case CARBON_MAJOR_COMPACTION_SIZE:
         isValid = CarbonUtil.validateValidIntType(value);
         if (!isValid) {
           throw new InvalidConfigurationException(
@@ -200,6 +213,14 @@ public class SessionParams implements Serializable, Cloneable {
       case CARBON_OPTIONS_SERIALIZATION_NULL_FORMAT:
         isValid = true;
         break;
+      case COMPACTION_SEGMENT_LEVEL_THRESHOLD:
+        int[] values = CarbonProperties.getInstance().getIntArray(value);
+        if (values.length != 2) {
+          throw new InvalidConfigurationException(
+              "Invalid COMPACTION_SEGMENT_LEVEL_THRESHOLD: " + value);
+        }
+        isValid = true;
+        break;
       default:
         if (key.startsWith(CarbonCommonConstants.CARBON_INPUT_SEGMENTS)) {
           isValid = CarbonUtil.validateRangeOfSegmentList(value);


Mime
View raw message