Repository: drill
Updated Branches:
refs/heads/master a3442021b -> a51c98b8b
http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java
index 3604eb7..ecaa7b7 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java
@@ -20,10 +20,11 @@ package org.apache.drill.exec.server.options;
import java.util.Set;
import com.google.common.collect.Sets;
+import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.exec.server.options.OptionValue.Kind;
import org.apache.drill.exec.server.options.OptionValue.OptionType;
-
+import org.apache.drill.exec.server.options.OptionValue.OptionScope;
import static com.google.common.base.Preconditions.checkArgument;
public class TypeValidators {
@@ -31,8 +32,8 @@ public class TypeValidators {
public static class PositiveLongValidator extends LongValidator {
private final long max;
- public PositiveLongValidator(String name, long max, long def) {
- super(name, def);
+ public PositiveLongValidator(String name, long max) {
+ super(name);
this.max = max;
}
@@ -49,8 +50,8 @@ public class TypeValidators {
public static class PowerOfTwoLongValidator extends PositiveLongValidator {
- public PowerOfTwoLongValidator(String name, long max, long def) {
- super(name, max, def);
+ public PowerOfTwoLongValidator(String name, long max) {
+ super(name, max);
}
@Override
@@ -72,8 +73,8 @@ public class TypeValidators {
private final double min;
private final double max;
- public RangeDoubleValidator(String name, double min, double max, double def) {
- super(name, def);
+ public RangeDoubleValidator(String name, double min, double max) {
+ super(name);
this.min = min;
this.max = max;
}
@@ -93,7 +94,7 @@ public class TypeValidators {
private final String maxValidatorName;
public MinRangeDoubleValidator(String name, double min, double max, double def, String maxValidatorName) {
- super(name, min, max, def);
+ super(name, min, max);
this.maxValidatorName = maxValidatorName;
}
@@ -114,7 +115,7 @@ public class TypeValidators {
private final String minValidatorName;
public MaxRangeDoubleValidator(String name, double min, double max, double def, String minValidatorName) {
- super(name, min, max, def);
+ super(name, min, max);
this.minValidatorName = minValidatorName;
}
@@ -132,42 +133,61 @@ public class TypeValidators {
}
public static class BooleanValidator extends TypeValidator {
- public BooleanValidator(String name, boolean def) {
- this(name, def, false);
+ public BooleanValidator(String name) {
+ this(name, false);
+ }
+
+ public BooleanValidator(String name, boolean isAdminOption) {
+ super(name, Kind.BOOLEAN, isAdminOption);
}
- public BooleanValidator(String name, boolean def, boolean isAdminOption) {
- super(name, Kind.BOOLEAN, OptionValue.createBoolean(OptionType.SYSTEM, name, def), isAdminOption);
+ public void loadDefault(DrillConfig bootConfig){
+ OptionValue value = OptionValue.createBoolean(OptionType.SYSTEM, getOptionName(), bootConfig.getBoolean(getConfigProperty()), OptionScope.BOOT);
+ setDefaultValue(value);
}
}
public static class StringValidator extends TypeValidator {
- public StringValidator(String name, String def) {
- this(name, def, false);
+ public StringValidator(String name) {
+ this(name, false);
+ }
+ public StringValidator(String name, boolean isAdminOption) {
+ super(name, Kind.STRING, isAdminOption);
}
- public StringValidator(String name, String def, boolean isAdminOption) {
- super(name, Kind.STRING, OptionValue.createString(OptionType.SYSTEM, name, def), isAdminOption);
+ public void loadDefault(DrillConfig bootConfig){
+ OptionValue value = OptionValue.createString(OptionType.SYSTEM, getOptionName(), bootConfig.getString(getConfigProperty()), OptionScope.BOOT);
+ setDefaultValue(value);
}
}
public static class LongValidator extends TypeValidator {
- public LongValidator(String name, long def) {
- this(name, def, false);
+ public LongValidator(String name) {
+ this(name, false);
}
- public LongValidator(String name, long def, boolean isAdminOption) {
- super(name, Kind.LONG, OptionValue.createLong(OptionType.SYSTEM, name, def), isAdminOption);
+ public LongValidator(String name, boolean isAdminOption) {
+ super(name, Kind.LONG, isAdminOption);
+ }
+
+ public void loadDefault(DrillConfig bootConfig){
+ OptionValue value = OptionValue.createLong(OptionType.SYSTEM, getOptionName(), bootConfig.getLong(getConfigProperty()), OptionScope.BOOT);
+ setDefaultValue(value);
}
}
public static class DoubleValidator extends TypeValidator {
- public DoubleValidator(String name, double def) {
- this(name, def, false);
+ public DoubleValidator(String name) {
+ this(name,false);
}
- public DoubleValidator(String name, double def, boolean isAdminOption) {
- super(name, Kind.DOUBLE, OptionValue.createDouble(OptionType.SYSTEM, name, def), isAdminOption);
+ public DoubleValidator(String name, boolean isAdminOption) {
+ super(name, Kind.DOUBLE, isAdminOption);
+ }
+
+ public void loadDefault(DrillConfig bootConfig){
+ OptionValue value = OptionValue.createDouble(OptionType.SYSTEM, getOptionName(),bootConfig.getDouble(getConfigProperty()), OptionScope.BOOT);
+ setDefaultValue(value);
}
}
@@ -175,8 +195,8 @@ public class TypeValidators {
private final long min;
private final long max;
- public RangeLongValidator(String name, long min, long max, long def) {
- super(name, def);
+ public RangeLongValidator(String name, long min, long max) {
+ super(name);
this.min = min;
this.max = max;
}
@@ -198,9 +218,8 @@ public class TypeValidators {
public static class EnumeratedStringValidator extends StringValidator {
private final Set<String> valuesSet = Sets.newLinkedHashSet();
- public EnumeratedStringValidator(String name, String def, String... values) {
- super(name, def);
- valuesSet.add(def.toLowerCase());
+ public EnumeratedStringValidator(String name, String... values) {
+ super(name);
for (String value : values) {
valuesSet.add(value.toLowerCase());
}
@@ -217,19 +236,51 @@ public class TypeValidators {
}
}
+ /** Max width is a special validator which computes and validates
+ * the maxwidth. If the maxwidth is already set in system/session
+ * the value is returned or else it is computed dynamically based on
+ * the available number of processors and cpu load average
+ */
+ public static class MaxWidthValidator extends LongValidator{
+
+ public MaxWidthValidator(String name) {
+ this(name, false);
+ }
+
+ public MaxWidthValidator(String name, boolean isAdminOption) {
+ super(name, isAdminOption);
+ }
+
+ public void loadDefault(DrillConfig bootConfig) {
+ OptionValue value = OptionValue.createLong(OptionType.SYSTEM, getOptionName(), bootConfig.getLong(getConfigProperty()), OptionScope.BOOT);
+ setDefaultValue(value);
+ }
+
+ public int computeMaxWidth(double cpuLoadAverage, long maxWidth) {
+ // if maxwidth is already set return it
+ if (maxWidth != 0) {
+ return (int) maxWidth;
+ }
+ // else compute the value and return
+ else {
+ int availProc = Runtime.getRuntime().availableProcessors();
+ long maxWidthPerNode = Math.max(1, Math.min(availProc, Math.round(availProc * cpuLoadAverage)));
+ return (int) maxWidthPerNode;
+ }
+ }
+ }
+
public static abstract class TypeValidator extends OptionValidator {
private final Kind kind;
- private final OptionValue defaultValue;
+ private OptionValue defaultValue = null;
- public TypeValidator(final String name, final Kind kind, final OptionValue defValue) {
- this(name, kind, defValue, false);
+ public TypeValidator(final String name, final Kind kind) {
+ this(name, kind, false);
}
- public TypeValidator(final String name, final Kind kind, final OptionValue defValue, final boolean isAdminOption) {
+ public TypeValidator(final String name, final Kind kind, final boolean isAdminOption) {
super(name, isAdminOption);
- checkArgument(defValue.type == OptionType.SYSTEM, "Default value must be SYSTEM type.");
this.kind = kind;
- this.defaultValue = defValue;
}
@Override
@@ -256,5 +307,13 @@ public class TypeValidators {
public Kind getKind() {
return kind;
}
+
+ protected void setDefaultValue(OptionValue defaultValue) {
+ this.defaultValue = defaultValue;
+ }
+
+ public String getConfigProperty() {
+ return OPTION_DEFAULTS_ROOT + getOptionName();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java
index d0007d3..1caf10b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java
@@ -102,7 +102,7 @@ public class StatusResources {
OptionValue.Kind.valueOf(kind),
OptionValue.OptionType.SYSTEM,
name,
- value));
+ value, OptionValue.OptionScope.SYSTEM));
} catch (Exception e) {
logger.debug("Could not update.", e);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/ExtendedOptionIterator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/ExtendedOptionIterator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/ExtendedOptionIterator.java
new file mode 100644
index 0000000..fdd5092
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/ExtendedOptionIterator.java
@@ -0,0 +1,155 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.sys;
+
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+
+import com.google.common.collect.Lists;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.server.options.OptionManager;
+import org.apache.drill.exec.server.options.OptionValue;
+import org.apache.drill.exec.server.options.OptionValue.Kind;
+import org.apache.drill.exec.server.options.OptionValue.OptionType;
+import org.apache.drill.exec.server.options.OptionValue.OptionScope;
+
+
+/*
+ * Extends the original Option iterator. The idea is to hide the implementation details and present the
+ * user with the rows which have values set at the top level of hierarchy and exclude the values set
+ * at lower levels. This is done by examining the scope and the precedence order of scope is session - system - default.
+ * All the values are represented as String instead of having multiple
+ * columns and the data type is provided as kind to preserve type information about the option.
+ * The query output is as follows -
+ * name,kind,type,val,optionScope
+ * planner.slice_target,BIGINT,SESSION,20,SESSION
+ * planner.width.max_per_node,BIGINT,SYSTEM,0,BOOT
+ * planner.affinity_factor,FLOAT,SYSTEM,1.7,SYSTEM
+ * In the above example, the query output contains single row for each option
+ * and we can infer that slice target is set at the session level and max width
+ * per node is set at the BOOT level and affinity factor is set at the SYSTEM level.
+ * The options set in the top level of hierarchy always takes precedence and they are returned
+ * in the query output. For example if the option is set at both SESSION level and
+ * SYSTEM level the value set at SESSION level takes precedence and query output has
+ * only the value set at SESSION level.
+ */
+public class ExtendedOptionIterator implements Iterator<Object> {
+// private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(OptionIterator.class);
+
+ private final OptionManager fragmentOptions;
+ private final Iterator<OptionValue> mergedOptions;
+
+ public ExtendedOptionIterator(FragmentContext context) {
+ fragmentOptions = context.getOptions();
+ mergedOptions = sortOptions(fragmentOptions.iterator());
+ }
+ /* *
+ * Remove the redundant rows for the same option based on the scope and return
+ * the value that takes precedence over others. For example the option set in session
+ * scope takes precedence over system and boot and etc.,
+ */
+ public Iterator<OptionValue> sortOptions(Iterator<OptionValue> options) {
+ List<OptionValue> optionslist = Lists.newArrayList(options);
+ HashMap<String, OptionValue> optionsmap = new HashMap<>();
+ final Map<OptionScope, Integer> preference = new HashMap<OptionScope, Integer>() {{
+ put(OptionScope.SESSION, 0);
+ put(OptionScope.SYSTEM, 1);
+ put(OptionScope.BOOT, 2);
+ }};
+
+ for (OptionValue option : optionslist) {
+ if (optionsmap.containsKey(option.getName())) {
+
+ if (preference.get(option.scope) < preference.get(optionsmap.get(option.getName()).scope)) {
+ optionsmap.put(option.getName(), option);
+ }
+
+ } else {
+ optionsmap.put(option.getName(), option);
+ }
+ }
+ optionslist.clear();
+ for (String name : optionsmap.keySet()) {
+ optionslist.add(optionsmap.get(name));
+ }
+
+ Collections.sort(optionslist, new Comparator<OptionValue>() {
+ @Override
+ public int compare(OptionValue v1, OptionValue v2) {
+ return v1.name.compareTo(v2.name);
+ }
+ });
+
+ return optionslist.iterator();
+ }
+
+ @Override
+ public boolean hasNext() {
+ return mergedOptions.hasNext();
+ }
+
+ @Override
+ public ExtendedOptionValueWrapper next() {
+ final OptionValue value = mergedOptions.next();
+ final HashMap<OptionValue.Kind,String> typeMapping = new HashMap() {{
+ put(Kind.STRING,"VARCHAR");
+ put(Kind.DOUBLE,"FLOAT");
+ put(Kind.LONG,"BIGINT");
+ put(Kind.BOOLEAN,"BIT");
+
+ }};
+ return new ExtendedOptionValueWrapper(value.name, typeMapping.get(value.kind), value.type,value.getValue().toString(), value.scope);
+ }
+
+ public enum Status {
+ BOOT, DEFAULT, CHANGED
+ }
+
+ /**
+ * Wrapper class for Extended Option Value
+ */
+ public static class ExtendedOptionValueWrapper {
+
+ public final String name;
+ public final String kind;
+ public final OptionType type;
+ public final String val;
+ public final OptionScope optionScope;
+
+
+ public ExtendedOptionValueWrapper(final String name, final String kind, final OptionType type, final String value, final OptionScope scope) {
+ this.name = name;
+ this.kind = kind;
+ this.type = type;
+ this.val = value;
+ this.optionScope = scope;
+ }
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+}
+
+
http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/OptionIterator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/OptionIterator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/OptionIterator.java
index 5c8a641..424ff95 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/OptionIterator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/OptionIterator.java
@@ -25,6 +25,7 @@ import com.google.common.collect.Iterators;
import com.google.common.collect.Lists;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.server.options.DrillConfigIterator;
+import org.apache.drill.exec.server.options.FragmentOptionManager;
import org.apache.drill.exec.server.options.OptionManager;
import org.apache.drill.exec.server.options.OptionValue;
import org.apache.drill.exec.server.options.OptionValue.Kind;
@@ -40,10 +41,12 @@ public class OptionIterator implements Iterator<Object> {
private final OptionManager fragmentOptions;
private final Iterator<OptionValue> mergedOptions;
+ private FragmentContext context;
public OptionIterator(FragmentContext context, Mode mode){
final DrillConfigIterator configOptions = new DrillConfigIterator(context.getConfig());
fragmentOptions = context.getOptions();
+ this.context = context;
final Iterator<OptionValue> optionList;
switch(mode){
case BOOT:
@@ -71,10 +74,12 @@ public class OptionIterator implements Iterator<Object> {
public OptionValueWrapper next() {
final OptionValue value = mergedOptions.next();
final Status status;
+ final FragmentOptionManager fragmentOptionManager = (FragmentOptionManager) fragmentOptions;
+ final SystemOptionManager systemOptionManager = (SystemOptionManager) fragmentOptionManager.getFallback();
if (value.type == OptionType.BOOT) {
status = Status.BOOT;
} else {
- final OptionValue def = SystemOptionManager.getValidator(value.name).getDefault();
+ final OptionValue def = systemOptionManager.getValidator(value.name).getDefault();
if (value.equalsIgnoreType(def)) {
status = Status.DEFAULT;
} else {
http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTable.java
index 7f99fb7..fa6981b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTable.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTable.java
@@ -21,6 +21,7 @@ import java.util.Iterator;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.store.sys.OptionIterator.OptionValueWrapper;
+import org.apache.drill.exec.store.sys.ExtendedOptionIterator.ExtendedOptionValueWrapper;
/**
* An enumeration of all tables in Drill's system ("sys") schema.
@@ -39,6 +40,13 @@ public enum SystemTable {
}
},
+ OPTION2("options2", false,ExtendedOptionIterator.ExtendedOptionValueWrapper.class ) {
+ @Override
+ public Iterator<Object> getIterator(final FragmentContext context) {
+ return new ExtendedOptionIterator(context);
+ }
+ },
+
BOOT("boot", false, OptionValueWrapper.class) {
@Override
public Iterator<Object> getIterator(final FragmentContext context) {
http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/main/java/org/apache/drill/exec/testing/ExecutionControls.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/testing/ExecutionControls.java b/exec/java-exec/src/main/java/org/apache/drill/exec/testing/ExecutionControls.java
index 000d90f..a046c77 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/testing/ExecutionControls.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/testing/ExecutionControls.java
@@ -22,6 +22,7 @@ import com.fasterxml.jackson.annotation.JsonSubTypes.Type;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.exec.ExecConstants;
@@ -75,13 +76,11 @@ public final class ExecutionControls {
/**
* Constructor for controls option validator.
- *
- * @param name the name of the validator
- * @param def the default JSON, specified as string
+ * @param name the name of the validator
* @param ttl the number of queries for which this option should be valid
*/
- public ControlsOptionValidator(final String name, final String def, final int ttl) {
- super(name, OptionValue.Kind.STRING, OptionValue.createString(OptionType.SYSTEM, name, def));
+ public ControlsOptionValidator(final String name, final int ttl) {
+ super(name, OptionValue.Kind.STRING);
assert ttl > 0;
this.ttl = ttl;
}
@@ -112,6 +111,11 @@ public final class ExecutionControls {
.build(logger);
}
}
+
+ public void loadDefault(DrillConfig bootConfig){
+ OptionValue value = OptionValue.createString(OptionType.SYSTEM, getOptionName(), bootConfig.getString(getConfigProperty()), OptionValue.OptionScope.BOOT);
+ setDefaultValue(value);
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/main/java/org/apache/drill/exec/util/MemoryAllocationUtilities.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/MemoryAllocationUtilities.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/MemoryAllocationUtilities.java
index 4580222..48724a4 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/MemoryAllocationUtilities.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/MemoryAllocationUtilities.java
@@ -58,7 +58,9 @@ public class MemoryAllocationUtilities {
// if there are any sorts, compute the maximum allocation, and set it on them
if (bufferedOpList.size() > 0) {
final OptionManager optionManager = queryContext.getOptions();
- final long maxWidthPerNode = optionManager.getOption(ExecConstants.MAX_WIDTH_PER_NODE_KEY).num_val;
+ double cpu_load_average = optionManager.getOption(ExecConstants.CPU_LOAD_AVERAGE);
+ final long maxWidth = optionManager.getOption(ExecConstants.MAX_WIDTH_PER_NODE);
+ final long maxWidthPerNode = ExecConstants.MAX_WIDTH_PER_NODE.computeMaxWidth(cpu_load_average,maxWidth);
long maxAllocPerNode = Math.min(DrillConfig.getMaxDirectMemory(),
queryContext.getConfig().getLong(RootAllocatorFactory.TOP_LEVEL_MAX_ALLOC));
maxAllocPerNode = Math.min(maxAllocPerNode,
http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/main/resources/drill-module.conf
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/resources/drill-module.conf b/exec/java-exec/src/main/resources/drill-module.conf
index 41ecc95..f387cda 100644
--- a/exec/java-exec/src/main/resources/drill-module.conf
+++ b/exec/java-exec/src/main/resources/drill-module.conf
@@ -349,3 +349,150 @@ drill.exec: {
drill.jdbc: {
batch_queue_throttling_threshold: 100
}
+
+# The following are defaults for system and session options.
+# Provided here for easy customization in Drill distributions
+# like the drill-distrib.conf file.
+# An addition of new system option should go through following procedure.
+# Before adding a system option a validator should be added for that
+# option and then the option should be configured below under the name
+# space drill.exec.options. When the system options are being loaded
+# options config options are read to find the default value for a give option
+# if the option is not set using ALTER SYSTEM or ALTER SESSION.But
+# if the option is not configured in the conf file under the name space
+# "drill.exec.options", option missing exception will be thrown.
+# Users are not supposed to set these options in the drill-override.conf file.
+# Users should use ALTER SYSTEM and ALTER SESSION to set the options.
+
+drill.exec.options: {
+ bootstrap-storage-plugins.json: .sys.drill,
+ debug.validate_iterators : false,
+ debug.validate_vectors :false,
+ drill.exec.functions.cast_empty_string_to_null: false,
+ drill.exec.hashagg.min_batches_per_partition : 3,
+ drill.exec.storage.file.partition.column.label: "dir",
+ drill.exec.storage.implicit.filename.column.label: "filename",
+ drill.exec.storage.implicit.filepath.column.label: "filepath",
+ drill.exec.storage.implicit.fqn.column.label: "fqn",
+ drill.exec.storage.implicit.suffix.column.label: "suffix",
+ drill.exec.testing.controls: "{}",
+ exec.bulk_load_table_list.bulk_size: 1000,
+ exec.compile.scalar_replacement: false,
+ exec.enable_bulk_load_table_list: false,
+ exec.enable_union_type: false,
+ exec.errors.verbose: false,
+ exec.hashagg.mem_limit : 0,
+ exec.hashagg.num_partitions :32,
+ exec.impersonation.inbound_policies: "[]",
+ exec.java.compiler.exp_in_method_size: 50,
+ exec.java_compiler : "DEFAULT",
+ exec.java_compiler_debug :true,
+ exec.java_compiler_janino_maxsize : 262144,
+ exec.max_hash_table_size: 1073741824,
+ exec.min_hash_table_size: 65536,
+ exec.persistent_table.umask: "002",
+ exec.query.progress.update: true,
+ exec.query_profile.debug_mode: false,
+ exec.query_profile.save: true,
+ exec.queue.enable: false,
+ exec.queue.large: 10,
+ exec.queue.small: 100,
+ exec.queue.threshold: 30000000,
+ exec.queue.timeout_millis: 300000,
+ exec.sort.disable_managed : false,
+ exec.storage.enable_new_text_reader: true,
+ exec.udf.enable_dynamic_support: true,
+ exec.udf.use_dynamic: true,
+ new_view_default_permissions: 700,
+ org.apache.drill.exec.compile.ClassTransformer.scalar_replacement : "try",
+ planner.add_producer_consumer:false,
+ planner.affinity_factor: 1.2,
+ planner.broadcast_factor:1.0,
+ planner.broadcast_threshold:10000000,
+ planner.cpu_load_average : 0.70,
+ planner.disable_exchanges:false,
+ planner.enable_broadcast_join:true,
+ planner.enable_constant_folding:true,
+ planner.enable_decimal_data_type:false,
+ planner.enable_demux_exchange:false,
+ planner.enable_hash_single_key:true,
+ planner.enable_hashagg:true,
+ planner.enable_hashjoin:true,
+ planner.enable_hashjoin_swap:true,
+ planner.enable_hep_opt:true,
+ planner.enable_hep_partition_pruning:true,
+ planner.enable_join_optimization: true,
+ planner.enable_limit0_optimization: false,
+ planner.enable_mergejoin:true,
+ planner.enable_multiphase_agg:true,
+ planner.enable_mux_exchange:true,
+ planner.enable_nestedloopjoin:true,
+ planner.enable_nljoin_for_scalar_only:true,
+ planner.enable_streamagg:true,
+ planner.enable_type_inference: true,
+ planner.enable_unionall_distribute:false,
+ planner.filter.max_selectivity_estimate_factor:1.0,
+ planner.filter.min_selectivity_estimate_factor:0.0,
+ planner.force_2phase_aggr : false,
+ planner.identifier_max_length:1024,
+ planner.in_subquery_threshold: 20,
+ planner.join.hash_join_swap_margin_factor:10,
+ planner.join.row_count_estimate_factor:1.0,
+ planner.memory.average_field_width: 8,
+ planner.memory.enable_memory_estimation: false,
+ planner.memory.hash_agg_table_factor: 1.1d,
+ planner.memory.hash_join_table_factor: 1.1d,
+ planner.memory.max_query_memory_per_node: 2147483648,
+ planner.memory.min_memory_per_buffered_op: 41943040,
+ planner.memory.non_blocking_operators_memory: 64,
+ planner.memory_limit:268435456,
+ planner.nestedloopjoin_factor:100.0,
+ planner.parser.quoting_identifiers :"`",
+ planner.partitioner_sender_max_threads:8,
+ planner.partitioner_sender_set_threads:-1,
+ planner.partitioner_sender_threads_factor:2,
+ planner.producer_consumer_queue_size:10,
+ planner.slice_target: 100000,
+ planner.store.parquet.rowgroup.filter.pushdown.enabled : true,
+ planner.store.parquet.rowgroup.filter.pushdown.threshold : 10000,
+ # Max per node should always be configured as zero and
+ # it is dynamically computed based on cpu_load_average
+ planner.width.max_per_node: 0,
+ planner.width.max_per_query: 1000,
+ prepare.statement.create_timeout_ms: 10000,
+ security.admin.user_groups: true,
+ security.admin.users: true,
+ store.format: "parquet",
+ store.hive.optimize_scan_with_native_readers: false,
+ store.json.all_text_mode: false,
+ store.json.extended_types: false,
+ store.json.read_numbers_as_double: false,
+ store.json.reader.print_skipped_invalid_record_number: false,
+ store.json.reader.skip_invalid_records: false,
+ store.json.writer.skip_null_fields: true,
+ store.json.writer.uglify: false,
+ store.mongo.all_text_mode: false,
+ store.mongo.bson.record.reader: true,
+ store.mongo.read_numbers_as_double: false,
+ store.parquet.block-size: 536870912,
+ store.parquet.compression: "none",
+ store.parquet.dictionary.page-size: 1048576,
+ store.parquet.enable_dictionary_encoding: false,
+ store.parquet.page-size: 1048576,
+ store.parquet.reader.columnreader.async: false,
+ store.parquet.reader.int96_as_timestamp: false,
+ store.parquet.reader.pagereader.async: true,
+ store.parquet.reader.pagereader.bufferedread: true,
+ store.parquet.reader.pagereader.buffersize: 1048576,
+ store.parquet.reader.pagereader.enforceTotalSize: false,
+ store.parquet.reader.pagereader.queuesize: 2,
+ store.parquet.reader.pagereader.usefadvise: false,
+ store.parquet.use_new_reader: false,
+ store.parquet.vector_fill_check_threshold: 10,
+ store.parquet.vector_fill_threshold: 85,
+ store.parquet.writer.use_single_fs_block : false,
+ store.partition.hash_distribute: false,
+ store.text.estimated_row_size_bytes: 100.0,
+ web.logs.max_lines: 10000,
+ window.enable: true,
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java b/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java
index df0c89f..b00d4d6 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java
@@ -80,7 +80,7 @@ public class PlanningBase extends ExecTest{
provider.start();
final ScanResult scanResult = ClassPathScanner.fromPrescan(config);
final LogicalPlanPersistence logicalPlanPersistence = new LogicalPlanPersistence(config, scanResult);
- final SystemOptionManager systemOptions = new SystemOptionManager(logicalPlanPersistence , provider);
+ final SystemOptionManager systemOptions = new SystemOptionManager(logicalPlanPersistence , provider, config);
systemOptions.init();
@SuppressWarnings("resource")
final UserSession userSession = UserSession.Builder.newBuilder().withOptionManager(systemOptions).build();
http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/test/java/org/apache/drill/QueryTestUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/QueryTestUtil.java b/exec/java-exec/src/test/java/org/apache/drill/QueryTestUtil.java
index 54ae774..8e767e4 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/QueryTestUtil.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/QueryTestUtil.java
@@ -41,6 +41,7 @@ import org.apache.drill.exec.server.DrillbitContext;
import org.apache.drill.exec.server.RemoteServiceSet;
import org.apache.drill.exec.server.options.OptionManager;
import org.apache.drill.exec.server.options.OptionValue;
+import org.apache.drill.exec.server.options.OptionValue.OptionScope;
import org.apache.drill.exec.util.VectorUtil;
/**
@@ -190,7 +191,7 @@ public class QueryTestUtil {
final OptionManager optionManager = drillbitContext.getOptionManager();
final OptionValue originalOptionValue = optionManager.getOption(ClassTransformer.SCALAR_REPLACEMENT_OPTION);
final OptionValue newOptionValue = OptionValue.createString(OptionValue.OptionType.SYSTEM,
- ClassTransformer.SCALAR_REPLACEMENT_OPTION, srOption.name().toLowerCase());
+ ClassTransformer.SCALAR_REPLACEMENT_OPTION, srOption.name().toLowerCase(), OptionScope.SYSTEM);
optionManager.setOption(newOptionValue);
// flush the code cache
http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/test/java/org/apache/drill/exec/ExecTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/ExecTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/ExecTest.java
index dead858..7dd7680 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/ExecTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/ExecTest.java
@@ -65,7 +65,7 @@ public class ExecTest extends DrillTest {
public static void setupOptionManager() throws Exception{
final LocalPersistentStoreProvider provider = new LocalPersistentStoreProvider(c);
provider.start();
- optionManager = new SystemOptionManager(PhysicalPlanReaderTestFactory.defaultLogicalPlanPersistence(c), provider);
+ optionManager = new SystemOptionManager(PhysicalPlanReaderTestFactory.defaultLogicalPlanPersistence(c), provider,c);
optionManager.init();
}
http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/test/java/org/apache/drill/exec/compile/CodeCompilerTestFactory.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/CodeCompilerTestFactory.java b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/CodeCompilerTestFactory.java
index f4903d8..7b74a3a 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/CodeCompilerTestFactory.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/CodeCompilerTestFactory.java
@@ -30,7 +30,7 @@ public class CodeCompilerTestFactory {
DrillConfig config = checkNotNull(c);
LogicalPlanPersistence persistence = new LogicalPlanPersistence(config, ClassPathScanner.fromPrescan(config));
LocalPersistentStoreProvider provider = new LocalPersistentStoreProvider(config);
- SystemOptionManager systemOptionManager = new SystemOptionManager(persistence, provider);
+ SystemOptionManager systemOptionManager = new SystemOptionManager(persistence, provider, config);
return new CodeCompiler(config, systemOptionManager.init());
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java
index 7728aae..3c8ca01 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java
@@ -29,6 +29,7 @@ import org.apache.drill.exec.expr.CodeGenerator;
import org.apache.drill.exec.rpc.user.UserSession;
import org.apache.drill.exec.server.options.OptionValue;
import org.apache.drill.exec.server.options.OptionValue.OptionType;
+import org.apache.drill.exec.server.options.OptionValue.OptionScope;
import org.apache.drill.exec.server.options.SessionOptionManager;
import org.codehaus.commons.compiler.CompileException;
import org.junit.Assert;
@@ -58,7 +59,7 @@ public class TestClassTransformation extends BaseTestQuery {
@Test
public void testJaninoClassCompiler() throws Exception {
logger.debug("Testing JaninoClassCompiler");
- sessionOptions.setOption(OptionValue.createString(OptionType.SESSION, ClassCompilerSelector.JAVA_COMPILER_OPTION, ClassCompilerSelector.CompilerPolicy.JANINO.name()));
+ sessionOptions.setOption(OptionValue.createString(OptionType.SESSION, ClassCompilerSelector.JAVA_COMPILER_OPTION, ClassCompilerSelector.CompilerPolicy.JANINO.name(), OptionScope.SESSION));
for (int i = 0; i < ITERATION_COUNT; i++) {
compilationInnerClass(false); // Traditional byte-code manipulation
compilationInnerClass(true); // Plain-old Java
@@ -68,7 +69,8 @@ public class TestClassTransformation extends BaseTestQuery {
@Test
public void testJDKClassCompiler() throws Exception {
logger.debug("Testing JDKClassCompiler");
- sessionOptions.setOption(OptionValue.createString(OptionType.SESSION, ClassCompilerSelector.JAVA_COMPILER_OPTION, ClassCompilerSelector.CompilerPolicy.JDK.name()));
+ OptionType type = OptionType.SESSION;
+ sessionOptions.setOption(OptionValue.createString(type, ClassCompilerSelector.JAVA_COMPILER_OPTION, ClassCompilerSelector.CompilerPolicy.JDK.name(), OptionScope.SESSION));
for (int i = 0; i < ITERATION_COUNT; i++) {
compilationInnerClass(false); // Traditional byte-code manipulation
compilationInnerClass(true); // Plain-old Java
@@ -80,9 +82,10 @@ public class TestClassTransformation extends BaseTestQuery {
CodeGenerator<ExampleInner> cg = newCodeGenerator(ExampleInner.class, ExampleTemplateWithInner.class);
ClassSet classSet = new ClassSet(null, cg.getDefinition().getTemplateClassName(), cg.getMaterializedClassName());
String sourceCode = cg.generateAndGet();
- sessionOptions.setOption(OptionValue.createString(OptionType.SESSION, ClassCompilerSelector.JAVA_COMPILER_OPTION, ClassCompilerSelector.CompilerPolicy.JDK.name()));
+ OptionType type = OptionType.SESSION;
+ sessionOptions.setOption(OptionValue.createString(type, ClassCompilerSelector.JAVA_COMPILER_OPTION, ClassCompilerSelector.CompilerPolicy.JDK.name(), OptionScope.SESSION));
- sessionOptions.setOption(OptionValue.createBoolean(OptionType.SESSION, ClassCompilerSelector.JAVA_COMPILER_DEBUG_OPTION, false));
+ sessionOptions.setOption(OptionValue.createBoolean(type, ClassCompilerSelector.JAVA_COMPILER_DEBUG_OPTION, false, OptionScope.SESSION));
@SuppressWarnings("resource")
QueryClassLoader loader = new QueryClassLoader(config, sessionOptions);
final byte[][] codeWithoutDebug = loader.getClassByteCode(classSet.generated, sourceCode);
@@ -92,7 +95,7 @@ public class TestClassTransformation extends BaseTestQuery {
sizeWithoutDebug += bs.length;
}
- sessionOptions.setOption(OptionValue.createBoolean(OptionType.SESSION, ClassCompilerSelector.JAVA_COMPILER_DEBUG_OPTION, true));
+ sessionOptions.setOption(OptionValue.createBoolean(type, ClassCompilerSelector.JAVA_COMPILER_DEBUG_OPTION, true, OptionScope.SESSION));
loader = new QueryClassLoader(config, sessionOptions);
final byte[][] codeWithDebug = loader.getClassByteCode(classSet.generated, sourceCode);
loader.close();
http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/test/java/org/apache/drill/exec/compile/bytecode/ReplaceMethodInvoke.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/bytecode/ReplaceMethodInvoke.java b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/bytecode/ReplaceMethodInvoke.java
index 9b507fe..1c36c46 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/bytecode/ReplaceMethodInvoke.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/bytecode/ReplaceMethodInvoke.java
@@ -56,7 +56,7 @@ public class ReplaceMethodInvoke {
check(output);
final DrillConfig c = DrillConfig.forClient();
- final SystemOptionManager m = new SystemOptionManager(PhysicalPlanReaderTestFactory.defaultLogicalPlanPersistence(c), new LocalPersistentStoreProvider(c));
+ final SystemOptionManager m = new SystemOptionManager(PhysicalPlanReaderTestFactory.defaultLogicalPlanPersistence(c), new LocalPersistentStoreProvider(c), c);
m.init();
try (QueryClassLoader ql = new QueryClassLoader(DrillConfig.create(), m)) {
ql.injectByteCode("org.apache.drill.Pickle$OutgoingBatch", output);
http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestInboundImpersonationPrivileges.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestInboundImpersonationPrivileges.java b/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestInboundImpersonationPrivileges.java
index 0d5393e..f9a6358 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestInboundImpersonationPrivileges.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestInboundImpersonationPrivileges.java
@@ -49,7 +49,7 @@ public class TestInboundImpersonationPrivileges extends BaseTestImpersonation {
ExecConstants.IMPERSONATION_POLICY_VALIDATOR.validate(
OptionValue.createString(OptionValue.OptionType.SYSTEM,
ExecConstants.IMPERSONATION_POLICIES_KEY,
- IMPERSONATION_POLICIES), null);
+ IMPERSONATION_POLICIES,OptionValue.OptionScope.SYSTEM), null);
try {
return InboundImpersonationManager.hasImpersonationPrivileges(proxyName, targetName, IMPERSONATION_POLICIES);
} catch (final Exception e) {
http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/partitionsender/TestPartitionSender.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/partitionsender/TestPartitionSender.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/partitionsender/TestPartitionSender.java
index 82241d7..598bdc2 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/partitionsender/TestPartitionSender.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/partitionsender/TestPartitionSender.java
@@ -63,6 +63,7 @@ import org.apache.drill.exec.server.DrillbitContext;
import org.apache.drill.exec.server.options.OptionList;
import org.apache.drill.exec.server.options.OptionValue;
import org.apache.drill.exec.server.options.OptionValue.OptionType;
+import org.apache.drill.exec.server.options.OptionValue.OptionScope;
import org.apache.drill.exec.util.Utilities;
import org.apache.drill.exec.work.QueryWorkUnit;
import org.junit.AfterClass;
@@ -180,20 +181,20 @@ public class TestPartitionSender extends PlanTestBase {
final OptionList options = new OptionList();
// try multiple scenarios with different set of options
- options.add(OptionValue.createLong(OptionType.SESSION, "planner.slice_target", 1));
+ options.add(OptionValue.createLong(OptionType.SESSION, "planner.slice_target", 1, OptionScope.SESSION));
testThreadsHelper(hashToRandomExchange, drillbitContext, options,
incoming, registry, planReader, planningSet, rootFragment, 1);
options.clear();
- options.add(OptionValue.createLong(OptionType.SESSION, "planner.slice_target", 1));
- options.add(OptionValue.createLong(OptionType.SESSION, "planner.partitioner_sender_max_threads", 10));
+ options.add(OptionValue.createLong(OptionType.SESSION, "planner.slice_target", 1, OptionScope.SESSION));
+ options.add(OptionValue.createLong(OptionType.SESSION, "planner.partitioner_sender_max_threads", 10, OptionScope.SESSION));
hashToRandomExchange.setCost(1000);
testThreadsHelper(hashToRandomExchange, drillbitContext, options,
incoming, registry, planReader, planningSet, rootFragment, 10);
options.clear();
- options.add(OptionValue.createLong(OptionType.SESSION, "planner.slice_target", 1000));
- options.add(OptionValue.createLong(OptionType.SESSION, "planner.partitioner_sender_threads_factor",2));
+ options.add(OptionValue.createLong(OptionType.SESSION, "planner.slice_target", 1000, OptionScope.SESSION));
+ options.add(OptionValue.createLong(OptionType.SESSION, "planner.partitioner_sender_threads_factor",2, OptionScope.SESSION));
hashToRandomExchange.setCost(14000);
testThreadsHelper(hashToRandomExchange, drillbitContext, options,
incoming, registry, planReader, planningSet, rootFragment, 2);
http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/test/java/org/apache/drill/exec/testing/ControlsInjectionUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/testing/ControlsInjectionUtil.java b/exec/java-exec/src/test/java/org/apache/drill/exec/testing/ControlsInjectionUtil.java
index 5365773..39c9ea3 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/testing/ControlsInjectionUtil.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/testing/ControlsInjectionUtil.java
@@ -32,6 +32,7 @@ import org.apache.drill.exec.rpc.user.UserSession;
import org.apache.drill.exec.rpc.user.UserSession.QueryCountIncrementer;
import org.apache.drill.exec.server.options.OptionManager;
import org.apache.drill.exec.server.options.OptionValue;
+import org.apache.drill.exec.server.options.OptionValue.OptionScope;
/**
* Static methods for constructing exception and pause injections for testing purposes.
@@ -71,7 +72,7 @@ public class ControlsInjectionUtil {
public static void setControls(final UserSession session, final String controls) {
validateControlsString(controls);
final OptionValue opValue = OptionValue.createString(OptionValue.OptionType.SESSION,
- DRILLBIT_CONTROL_INJECTIONS, controls);
+ DRILLBIT_CONTROL_INJECTIONS, controls, OptionScope.SESSION);
final OptionManager options = session.getOptions();
try {
http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/test/java/org/apache/drill/exec/work/metadata/TestMetadataProvider.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/work/metadata/TestMetadataProvider.java b/exec/java-exec/src/test/java/org/apache/drill/exec/work/metadata/TestMetadataProvider.java
index f3bd63a..9ed3527 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/work/metadata/TestMetadataProvider.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/work/metadata/TestMetadataProvider.java
@@ -152,7 +152,7 @@ public class TestMetadataProvider extends BaseTestQuery {
assertEquals(RequestStatus.OK, resp.getStatus());
List<TableMetadata> tables = resp.getTablesList();
- assertEquals(11, tables.size());
+ assertEquals(12, tables.size());
verifyTable("INFORMATION_SCHEMA", "CATALOGS", tables);
verifyTable("INFORMATION_SCHEMA", "COLUMNS", tables);
@@ -186,7 +186,7 @@ public class TestMetadataProvider extends BaseTestQuery {
assertEquals(RequestStatus.OK, resp.getStatus());
List<TableMetadata> tables = resp.getTablesList();
- assertEquals(11, tables.size());
+ assertEquals(12, tables.size());
verifyTable("INFORMATION_SCHEMA", "CATALOGS", tables);
verifyTable("INFORMATION_SCHEMA", "COLUMNS", tables);
@@ -211,7 +211,7 @@ public class TestMetadataProvider extends BaseTestQuery {
assertEquals(RequestStatus.OK, resp.getStatus());
List<TableMetadata> tables = resp.getTablesList();
- assertEquals(4, tables.size());
+ assertEquals(5, tables.size());
verifyTable("sys", "boot", tables);
verifyTable("sys", "memory", tables);
@@ -242,7 +242,7 @@ public class TestMetadataProvider extends BaseTestQuery {
assertEquals(RequestStatus.OK, resp.getStatus());
List<ColumnMetadata> columns = resp.getColumnsList();
- assertEquals(71, columns.size());
+ assertEquals(76, columns.size());
// too many records to verify the output.
}
http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/test/java/org/apache/drill/test/FixtureBuilder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/FixtureBuilder.java b/exec/java-exec/src/test/java/org/apache/drill/test/FixtureBuilder.java
index b305609..d6bb378 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/FixtureBuilder.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/FixtureBuilder.java
@@ -49,6 +49,7 @@ public class FixtureBuilder {
public static final int DEFAULT_ZK_REFRESH = 500; // ms
public static final int DEFAULT_SERVER_RPC_THREADS = 10;
public static final int DEFAULT_SCAN_THREADS = 8;
+ public static final String OPTION_DEFAULTS_ROOT = "drill.exec.options.";
protected ConfigBuilder configBuilder = new ConfigBuilder();
protected List<RuntimeOption> sessionOptions;
@@ -97,6 +98,14 @@ public class FixtureBuilder {
}
/**
+ *
+ */
+ public FixtureBuilder setOptionDefault(String key, Object value) {
+ String option_name = OPTION_DEFAULTS_ROOT + key;
+ configBuilder().put(option_name, value.toString());
+ return this;
+ }
+ /**
* Add an additional boot-time property for the embedded Drillbit.
* @param key config property name
* @param value property value
http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/test/java/org/apache/drill/test/OperatorFixture.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/OperatorFixture.java b/exec/java-exec/src/test/java/org/apache/drill/test/OperatorFixture.java
index 976812c..4a63bb5 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/OperatorFixture.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/OperatorFixture.java
@@ -46,6 +46,7 @@ import org.apache.drill.exec.server.options.BaseOptionManager;
import org.apache.drill.exec.server.options.OptionSet;
import org.apache.drill.exec.server.options.OptionValue;
import org.apache.drill.exec.server.options.OptionValue.OptionType;
+import org.apache.drill.exec.server.options.OptionValue.OptionScope;
import org.apache.drill.exec.testing.ExecutionControls;
import org.apache.drill.test.rowSet.DirectRowSet;
import org.apache.drill.test.rowSet.HyperRowSetImpl;
@@ -127,19 +128,19 @@ public class OperatorFixture extends BaseFixture implements AutoCloseable {
}
public void set(String key, long value) {
- values.put(key, OptionValue.createLong(OptionType.SYSTEM, key, value));
+ values.put(key, OptionValue.createLong(OptionType.SYSTEM, key, value, OptionScope.SYSTEM));
}
public void set(String key, boolean value) {
- values.put(key, OptionValue.createBoolean(OptionType.SYSTEM, key, value));
+ values.put(key, OptionValue.createBoolean(OptionType.SYSTEM, key, value, OptionScope.SYSTEM));
}
public void set(String key, double value) {
- values.put(key, OptionValue.createDouble(OptionType.SYSTEM, key, value));
+ values.put(key, OptionValue.createDouble(OptionType.SYSTEM, key, value, OptionScope.SYSTEM));
}
public void set(String key, String value) {
- values.put(key, OptionValue.createString(OptionType.SYSTEM, key, value));
+ values.put(key, OptionValue.createString(OptionType.SYSTEM, key, value, OptionScope.SYSTEM));
}
@Override
http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/test/java/org/apache/drill/test/TestConfigLinkage.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/TestConfigLinkage.java b/exec/java-exec/src/test/java/org/apache/drill/test/TestConfigLinkage.java
new file mode 100644
index 0000000..19f2f06
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/TestConfigLinkage.java
@@ -0,0 +1,209 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.drill.test;
+
+import org.apache.drill.exec.ExecConstants;
+import org.junit.Assert;
+import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+/*
+ * Tests to test if the linkage between the two config option systems
+ * i.e., the linkage between boot-config system and system/session options.
+ * Tests to assert if the config options are read in the order of session , system, boot-config.
+ * Max width per node is slightly different from other options since it is set to zero by default
+ * in the config and the option value is computed dynamically everytime if the value is zero
+ * i.e., if the value is not set in system/session.
+ * */
+
+public class TestConfigLinkage {
+
+ /* Test if session option takes precedence */
+ @Test
+ public void testSessionOption() throws Exception {
+ FixtureBuilder builder = ClusterFixture.builder().sessionOption(ExecConstants.SLICE_TARGET, 10);
+ try (ClusterFixture cluster = builder.build();
+ ClientFixture client = cluster.clientFixture()) {
+ String slice_target = client.queryBuilder().sql("SELECT val FROM sys.options2 where name='planner.slice_target'").singletonString();
+ assertEquals(slice_target, "10");
+ }
+ }
+
+ /* Test if system option takes precedence over the boot option */
+ @Test
+ public void testSystemOption() throws Exception {
+ FixtureBuilder builder = ClusterFixture.builder().systemOption(ExecConstants.SLICE_TARGET, 10000);
+ try (ClusterFixture cluster = builder.build();
+ ClientFixture client = cluster.clientFixture()) {
+ String slice_target = client.queryBuilder().sql("SELECT val FROM sys.options2 where name='planner.slice_target'").singletonString();
+ assertEquals(slice_target, "10000");
+ }
+ }
+
+ /* Test if config option takes effect if system/session are not set */
+ @Test
+ public void testConfigOption() throws Exception {
+ FixtureBuilder builder = ClusterFixture.builder()
+ .setOptionDefault(ExecConstants.SLICE_TARGET, 1000);
+ try (ClusterFixture cluster = builder.build();
+ ClientFixture client = cluster.clientFixture()) {
+ String slice_target = client.queryBuilder().sql("SELECT val FROM sys.options2 where name='planner.slice_target'").singletonString();
+ assertEquals(slice_target, "1000");
+ }
+ }
+
+ /* Test if altering system option takes precedence over config option */
+ @Test
+ public void testAlterSystem() throws Exception {
+ try (ClusterFixture cluster = ClusterFixture.standardCluster();
+ ClientFixture client = cluster.clientFixture()) {
+ client.queryBuilder().sql("ALTER SYSTEM SET `planner.slice_target` = 10000").run();
+ String slice_target = client.queryBuilder().sql("SELECT val FROM sys.options2 where name='planner.slice_target'").singletonString();
+ assertEquals(slice_target, "10000");
+ }
+ }
+
+ /* Test if altering session option takes precedence over system option */
+ @Test
+ public void testSessionPrecedence() throws Exception {
+ FixtureBuilder builder = ClusterFixture.builder().systemOption(ExecConstants.SLICE_TARGET, 100000);
+ try (ClusterFixture cluster = builder.build();
+ ClientFixture client = cluster.clientFixture()) {
+ client.queryBuilder().sql("ALTER SESSION SET `planner.slice_target` = 10000").run();
+ String slice_target = client.queryBuilder().sql("SELECT val FROM sys.options2 where name='planner.slice_target'").singletonString();
+ assertEquals(slice_target, "10000");
+ }
+ }
+
+ /* Test if setting maxwidth option through config takes effect */
+ @Test
+ public void testMaxWidthPerNodeConfig() throws Exception {
+ FixtureBuilder builder = ClusterFixture.bareBuilder().setOptionDefault(ExecConstants.MAX_WIDTH_PER_NODE_KEY, 2);
+ try (ClusterFixture cluster = builder.build();
+ ClientFixture client = cluster.clientFixture()) {
+ String maxWidth = client.queryBuilder().sql("SELECT val FROM sys.options2 where name='planner.width.max_per_node'").singletonString();
+ assertEquals("2", maxWidth);
+ }
+ }
+
+ /* Test if setting maxwidth at system level takes precedence */
+ @Test
+ public void testMaxWidthPerNodeSystem() throws Exception {
+ FixtureBuilder builder = ClusterFixture.bareBuilder().systemOption(ExecConstants.MAX_WIDTH_PER_NODE_KEY, 3);
+ try (ClusterFixture cluster = builder.build();
+ ClientFixture client = cluster.clientFixture()) {
+ String maxWidth = client.queryBuilder().sql("SELECT val FROM sys.options2 where name='planner.width.max_per_node'").singletonString();
+ assertEquals("3", maxWidth);
+ }
+ }
+
+ /* Test if setting maxwidth at session level takes precedence */
+ @Test
+ public void testMaxWidthPerNodeSession() throws Exception {
+ FixtureBuilder builder = ClusterFixture.bareBuilder().sessionOption(ExecConstants.MAX_WIDTH_PER_NODE_KEY, 2);
+ try (ClusterFixture cluster = builder.build();
+ ClientFixture client = cluster.clientFixture()) {
+ String maxWidth = client.queryBuilder().sql("SELECT val FROM sys.options2 where name='planner.width.max_per_node'").singletonString();
+ assertEquals("2", maxWidth);
+ }
+ }
+
+ /* Test if max width is computed correctly using the cpu load average
+ when the option is not set at either system or session level
+ */
+ @Test
+ public void testMaxWidthPerNodeDefault() throws Exception {
+ FixtureBuilder builder = ClusterFixture.bareBuilder().setOptionDefault(ExecConstants.CPU_LOAD_AVERAGE_KEY, 0.70);
+ try (ClusterFixture cluster = builder.build();
+ ClientFixture client = cluster.clientFixture()) {
+ long maxWidth = ExecConstants.MAX_WIDTH_PER_NODE.computeMaxWidth(0.70, 0);
+ int availProc = Runtime.getRuntime().availableProcessors();
+ long maxWidthPerNode = Math.max(1, Math.min(availProc, Math.round(availProc * 0.70)));
+ assertEquals(maxWidthPerNode, maxWidth);
+ }
+ }
+
+ /* Test if the scope is set during BOOT time and scope is actually BOOT */
+ @Test
+ public void testScope() throws Exception {
+ FixtureBuilder builder = ClusterFixture.bareBuilder().setOptionDefault(ExecConstants.SLICE_TARGET, 100000);
+ try (ClusterFixture cluster = builder.build();
+ ClientFixture client = cluster.clientFixture()) {
+ String scope = client.queryBuilder()
+ .sql("SELECT optionScope from sys.options2 where name='planner.slice_target'")
+ .singletonString();
+ Assert.assertEquals("BOOT",scope);
+ }
+ }
+
+ /* Test if the option is set at SYSTEM scope and the scope is actually SYSTEM */
+ @Test
+ public void testScopeSystem() throws Exception {
+ FixtureBuilder builder = ClusterFixture.bareBuilder().systemOption(ExecConstants.SLICE_TARGET, 10000);
+ try (ClusterFixture cluster = builder.build();
+ ClientFixture client = cluster.clientFixture()) {
+ String scope = client.queryBuilder()
+ .sql("SELECT optionScope from sys.options2 where name='planner.slice_target'")
+ .singletonString();
+ Assert.assertEquals("SYSTEM",scope);
+ }
+ }
+
+ /* Test if the option is set at SESSION scope and the scope is actually SESSION */
+ @Test
+ public void testScopeSession() throws Exception {
+ FixtureBuilder builder = ClusterFixture.bareBuilder().sessionOption(ExecConstants.SLICE_TARGET, 100000);
+ try (ClusterFixture cluster = builder.build();
+ ClientFixture client = cluster.clientFixture()) {
+ String scope = client.queryBuilder()
+ .sql("SELECT optionScope from sys.options2 where name='planner.slice_target'")
+ .singletonString();
+ Assert.assertEquals("SESSION",scope);
+ }
+ }
+
+ /* Test if the option is altered at SYSTEM scope and the scope is actually SYSTEM */
+ @Test
+ public void testScopeAlterSystem() throws Exception {
+ FixtureBuilder builder = ClusterFixture.bareBuilder();
+ try (ClusterFixture cluster = builder.build();
+ ClientFixture client = cluster.clientFixture()) {
+ client.queryBuilder().sql("ALTER SYSTEM set `planner.slice_target`= 10000").run();
+ String scope = client.queryBuilder()
+ .sql("SELECT optionScope from sys.options2 where name='planner.slice_target'")
+ .singletonString();
+ Assert.assertEquals("SYSTEM",scope);
+ }
+ }
+
+ /* Test if the option is altered at SESSION scope and the scope is actually SESSION */
+ @Test
+ public void testScopeAlterSession() throws Exception {
+ FixtureBuilder builder = ClusterFixture.bareBuilder();
+ try (ClusterFixture cluster = builder.build();
+ ClientFixture client = cluster.clientFixture()) {
+ client.queryBuilder().sql("ALTER SESSION set `planner.slice_target`= 10000").run();
+ String scope = client.queryBuilder()
+ .sql("SELECT optionScope from sys.options2 where name='planner.slice_target'")
+ .singletonString();
+ Assert.assertEquals("SESSION",scope);
+ }
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcMetadata.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcMetadata.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcMetadata.java
index b859650..913834b 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcMetadata.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcMetadata.java
@@ -81,7 +81,7 @@ public class TestJdbcMetadata extends JdbcTestActionBase {
public ResultSet getResult(Connection c) throws SQLException {
return c.getMetaData().getTables("DRILL", "sys", "opt%", new String[]{"SYSTEM_TABLE", "SYSTEM_VIEW"});
}
- }, 1);
+ }, 2);
}
@Test
@@ -101,6 +101,6 @@ public class TestJdbcMetadata extends JdbcTestActionBase {
public ResultSet getResult(Connection c) throws SQLException {
return c.getMetaData().getColumns("DRILL", "sys", "opt%", "%ame");
}
- }, 1);
+ }, 2);
}
}
|