http://git-wip-us.apache.org/repos/asf/drill/blob/6adeb986/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValue.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValue.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValue.java
index 5883480..0a49f5d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValue.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValue.java
@@ -25,15 +25,18 @@ import com.fasterxml.jackson.annotation.JsonProperty;
import org.apache.drill.exec.store.sys.PersistentStore;
import com.google.common.base.Preconditions;
+import java.util.EnumSet;
+
/**
* <p>
- * An {@link OptionValue option value} is used by an {@link OptionManager} to store a run-time setting. This setting,
- * for example, could affect a query in execution stage. Instances of this class are JSON serializable and can be stored
+ * An {@link OptionValue option value} is used internally by an {@link OptionManager} to store a run-time setting. This setting,
+ * for example, could affect a query in an execution stage. Instances of this class are JSON serializable and can be stored
* in a {@link PersistentStore persistent store} (see {@link SystemOptionManager#options}), or
* in memory (see {@link InMemoryOptionManager#options}).
* </p>
- *<p> OptionType defines the permission level of the option.If it can be set at System level or Session level or so on.
- * Whereas OptionScope defines the scope at which the option is being set. If the option is being set at the BOOT time
+ * <p>
+ * {@link AccessibleScopes} defines the scopes at which the option can be set. If it can be set at System level or Session level or so on.
+ * Whereas {@link OptionScope} defines the scope at which the option is being set. If the option is being set at the BOOT time
* the scope of the option is BOOT. If it is set at SYSTEM level the scope is SYSTEM. Although they look similar there
* is a fine level which differentiates both of them which is at which level of hierarchy they can be set and
* at what at level of hierarchy they were actually set.
@@ -42,61 +45,82 @@ import com.google.common.base.Preconditions;
@JsonInclude(Include.NON_NULL)
public class OptionValue implements Comparable<OptionValue> {
- public enum OptionType {
- BOOT, SYSTEM, SESSION, QUERY
+ /**
+ * Defines where an option can be configured.
+ */
+ public enum AccessibleScopes {
+ BOOT(EnumSet.of(OptionScope.BOOT)), // Can only be configured at boot time
+ SYSTEM(EnumSet.of(OptionScope.BOOT, OptionScope.SYSTEM)), // Can only be configured at the system level
+ SESSION(EnumSet.of(OptionScope.BOOT, OptionScope.SESSION)), // Can only be configured at the session level
+ QUERY(EnumSet.of(OptionScope.BOOT, OptionScope.QUERY)), // Can only be configured at the query level
+ SYSTEM_AND_SESSION(EnumSet.of(OptionScope.BOOT, OptionScope.SYSTEM, OptionScope.SESSION)), // Can only be configured at the system or session level
+ SESSION_AND_QUERY(EnumSet.of(OptionScope.BOOT, OptionScope.SESSION, OptionScope.QUERY)), // Can only be configured at the session or query level
+ ALL(EnumSet.of(OptionScope.BOOT, OptionScope.SYSTEM, OptionScope.SESSION, OptionScope.QUERY)); // Can be configured at the system, session, or query level
+
+ private EnumSet<OptionScope> scopes;
+
+ AccessibleScopes(EnumSet<OptionScope> scopes) {
+ this.scopes = Preconditions.checkNotNull(scopes);
+ }
+
+ public boolean inScopeOf(OptionScope scope) {
+ return scopes.contains(scope);
+ }
}
public enum Kind {
BOOLEAN, LONG, STRING, DOUBLE
}
+ /**
+ * This defines where an option was actually configured.
+ */
public enum OptionScope {
- BOOT, SYSTEM, SESSION, QUERY
+ BOOT, SYSTEM, SESSION, QUERY;
}
public final String name;
public final Kind kind;
- public final OptionType type;
+ public final AccessibleScopes accessibleScopes;
public final Long num_val;
public final String string_val;
public final Boolean bool_val;
public final Double float_val;
public final OptionScope scope;
- public static OptionValue createLong(OptionType type, String name, long val, OptionScope scope) {
+ public static OptionValue create(AccessibleScopes type, String name, long val, OptionScope scope) {
return new OptionValue(Kind.LONG, type, name, val, null, null, null, scope);
}
- public static OptionValue createBoolean(OptionType type, String name, boolean bool, OptionScope scope) {
+ public static OptionValue create(AccessibleScopes type, String name, boolean bool, OptionScope scope) {
return new OptionValue(Kind.BOOLEAN, type, name, null, null, bool, null, scope);
}
- public static OptionValue createString(OptionType type, String name, String val, OptionScope scope) {
+ public static OptionValue create(AccessibleScopes type, String name, String val, OptionScope scope) {
return new OptionValue(Kind.STRING, type, name, null, val, null, null, scope);
}
- public static OptionValue createDouble(OptionType type, String name, double val, OptionScope scope) {
+ public static OptionValue create(AccessibleScopes type, String name, double val, OptionScope scope) {
return new OptionValue(Kind.DOUBLE, type, name, null, null, null, val, scope);
}
- public static OptionValue createOption(Kind kind, OptionType type, String name, String val, OptionScope scope) {
- switch (kind) {
- case BOOLEAN:
- return createBoolean(type, name, Boolean.valueOf(val), scope);
- case LONG:
- return createLong(type, name, Long.valueOf(val), scope);
- case STRING:
- return createString(type, name, val, scope);
- case DOUBLE:
- return createDouble(type, name, Double.valueOf(val), scope);
- default:
- return null;
+ public static OptionValue create(AccessibleScopes type, String name, Object val, OptionScope scope) {
+ if (val instanceof Boolean) {
+ return create(type, name, ((Boolean) val).booleanValue(), scope);
+ } else if (val instanceof Long) {
+ return create(type, name, ((Long) val).longValue(), scope);
+ } else if (val instanceof String) {
+ return create(type, name, (String) val, scope);
+ } else if (val instanceof Double) {
+ return create(type, name, ((Double) val).doubleValue(), scope);
}
+
+ throw new IllegalArgumentException(String.format("Unsupported type %s", val.getClass()));
}
@JsonCreator
private OptionValue(@JsonProperty("kind") Kind kind,
- @JsonProperty("type") OptionType type,
+ @JsonProperty("accessibleScopes") AccessibleScopes accessibleScopes,
@JsonProperty("name") String name,
@JsonProperty("num_val") Long num_val,
@JsonProperty("string_val") String string_val,
@@ -105,7 +129,7 @@ public class OptionValue implements Comparable<OptionValue> {
@JsonProperty("scope") OptionScope scope) {
Preconditions.checkArgument(num_val != null || string_val != null || bool_val != null || float_val != null);
this.kind = kind;
- this.type = type;
+ this.accessibleScopes = accessibleScopes;
this.name = name;
this.float_val = float_val;
this.num_val = num_val;
@@ -144,7 +168,7 @@ public class OptionValue implements Comparable<OptionValue> {
result = prime * result + ((name == null) ? 0 : name.hashCode());
result = prime * result + ((num_val == null) ? 0 : num_val.hashCode());
result = prime * result + ((string_val == null) ? 0 : string_val.hashCode());
- result = prime * result + ((type == null) ? 0 : type.hashCode());
+ result = prime * result + ((accessibleScopes == null) ? 0 : accessibleScopes.hashCode());
return result;
}
@@ -206,7 +230,7 @@ public class OptionValue implements Comparable<OptionValue> {
return false;
}
final OptionValue other = (OptionValue) obj;
- return type == other.type;
+ return accessibleScopes == other.accessibleScopes;
}
@Override
@@ -216,6 +240,6 @@ public class OptionValue implements Comparable<OptionValue> {
@Override
public String toString() {
- return "OptionValue [ type=" + type + ", name=" + name + ", value=" + getValue() + " ]";
+ return "OptionValue [ accessibleScopes=" + accessibleScopes + ", optionScope=" + scope + ", name=" + name + ", value=" + getValue() + " ]";
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/6adeb986/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/QueryOptionManager.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/QueryOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/QueryOptionManager.java
index 77ca3d0..a111db4 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/QueryOptionManager.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/QueryOptionManager.java
@@ -18,7 +18,6 @@
package org.apache.drill.exec.server.options;
import org.apache.drill.common.map.CaseInsensitiveMap;
-import org.apache.drill.exec.server.options.OptionValue.OptionType;
/**
* {@link OptionManager} that holds options within {@link org.apache.drill.exec.ops.QueryContext}.
@@ -38,7 +37,31 @@ public class QueryOptionManager extends InMemoryOptionManager {
}
@Override
- boolean supportsOptionType(OptionType type) {
- return type == OptionType.QUERY;
+ public OptionValue getDefault(String optionName) {
+ return fallback.getDefault(optionName);
+ }
+
+ public SessionOptionManager getSessionOptionManager() {
+ return (SessionOptionManager) fallback;
+ }
+
+ public OptionManager getOptionManager(OptionValue.OptionScope scope) {
+ switch (scope) {
+ case SYSTEM:
+ return getSessionOptionManager().getSystemOptionManager();
+ case SESSION:
+ return getSessionOptionManager();
+ case QUERY:
+ return this;
+ case BOOT:
+ throw new UnsupportedOperationException("There is no option manager for " + OptionValue.OptionScope.BOOT);
+ default:
+ throw new UnsupportedOperationException("Invalid type: " + scope);
+ }
+ }
+
+ @Override
+ protected OptionValue.OptionScope getScope() {
+ return OptionValue.OptionScope.QUERY;
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/6adeb986/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SessionOptionManager.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SessionOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SessionOptionManager.java
index 512c8e9..52bf403 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SessionOptionManager.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SessionOptionManager.java
@@ -20,10 +20,8 @@ package org.apache.drill.exec.server.options;
import com.google.common.base.Predicate;
import com.google.common.collect.Collections2;
import org.apache.commons.lang3.tuple.ImmutablePair;
-import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.common.map.CaseInsensitiveMap;
import org.apache.drill.exec.rpc.user.UserSession;
-import org.apache.drill.exec.server.options.OptionValue.OptionType;
import java.util.Collection;
import java.util.Map;
@@ -55,13 +53,11 @@ public class SessionOptionManager extends InMemoryOptionManager {
}
@Override
- boolean setLocalOption(final OptionValue value) {
- final boolean set = super.setLocalOption(value);
- if (!set) {
- return false;
- }
+ public void setLocalOptionHelper(final OptionValue value) {
+ super.setLocalOptionHelper(value);
final String name = value.name;
- final OptionValidator validator = getFallbackOptionManager().getValidator(name); // if set, validator must exist.
+ final OptionDefinition definition = getOptionDefinition(name); // if set, validator must exist.
+ final OptionValidator validator = definition.getValidator();
final boolean shortLived = validator.isShortLived();
if (shortLived) {
final int start = session.getQueryCount() + 1; // start from the next query
@@ -69,7 +65,6 @@ public class SessionOptionManager extends InMemoryOptionManager {
final int end = start + ttl;
shortLivedOptions.put(name, new ImmutablePair<>(start, end));
}
- return true;
}
@Override
@@ -83,7 +78,7 @@ public class SessionOptionManager extends InMemoryOptionManager {
final int start = shortLivedOptions.get(name).getLeft();
// option is not in effect if queryNumber < start
if (queryNumber < start) {
- return getFallbackOptionManager().getValidator(name).getDefault();
+ return fallback.getOption(name);
// reset if queryNumber <= end
} else {
options.remove(name);
@@ -116,14 +111,22 @@ public class SessionOptionManager extends InMemoryOptionManager {
return liveOptions;
}
+ /**
+ * Gets the SystemOptionManager.
+ * @return The SystemOptionManager.
+ */
+ public SystemOptionManager getSystemOptionManager() {
+ final SystemOptionManager systemOptionManager = (SystemOptionManager) fallback;
+ return systemOptionManager;
+ }
+
@Override
- boolean supportsOptionType(OptionType type) {
- return type == OptionType.SESSION;
+ public OptionValue getDefault(String optionName) {
+ return fallback.getDefault(optionName);
}
- /* Gets fallback manager and returns it as SystemOptionManager */
- public SystemOptionManager getFallbackOptionManager() {
- final SystemOptionManager systemOptionManager = (SystemOptionManager) getFallback();
- return systemOptionManager;
+ @Override
+ protected OptionValue.OptionScope getScope() {
+ return OptionValue.OptionScope.SESSION;
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/6adeb986/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
index 4e362ff..75bcc1f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
@@ -26,7 +26,6 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
-import com.typesafe.config.ConfigException;
import org.apache.commons.collections.IteratorUtils;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.config.LogicalPlanPersistence;
@@ -36,7 +35,6 @@ import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.compile.ClassCompilerSelector;
import org.apache.drill.exec.compile.ClassTransformer;
import org.apache.drill.exec.planner.physical.PlannerSettings;
-import org.apache.drill.exec.server.options.OptionValue.OptionType;
import org.apache.drill.exec.store.sys.PersistentStore;
import org.apache.drill.exec.store.sys.PersistentStoreConfig;
import org.apache.drill.exec.store.sys.PersistentStoreProvider;
@@ -69,148 +67,154 @@ import com.google.common.collect.Sets;
* that is loaded into validator from the config will be returned.
* </p>
*/
-
-public class SystemOptionManager extends BaseOptionManager implements OptionManager, AutoCloseable {
+public class SystemOptionManager extends BaseOptionManager implements AutoCloseable {
private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SystemOptionManager.class);
- private CaseInsensitiveMap<OptionValidator> VALIDATORS;
- public void populateValidators() {
- final OptionValidator[] validators = new OptionValidator[]{
- PlannerSettings.CONSTANT_FOLDING,
- PlannerSettings.EXCHANGE,
- PlannerSettings.HASHAGG,
- PlannerSettings.STREAMAGG,
- PlannerSettings.HASHJOIN,
- PlannerSettings.MERGEJOIN,
- PlannerSettings.NESTEDLOOPJOIN,
- PlannerSettings.MULTIPHASE,
- PlannerSettings.BROADCAST,
- PlannerSettings.BROADCAST_THRESHOLD,
- PlannerSettings.BROADCAST_FACTOR,
- PlannerSettings.NESTEDLOOPJOIN_FACTOR,
- PlannerSettings.NLJOIN_FOR_SCALAR,
- PlannerSettings.JOIN_ROW_COUNT_ESTIMATE_FACTOR,
- PlannerSettings.MUX_EXCHANGE,
- PlannerSettings.DEMUX_EXCHANGE,
- PlannerSettings.PRODUCER_CONSUMER,
- PlannerSettings.PRODUCER_CONSUMER_QUEUE_SIZE,
- PlannerSettings.HASH_SINGLE_KEY,
- PlannerSettings.IDENTIFIER_MAX_LENGTH,
- PlannerSettings.HASH_JOIN_SWAP,
- PlannerSettings.HASH_JOIN_SWAP_MARGIN_FACTOR,
- PlannerSettings.PARTITION_SENDER_THREADS_FACTOR,
- PlannerSettings.PARTITION_SENDER_MAX_THREADS,
- PlannerSettings.PARTITION_SENDER_SET_THREADS,
- PlannerSettings.ENABLE_DECIMAL_DATA_TYPE,
- PlannerSettings.HEP_OPT,
- PlannerSettings.PLANNER_MEMORY_LIMIT,
- PlannerSettings.HEP_PARTITION_PRUNING,
- PlannerSettings.FILTER_MIN_SELECTIVITY_ESTIMATE_FACTOR,
- PlannerSettings.FILTER_MAX_SELECTIVITY_ESTIMATE_FACTOR,
- PlannerSettings.TYPE_INFERENCE,
- PlannerSettings.IN_SUBQUERY_THRESHOLD,
- PlannerSettings.UNIONALL_DISTRIBUTE,
- PlannerSettings.PARQUET_ROWGROUP_FILTER_PUSHDOWN_PLANNING,
- PlannerSettings.PARQUET_ROWGROUP_FILTER_PUSHDOWN_PLANNING_THRESHOLD,
- PlannerSettings.QUOTING_IDENTIFIERS,
- PlannerSettings.JOIN_OPTIMIZATION,
- PlannerSettings.FORCE_2PHASE_AGGR, // for testing
- ExecConstants.HASHAGG_NUM_PARTITIONS_VALIDATOR,
- ExecConstants.HASHAGG_MAX_MEMORY_VALIDATOR,
- ExecConstants.HASHAGG_MIN_BATCHES_PER_PARTITION_VALIDATOR, // for tuning
- ExecConstants.HASHAGG_FALLBACK_ENABLED_VALIDATOR, // for enable/disable unbounded HashAgg
- ExecConstants.CAST_TO_NULLABLE_NUMERIC_OPTION,
- ExecConstants.OUTPUT_FORMAT_VALIDATOR,
- ExecConstants.PARQUET_BLOCK_SIZE_VALIDATOR,
- ExecConstants.PARQUET_WRITER_USE_SINGLE_FS_BLOCK_VALIDATOR,
- ExecConstants.PARQUET_PAGE_SIZE_VALIDATOR,
- ExecConstants.PARQUET_DICT_PAGE_SIZE_VALIDATOR,
- ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE_VALIDATOR,
- ExecConstants.PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING_VALIDATOR,
- ExecConstants.PARQUET_VECTOR_FILL_THRESHOLD_VALIDATOR,
- ExecConstants.PARQUET_VECTOR_FILL_CHECK_THRESHOLD_VALIDATOR,
- ExecConstants.PARQUET_RECORD_READER_IMPLEMENTATION_VALIDATOR,
- ExecConstants.PARQUET_PAGEREADER_ASYNC_VALIDATOR,
- ExecConstants.PARQUET_PAGEREADER_QUEUE_SIZE_VALIDATOR,
- ExecConstants.PARQUET_PAGEREADER_ENFORCETOTALSIZE_VALIDATOR,
- ExecConstants.PARQUET_COLUMNREADER_ASYNC_VALIDATOR,
- ExecConstants.PARQUET_PAGEREADER_USE_BUFFERED_READ_VALIDATOR,
- ExecConstants.PARQUET_PAGEREADER_BUFFER_SIZE_VALIDATOR,
- ExecConstants.PARQUET_PAGEREADER_USE_FADVISE_VALIDATOR,
- ExecConstants.PARQUET_READER_INT96_AS_TIMESTAMP_VALIDATOR,
- ExecConstants.JSON_READER_ALL_TEXT_MODE_VALIDATOR,
- ExecConstants.ENABLE_UNION_TYPE,
- ExecConstants.TEXT_ESTIMATED_ROW_SIZE,
- ExecConstants.JSON_EXTENDED_TYPES,
- ExecConstants.JSON_WRITER_UGLIFY,
- ExecConstants.JSON_WRITER_SKIPNULLFIELDS,
- ExecConstants.JSON_READ_NUMBERS_AS_DOUBLE_VALIDATOR,
- ExecConstants.JSON_SKIP_MALFORMED_RECORDS_VALIDATOR,
- ExecConstants.JSON_READER_PRINT_INVALID_RECORDS_LINE_NOS_FLAG_VALIDATOR,
- ExecConstants.FILESYSTEM_PARTITION_COLUMN_LABEL_VALIDATOR,
- ExecConstants.MONGO_READER_ALL_TEXT_MODE_VALIDATOR,
- ExecConstants.MONGO_READER_READ_NUMBERS_AS_DOUBLE_VALIDATOR,
- ExecConstants.MONGO_BSON_RECORD_READER_VALIDATOR,
- ExecConstants.HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS_VALIDATOR,
- ExecConstants.SLICE_TARGET_OPTION,
- ExecConstants.AFFINITY_FACTOR,
- ExecConstants.MAX_WIDTH_GLOBAL,
- ExecConstants.MAX_WIDTH_PER_NODE,
- ExecConstants.ENABLE_QUEUE,
- ExecConstants.LARGE_QUEUE_SIZE,
- ExecConstants.QUEUE_THRESHOLD_SIZE,
- ExecConstants.QUEUE_TIMEOUT,
- ExecConstants.SMALL_QUEUE_SIZE,
- ExecConstants.MIN_HASH_TABLE_SIZE,
- ExecConstants.MAX_HASH_TABLE_SIZE,
- ExecConstants.EARLY_LIMIT0_OPT,
- ExecConstants.ENABLE_MEMORY_ESTIMATION,
- ExecConstants.MAX_QUERY_MEMORY_PER_NODE,
- ExecConstants.MIN_MEMORY_PER_BUFFERED_OP,
- ExecConstants.NON_BLOCKING_OPERATORS_MEMORY,
- ExecConstants.HASH_JOIN_TABLE_FACTOR,
- ExecConstants.HASH_AGG_TABLE_FACTOR,
- ExecConstants.AVERAGE_FIELD_WIDTH,
- ExecConstants.NEW_VIEW_DEFAULT_PERMS_VALIDATOR,
- ExecConstants.CTAS_PARTITIONING_HASH_DISTRIBUTE_VALIDATOR,
- ExecConstants.ADMIN_USERS_VALIDATOR,
- ExecConstants.ADMIN_USER_GROUPS_VALIDATOR,
- ExecConstants.IMPERSONATION_POLICY_VALIDATOR,
- ClassCompilerSelector.JAVA_COMPILER_VALIDATOR,
- ClassCompilerSelector.JAVA_COMPILER_JANINO_MAXSIZE,
- ClassCompilerSelector.JAVA_COMPILER_DEBUG,
- ExecConstants.ENABLE_VERBOSE_ERRORS,
- ExecConstants.ENABLE_WINDOW_FUNCTIONS_VALIDATOR,
- ClassTransformer.SCALAR_REPLACEMENT_VALIDATOR,
- ExecConstants.ENABLE_NEW_TEXT_READER,
- ExecConstants.ENABLE_BULK_LOAD_TABLE_LIST,
- ExecConstants.BULK_LOAD_TABLE_LIST_BULK_SIZE,
- ExecConstants.WEB_LOGS_MAX_LINES_VALIDATOR,
- ExecConstants.IMPLICIT_FILENAME_COLUMN_LABEL_VALIDATOR,
- ExecConstants.IMPLICIT_SUFFIX_COLUMN_LABEL_VALIDATOR,
- ExecConstants.IMPLICIT_FQN_COLUMN_LABEL_VALIDATOR,
- ExecConstants.IMPLICIT_FILEPATH_COLUMN_LABEL_VALIDATOR,
- ExecConstants.CODE_GEN_EXP_IN_METHOD_SIZE_VALIDATOR,
- ExecConstants.CREATE_PREPARE_STATEMENT_TIMEOUT_MILLIS_VALIDATOR,
- ExecConstants.DYNAMIC_UDF_SUPPORT_ENABLED_VALIDATOR,
- ExecConstants.EXTERNAL_SORT_DISABLE_MANAGED_OPTION,
- ExecConstants.ENABLE_QUERY_PROFILE_VALIDATOR,
- ExecConstants.QUERY_PROFILE_DEBUG_VALIDATOR,
- ExecConstants.USE_DYNAMIC_UDFS,
- ExecConstants.QUERY_TRANSIENT_STATE_UPDATE,
- ExecConstants.PERSISTENT_TABLE_UMASK_VALIDATOR,
- ExecConstants.CPU_LOAD_AVERAGE,
- ExecConstants.ENABLE_VECTOR_VALIDATOR,
- ExecConstants.ENABLE_ITERATOR_VALIDATOR
+ /**
+ * Creates all the OptionDefinitions to be registered with the {@link SystemOptionManager}.
+ * @return A map
+ */
+ public static CaseInsensitiveMap<OptionDefinition> createDefaultOptionDefinitions() {
+ final OptionDefinition[] definitions = new OptionDefinition[]{
+ new OptionDefinition(PlannerSettings.CONSTANT_FOLDING),
+ new OptionDefinition(PlannerSettings.EXCHANGE),
+ new OptionDefinition(PlannerSettings.HASHAGG),
+ new OptionDefinition(PlannerSettings.STREAMAGG),
+ new OptionDefinition(PlannerSettings.HASHJOIN),
+ new OptionDefinition(PlannerSettings.MERGEJOIN),
+ new OptionDefinition(PlannerSettings.NESTEDLOOPJOIN),
+ new OptionDefinition(PlannerSettings.MULTIPHASE),
+ new OptionDefinition(PlannerSettings.BROADCAST),
+ new OptionDefinition(PlannerSettings.BROADCAST_THRESHOLD),
+ new OptionDefinition(PlannerSettings.BROADCAST_FACTOR),
+ new OptionDefinition(PlannerSettings.NESTEDLOOPJOIN_FACTOR),
+ new OptionDefinition(PlannerSettings.NLJOIN_FOR_SCALAR),
+ new OptionDefinition(PlannerSettings.JOIN_ROW_COUNT_ESTIMATE_FACTOR),
+ new OptionDefinition(PlannerSettings.MUX_EXCHANGE),
+ new OptionDefinition(PlannerSettings.DEMUX_EXCHANGE),
+ new OptionDefinition(PlannerSettings.PRODUCER_CONSUMER),
+ new OptionDefinition(PlannerSettings.PRODUCER_CONSUMER_QUEUE_SIZE),
+ new OptionDefinition(PlannerSettings.HASH_SINGLE_KEY),
+ new OptionDefinition(PlannerSettings.IDENTIFIER_MAX_LENGTH),
+ new OptionDefinition(PlannerSettings.HASH_JOIN_SWAP),
+ new OptionDefinition(PlannerSettings.HASH_JOIN_SWAP_MARGIN_FACTOR),
+ new OptionDefinition(PlannerSettings.PARTITION_SENDER_THREADS_FACTOR),
+ new OptionDefinition(PlannerSettings.PARTITION_SENDER_MAX_THREADS),
+ new OptionDefinition(PlannerSettings.PARTITION_SENDER_SET_THREADS),
+ new OptionDefinition(PlannerSettings.ENABLE_DECIMAL_DATA_TYPE),
+ new OptionDefinition(PlannerSettings.HEP_OPT),
+ new OptionDefinition(PlannerSettings.PLANNER_MEMORY_LIMIT),
+ new OptionDefinition(PlannerSettings.HEP_PARTITION_PRUNING),
+ new OptionDefinition(PlannerSettings.FILTER_MIN_SELECTIVITY_ESTIMATE_FACTOR),
+ new OptionDefinition(PlannerSettings.FILTER_MAX_SELECTIVITY_ESTIMATE_FACTOR),
+ new OptionDefinition(PlannerSettings.TYPE_INFERENCE),
+ new OptionDefinition(PlannerSettings.IN_SUBQUERY_THRESHOLD),
+ new OptionDefinition(PlannerSettings.UNIONALL_DISTRIBUTE),
+ new OptionDefinition(PlannerSettings.PARQUET_ROWGROUP_FILTER_PUSHDOWN_PLANNING),
+ new OptionDefinition(PlannerSettings.PARQUET_ROWGROUP_FILTER_PUSHDOWN_PLANNING_THRESHOLD),
+ new OptionDefinition(PlannerSettings.QUOTING_IDENTIFIERS),
+ new OptionDefinition(PlannerSettings.JOIN_OPTIMIZATION),
+ new OptionDefinition(PlannerSettings.FORCE_2PHASE_AGGR), // for testing
+ new OptionDefinition(ExecConstants.HASHAGG_NUM_PARTITIONS_VALIDATOR),
+ new OptionDefinition(ExecConstants.HASHAGG_MAX_MEMORY_VALIDATOR),
+ new OptionDefinition(ExecConstants.HASHAGG_MIN_BATCHES_PER_PARTITION_VALIDATOR), // for tuning
+ new OptionDefinition(ExecConstants.HASHAGG_FALLBACK_ENABLED_VALIDATOR), // for enable/disable unbounded HashAgg
+ new OptionDefinition(ExecConstants.CAST_TO_NULLABLE_NUMERIC_OPTION),
+ new OptionDefinition(ExecConstants.OUTPUT_FORMAT_VALIDATOR),
+ new OptionDefinition(ExecConstants.PARQUET_BLOCK_SIZE_VALIDATOR),
+ new OptionDefinition(ExecConstants.PARQUET_WRITER_USE_SINGLE_FS_BLOCK_VALIDATOR),
+ new OptionDefinition(ExecConstants.PARQUET_PAGE_SIZE_VALIDATOR),
+ new OptionDefinition(ExecConstants.PARQUET_DICT_PAGE_SIZE_VALIDATOR),
+ new OptionDefinition(ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE_VALIDATOR),
+ new OptionDefinition(ExecConstants.PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING_VALIDATOR),
+ new OptionDefinition(ExecConstants.PARQUET_VECTOR_FILL_THRESHOLD_VALIDATOR),
+ new OptionDefinition(ExecConstants.PARQUET_VECTOR_FILL_CHECK_THRESHOLD_VALIDATOR),
+ new OptionDefinition(ExecConstants.PARQUET_RECORD_READER_IMPLEMENTATION_VALIDATOR),
+ new OptionDefinition(ExecConstants.PARQUET_PAGEREADER_ASYNC_VALIDATOR),
+ new OptionDefinition(ExecConstants.PARQUET_PAGEREADER_QUEUE_SIZE_VALIDATOR),
+ new OptionDefinition(ExecConstants.PARQUET_PAGEREADER_ENFORCETOTALSIZE_VALIDATOR),
+ new OptionDefinition(ExecConstants.PARQUET_COLUMNREADER_ASYNC_VALIDATOR),
+ new OptionDefinition(ExecConstants.PARQUET_PAGEREADER_USE_BUFFERED_READ_VALIDATOR),
+ new OptionDefinition(ExecConstants.PARQUET_PAGEREADER_BUFFER_SIZE_VALIDATOR),
+ new OptionDefinition(ExecConstants.PARQUET_PAGEREADER_USE_FADVISE_VALIDATOR),
+ new OptionDefinition(ExecConstants.PARQUET_READER_INT96_AS_TIMESTAMP_VALIDATOR),
+ new OptionDefinition(ExecConstants.JSON_READER_ALL_TEXT_MODE_VALIDATOR),
+ new OptionDefinition(ExecConstants.ENABLE_UNION_TYPE),
+ new OptionDefinition(ExecConstants.TEXT_ESTIMATED_ROW_SIZE),
+ new OptionDefinition(ExecConstants.JSON_EXTENDED_TYPES),
+ new OptionDefinition(ExecConstants.JSON_WRITER_UGLIFY),
+ new OptionDefinition(ExecConstants.JSON_WRITER_SKIPNULLFIELDS),
+ new OptionDefinition(ExecConstants.JSON_READ_NUMBERS_AS_DOUBLE_VALIDATOR),
+ new OptionDefinition(ExecConstants.JSON_SKIP_MALFORMED_RECORDS_VALIDATOR),
+ new OptionDefinition(ExecConstants.JSON_READER_PRINT_INVALID_RECORDS_LINE_NOS_FLAG_VALIDATOR),
+ new OptionDefinition(ExecConstants.FILESYSTEM_PARTITION_COLUMN_LABEL_VALIDATOR),
+ new OptionDefinition(ExecConstants.MONGO_READER_ALL_TEXT_MODE_VALIDATOR),
+ new OptionDefinition(ExecConstants.MONGO_READER_READ_NUMBERS_AS_DOUBLE_VALIDATOR),
+ new OptionDefinition(ExecConstants.MONGO_BSON_RECORD_READER_VALIDATOR),
+ new OptionDefinition(ExecConstants.HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS_VALIDATOR),
+ new OptionDefinition(ExecConstants.SLICE_TARGET_OPTION),
+ new OptionDefinition(ExecConstants.AFFINITY_FACTOR),
+ new OptionDefinition(ExecConstants.MAX_WIDTH_GLOBAL),
+ new OptionDefinition(ExecConstants.MAX_WIDTH_PER_NODE),
+ new OptionDefinition(ExecConstants.ENABLE_QUEUE),
+ new OptionDefinition(ExecConstants.LARGE_QUEUE_SIZE),
+ new OptionDefinition(ExecConstants.QUEUE_THRESHOLD_SIZE),
+ new OptionDefinition(ExecConstants.QUEUE_TIMEOUT),
+ new OptionDefinition(ExecConstants.SMALL_QUEUE_SIZE),
+ new OptionDefinition(ExecConstants.MIN_HASH_TABLE_SIZE),
+ new OptionDefinition(ExecConstants.MAX_HASH_TABLE_SIZE),
+ new OptionDefinition(ExecConstants.EARLY_LIMIT0_OPT),
+ new OptionDefinition(ExecConstants.ENABLE_MEMORY_ESTIMATION),
+ new OptionDefinition(ExecConstants.MAX_QUERY_MEMORY_PER_NODE),
+ new OptionDefinition(ExecConstants.MIN_MEMORY_PER_BUFFERED_OP),
+ new OptionDefinition(ExecConstants.NON_BLOCKING_OPERATORS_MEMORY),
+ new OptionDefinition(ExecConstants.HASH_JOIN_TABLE_FACTOR),
+ new OptionDefinition(ExecConstants.HASH_AGG_TABLE_FACTOR),
+ new OptionDefinition(ExecConstants.AVERAGE_FIELD_WIDTH),
+ new OptionDefinition(ExecConstants.NEW_VIEW_DEFAULT_PERMS_VALIDATOR),
+ new OptionDefinition(ExecConstants.CTAS_PARTITIONING_HASH_DISTRIBUTE_VALIDATOR),
+ new OptionDefinition(ExecConstants.ADMIN_USERS_VALIDATOR, new OptionMetaData(OptionValue.AccessibleScopes.SYSTEM, true, false)),
+ new OptionDefinition(ExecConstants.ADMIN_USER_GROUPS_VALIDATOR, new OptionMetaData(OptionValue.AccessibleScopes.SYSTEM, true, false)),
+ new OptionDefinition(ExecConstants.IMPERSONATION_POLICY_VALIDATOR, new OptionMetaData(OptionValue.AccessibleScopes.SYSTEM, true, false)),
+ new OptionDefinition(ClassCompilerSelector.JAVA_COMPILER_VALIDATOR),
+ new OptionDefinition(ClassCompilerSelector.JAVA_COMPILER_JANINO_MAXSIZE),
+ new OptionDefinition(ClassCompilerSelector.JAVA_COMPILER_DEBUG),
+ new OptionDefinition(ExecConstants.ENABLE_VERBOSE_ERRORS),
+ new OptionDefinition(ExecConstants.ENABLE_WINDOW_FUNCTIONS_VALIDATOR),
+ new OptionDefinition(ClassTransformer.SCALAR_REPLACEMENT_VALIDATOR),
+ new OptionDefinition(ExecConstants.ENABLE_NEW_TEXT_READER),
+ new OptionDefinition(ExecConstants.ENABLE_BULK_LOAD_TABLE_LIST),
+ new OptionDefinition(ExecConstants.BULK_LOAD_TABLE_LIST_BULK_SIZE),
+ new OptionDefinition(ExecConstants.WEB_LOGS_MAX_LINES_VALIDATOR),
+ new OptionDefinition(ExecConstants.IMPLICIT_FILENAME_COLUMN_LABEL_VALIDATOR),
+ new OptionDefinition(ExecConstants.IMPLICIT_SUFFIX_COLUMN_LABEL_VALIDATOR),
+ new OptionDefinition(ExecConstants.IMPLICIT_FQN_COLUMN_LABEL_VALIDATOR),
+ new OptionDefinition(ExecConstants.IMPLICIT_FILEPATH_COLUMN_LABEL_VALIDATOR),
+ new OptionDefinition(ExecConstants.CODE_GEN_EXP_IN_METHOD_SIZE_VALIDATOR),
+ new OptionDefinition(ExecConstants.CREATE_PREPARE_STATEMENT_TIMEOUT_MILLIS_VALIDATOR),
+ new OptionDefinition(ExecConstants.DYNAMIC_UDF_SUPPORT_ENABLED_VALIDATOR, new OptionMetaData(OptionValue.AccessibleScopes.SYSTEM, true, false)),
+ new OptionDefinition(ExecConstants.EXTERNAL_SORT_DISABLE_MANAGED_OPTION),
+ new OptionDefinition(ExecConstants.ENABLE_QUERY_PROFILE_VALIDATOR),
+ new OptionDefinition(ExecConstants.QUERY_PROFILE_DEBUG_VALIDATOR),
+ new OptionDefinition(ExecConstants.USE_DYNAMIC_UDFS),
+ new OptionDefinition(ExecConstants.QUERY_TRANSIENT_STATE_UPDATE),
+ new OptionDefinition(ExecConstants.PERSISTENT_TABLE_UMASK_VALIDATOR),
+ new OptionDefinition(ExecConstants.CPU_LOAD_AVERAGE),
+ new OptionDefinition(ExecConstants.ENABLE_VECTOR_VALIDATOR),
+ new OptionDefinition(ExecConstants.ENABLE_ITERATOR_VALIDATOR)
};
- final Map<String, OptionValidator> tmp = new HashMap<>();
- for (final OptionValidator validator : validators) {
- tmp.put(validator.getOptionName(), validator);
+
+ final CaseInsensitiveMap<OptionDefinition> map = CaseInsensitiveMap.newHashMap();
+
+ for (final OptionDefinition definition: definitions) {
+ map.put(definition.getValidator().getOptionName(), definition);
}
+
if (AssertionUtil.isAssertionsEnabled()) {
- tmp.put(ExecConstants.DRILLBIT_CONTROL_INJECTIONS, ExecConstants.DRILLBIT_CONTROLS_VALIDATOR);
+ map.put(ExecConstants.DRILLBIT_CONTROL_INJECTIONS, new OptionDefinition(ExecConstants.DRILLBIT_CONTROLS_VALIDATOR));
}
- VALIDATORS = CaseInsensitiveMap.newImmutableMap(tmp);
+
+ return map;
}
private final PersistentStoreConfig<OptionValue> config;
@@ -223,37 +227,23 @@ public class SystemOptionManager extends BaseOptionManager implements OptionMana
* NOTE: CRUD operations must use lowercase keys.
*/
private PersistentStore<OptionValue> options;
+ private CaseInsensitiveMap<OptionDefinition> definitions;
+ private CaseInsensitiveMap<OptionValue> defaults;
- public SystemOptionManager(LogicalPlanPersistence lpPersistence, final PersistentStoreProvider provider) {
- this(lpPersistence,provider,null);
+ public SystemOptionManager(LogicalPlanPersistence lpPersistence, final PersistentStoreProvider provider,
+ final DrillConfig bootConfig) {
+ this(lpPersistence, provider, bootConfig, SystemOptionManager.createDefaultOptionDefinitions());
}
- public SystemOptionManager(LogicalPlanPersistence lpPersistence, final PersistentStoreProvider provider, final DrillConfig bootConfig) {
+ public SystemOptionManager(final LogicalPlanPersistence lpPersistence, final PersistentStoreProvider provider,
+ final DrillConfig bootConfig, final CaseInsensitiveMap<OptionDefinition> definitions) {
this.provider = provider;
this.config = PersistentStoreConfig.newJacksonBuilder(lpPersistence.getMapper(), OptionValue.class)
.name("sys.options")
.build();
this.bootConfig = bootConfig;
- populateValidators();
- populateDefaultValues();
-
- }
-
- /**
- * Gets the {@link OptionValidator} associated with the name.
- *
- * @param name name of the option
- * @return the associated validator
- * @throws UserException - if the validator is not found
- */
- public OptionValidator getValidator(final String name) {
- final OptionValidator validator = VALIDATORS.get(name);
- if (validator == null) {
- throw UserException.validationError()
- .message(String.format("The option '%s' does not exist.", name.toLowerCase()))
- .build(logger);
- }
- return validator;
+ this.definitions = definitions;
+ this.defaults = populateDefaultValues(definitions, bootConfig);
}
/**
@@ -267,12 +257,13 @@ public class SystemOptionManager extends BaseOptionManager implements OptionMana
// if necessary, deprecate and replace options from persistent store
for (final Entry<String, OptionValue> option : Lists.newArrayList(options.getAll())) {
final String name = option.getKey();
- final OptionValidator validator = VALIDATORS.get(name);
- if (validator == null) {
+ final OptionDefinition definition = definitions.get(name);
+ if (definition == null) {
// deprecated option, delete.
options.delete(name);
logger.warn("Deleting deprecated option `{}`", name);
} else {
+ OptionValidator validator = definition.getValidator();
final String canonicalName = validator.getOptionName().toLowerCase();
if (!name.equals(canonicalName)) {
// for backwards compatibility <= 1.1, rename to lower case.
@@ -283,6 +274,7 @@ public class SystemOptionManager extends BaseOptionManager implements OptionMana
}
}
}
+
return this;
}
@@ -290,8 +282,8 @@ public class SystemOptionManager extends BaseOptionManager implements OptionMana
public Iterator<OptionValue> iterator() {
final Map<String, OptionValue> buildList = CaseInsensitiveMap.newHashMap();
// populate the default options
- for (final Map.Entry<String, OptionValidator> entry : VALIDATORS.entrySet()) {
- buildList.put(entry.getKey(), entry.getValue().getDefault());
+ for (final Map.Entry<String, OptionValue> entry : defaults.entrySet()) {
+ buildList.put(entry.getKey(), entry.getValue());
}
// override if changed
for (final Map.Entry<String, OptionValue> entry : Lists.newArrayList(options.getAll())) {
@@ -304,39 +296,48 @@ public class SystemOptionManager extends BaseOptionManager implements OptionMana
public OptionValue getOption(final String name) {
// check local space (persistent store)
final OptionValue value = options.get(name.toLowerCase());
+
if (value != null) {
return value;
}
- // If option is not set return the default set in the validator.
- final OptionValidator validator = getValidator(name);
- return validator.getDefault();
+ // otherwise, return default set in the validator.
+ return defaults.get(name);
+ }
+
+ @Override
+ public OptionValue getDefault(String optionName) {
+ return defaults.get(optionName);
}
@Override
- public void setOption(final OptionValue value) {
- checkArgument(value.type == OptionType.SYSTEM, "OptionType must be SYSTEM.");
+ protected void setLocalOptionHelper(final OptionValue value) {
final String name = value.name.toLowerCase();
- final OptionValidator validator = getValidator(name);
+ final OptionDefinition definition = getOptionDefinition(name);
+ final OptionValidator validator = definition.getValidator();
+
+ validator.validate(value, definition.getMetaData(), this); // validate the option
- validator.validate(value, this); // validate the option
- if (options.get(name) == null && value.equals(validator.getDefault())) {
+ if (options.get(name) == null && value.equals(getDefault(name))) {
return; // if the option is not overridden, ignore setting option to default
}
+
options.put(name, value);
}
@Override
- public void deleteOption(final String name, OptionType type) {
- checkArgument(type == OptionType.SYSTEM, "OptionType must be SYSTEM.");
+ protected OptionValue.OptionScope getScope() {
+ return OptionValue.OptionScope.SYSTEM;
+ }
- getValidator(name); // ensure option exists
+ @Override
+ public void deleteLocalOption(final String name) {
+ getOptionDefinition(name); // ensure option exists
options.delete(name.toLowerCase());
}
@Override
- public void deleteAllOptions(OptionType type) {
- checkArgument(type == OptionType.SYSTEM, "OptionType must be SYSTEM.");
+ public void deleteAllLocalOptions() {
final Set<String> names = Sets.newHashSet();
for (final Map.Entry<String, OptionValue> entry : Lists.newArrayList(options.getAll())) {
names.add(entry.getKey());
@@ -346,17 +347,63 @@ public class SystemOptionManager extends BaseOptionManager implements OptionMana
}
}
- public void populateDefaultValues() {
-
+ public static CaseInsensitiveMap<OptionValue> populateDefaultValues(Map<String, OptionDefinition> definitions, DrillConfig bootConfig) {
// populate the options from the config
- final Map<String, OptionValidator> tmp = new HashMap<>();
- for (OptionValidator validator: VALIDATORS.values()) {
- try {
- validator.loadDefault(bootConfig);
- } catch (ConfigException.Missing e) {
- throw new IllegalStateException("Config Option is missing"+ validator.getOptionName());
+ final Map<String, OptionValue> defaults = new HashMap<>();
+
+ for (final Map.Entry<String, OptionDefinition> entry : definitions.entrySet()) {
+ final OptionDefinition definition = entry.getValue();
+ final OptionMetaData metaData = definition.getMetaData();
+ final OptionValue.AccessibleScopes type = metaData.getType();
+ final OptionValidator validator = definition.getValidator();
+ final String name = validator.getOptionName();
+ final String configName = validator.getConfigProperty();
+ final OptionValue.Kind kind = validator.getKind();
+ OptionValue optionValue;
+
+ switch (kind) {
+ case BOOLEAN:
+ optionValue = OptionValue.create(type, name,
+ bootConfig.getBoolean(configName), OptionValue.OptionScope.BOOT);
+ break;
+ case LONG:
+ optionValue = OptionValue.create(type, name,
+ bootConfig.getLong(configName), OptionValue.OptionScope.BOOT);
+ break;
+ case STRING:
+ optionValue = OptionValue.create(type, name,
+ bootConfig.getString(configName), OptionValue.OptionScope.BOOT);
+ break;
+ case DOUBLE:
+ optionValue = OptionValue.create(type, name,
+ bootConfig.getDouble(configName), OptionValue.OptionScope.BOOT);
+ break;
+ default:
+ throw new UnsupportedOperationException();
}
+
+ defaults.put(name, optionValue);
+ }
+
+ return CaseInsensitiveMap.newImmutableMap(defaults);
+ }
+
+ /**
+ * Gets the {@link OptionDefinition} associated with the name.
+ *
+ * @param name name of the option
+ * @return the associated option definition
+ * @throws UserException - if the definition is not found
+ */
+ @Override
+ public OptionDefinition getOptionDefinition(String name) {
+ final OptionDefinition definition = definitions.get(name);
+ if (definition == null) {
+ throw UserException.validationError()
+ .message(String.format("The option '%s' does not exist.", name.toLowerCase()))
+ .build(logger);
}
+ return definition;
}
@Override
http://git-wip-us.apache.org/repos/asf/drill/blob/6adeb986/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java
index ecaa7b7..b9861a6 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java
@@ -20,17 +20,13 @@ package org.apache.drill.exec.server.options;
import java.util.Set;
import com.google.common.collect.Sets;
-import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.exec.server.options.OptionValue.Kind;
-import org.apache.drill.exec.server.options.OptionValue.OptionType;
-import org.apache.drill.exec.server.options.OptionValue.OptionScope;
-import static com.google.common.base.Preconditions.checkArgument;
public class TypeValidators {
private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TypeValidators.class);
public static class PositiveLongValidator extends LongValidator {
- private final long max;
+ protected final long max;
public PositiveLongValidator(String name, long max) {
super(name);
@@ -38,8 +34,8 @@ public class TypeValidators {
}
@Override
- public void validate(final OptionValue v, final OptionSet manager) {
- super.validate(v, manager);
+ public void validate(final OptionValue v, final OptionMetaData metaData, final OptionSet manager) {
+ super.validate(v, metaData, manager);
if (v.num_val > max || v.num_val < 1) {
throw UserException.validationError()
.message(String.format("Option %s must be between %d and %d.", getOptionName(), 1, max))
@@ -55,8 +51,8 @@ public class TypeValidators {
}
@Override
- public void validate(final OptionValue v, final OptionSet manager) {
- super.validate(v, manager);
+ public void validate(final OptionValue v, final OptionMetaData metaData, final OptionSet manager) {
+ super.validate(v, metaData, manager);
if (!isPowerOfTwo(v.num_val)) {
throw UserException.validationError()
.message(String.format("Option %s must be a power of two.", getOptionName()))
@@ -70,8 +66,8 @@ public class TypeValidators {
}
public static class RangeDoubleValidator extends DoubleValidator {
- private final double min;
- private final double max;
+ protected final double min;
+ protected final double max;
public RangeDoubleValidator(String name, double min, double max) {
super(name);
@@ -80,8 +76,8 @@ public class TypeValidators {
}
@Override
- public void validate(final OptionValue v, final OptionSet manager) {
- super.validate(v, manager);
+ public void validate(final OptionValue v, final OptionMetaData metaData, final OptionSet manager) {
+ super.validate(v, metaData, manager);
if (v.float_val > max || v.float_val < min) {
throw UserException.validationError()
.message(String.format("Option %s must be between %f and %f.", getOptionName(), min, max))
@@ -93,14 +89,14 @@ public class TypeValidators {
public static class MinRangeDoubleValidator extends RangeDoubleValidator {
private final String maxValidatorName;
- public MinRangeDoubleValidator(String name, double min, double max, double def, String maxValidatorName) {
+ public MinRangeDoubleValidator(String name, double min, double max, String maxValidatorName) {
super(name, min, max);
this.maxValidatorName = maxValidatorName;
}
@Override
- public void validate(final OptionValue v, final OptionSet manager) {
- super.validate(v, manager);
+ public void validate(final OptionValue v, final OptionMetaData metaData, final OptionSet manager) {
+ super.validate(v, metaData, manager);
OptionValue maxValue = manager.getOption(maxValidatorName);
if (v.float_val > maxValue.float_val) {
throw UserException.validationError()
@@ -114,14 +110,14 @@ public class TypeValidators {
public static class MaxRangeDoubleValidator extends RangeDoubleValidator {
private final String minValidatorName;
- public MaxRangeDoubleValidator(String name, double min, double max, double def, String minValidatorName) {
+ public MaxRangeDoubleValidator(String name, double min, double max, String minValidatorName) {
super(name, min, max);
this.minValidatorName = minValidatorName;
}
@Override
- public void validate(final OptionValue v, final OptionSet manager) {
- super.validate(v, manager);
+ public void validate(final OptionValue v, final OptionMetaData metaData, final OptionSet manager) {
+ super.validate(v, metaData, manager);
OptionValue minValue = manager.getOption(minValidatorName);
if (v.float_val < minValue.float_val) {
throw UserException.validationError()
@@ -134,60 +130,25 @@ public class TypeValidators {
public static class BooleanValidator extends TypeValidator {
public BooleanValidator(String name) {
- this(name, false);
- }
-
- public BooleanValidator(String name, boolean isAdminOption) {
- super(name, Kind.BOOLEAN, isAdminOption);
- }
-
- public void loadDefault(DrillConfig bootConfig){
- OptionValue value = OptionValue.createBoolean(OptionType.SYSTEM, getOptionName(), bootConfig.getBoolean(getConfigProperty()), OptionScope.BOOT);
- setDefaultValue(value);
+ super(name, Kind.BOOLEAN);
}
}
public static class StringValidator extends TypeValidator {
public StringValidator(String name) {
- this(name, false);
- }
- public StringValidator(String name, boolean isAdminOption) {
- super(name, Kind.STRING, isAdminOption);
- }
-
- public void loadDefault(DrillConfig bootConfig){
- OptionValue value = OptionValue.createString(OptionType.SYSTEM, getOptionName(), bootConfig.getString(getConfigProperty()), OptionScope.BOOT);
- setDefaultValue(value);
+ super(name, Kind.STRING);
}
}
public static class LongValidator extends TypeValidator {
public LongValidator(String name) {
- this(name, false);
- }
-
- public LongValidator(String name, boolean isAdminOption) {
- super(name, Kind.LONG, isAdminOption);
- }
-
- public void loadDefault(DrillConfig bootConfig){
- OptionValue value = OptionValue.createLong(OptionType.SYSTEM, getOptionName(), bootConfig.getLong(getConfigProperty()), OptionScope.BOOT);
- setDefaultValue(value);
+ super(name, Kind.LONG);
}
}
public static class DoubleValidator extends TypeValidator {
public DoubleValidator(String name) {
- this(name,false);
- }
-
- public DoubleValidator(String name, boolean isAdminOption) {
- super(name, Kind.DOUBLE, isAdminOption);
- }
-
- public void loadDefault(DrillConfig bootConfig){
- OptionValue value = OptionValue.createDouble(OptionType.SYSTEM, getOptionName(),bootConfig.getDouble(getConfigProperty()), OptionScope.BOOT);
- setDefaultValue(value);
+ super(name, Kind.DOUBLE);
}
}
@@ -202,8 +163,8 @@ public class TypeValidators {
}
@Override
- public void validate(final OptionValue v, final OptionSet manager) {
- super.validate(v, manager);
+ public void validate(final OptionValue v, final OptionMetaData metaData, final OptionSet manager) {
+ super.validate(v, metaData, manager);
if (v.num_val > max || v.num_val < min) {
throw UserException.validationError()
.message(String.format("Option %s must be between %d and %d.", getOptionName(), min, max))
@@ -226,8 +187,8 @@ public class TypeValidators {
}
@Override
- public void validate(final OptionValue v, final OptionSet manager) {
- super.validate(v, manager);
+ public void validate(final OptionValue v, final OptionMetaData metaData, final OptionSet manager) {
+ super.validate(v, metaData, manager);
if (!valuesSet.contains(v.string_val.toLowerCase())) {
throw UserException.validationError()
.message(String.format("Option %s must be one of: %s.", getOptionName(), valuesSet))
@@ -242,18 +203,8 @@ public class TypeValidators {
* the available number of processors and cpu load average
*/
public static class MaxWidthValidator extends LongValidator{
-
public MaxWidthValidator(String name) {
- this(name, false);
- }
-
- public MaxWidthValidator(String name, boolean isAdminOption) {
- super(name, isAdminOption);
- }
-
- public void loadDefault(DrillConfig bootConfig) {
- OptionValue value = OptionValue.createLong(OptionType.SYSTEM, getOptionName(), bootConfig.getLong(getConfigProperty()), OptionScope.BOOT);
- setDefaultValue(value);
+ super(name);
}
public int computeMaxWidth(double cpuLoadAverage, long maxWidth) {
@@ -272,48 +223,25 @@ public class TypeValidators {
public static abstract class TypeValidator extends OptionValidator {
private final Kind kind;
- private OptionValue defaultValue = null;
public TypeValidator(final String name, final Kind kind) {
- this(name, kind, false);
- }
-
- public TypeValidator(final String name, final Kind kind, final boolean isAdminOption) {
- super(name, isAdminOption);
+ super(name);
this.kind = kind;
}
@Override
- public OptionValue getDefault() {
- return defaultValue;
- }
-
- @Override
- public void validate(final OptionValue v, final OptionSet manager) {
+ public void validate(final OptionValue v, final OptionMetaData metaData, final OptionSet manager) {
if (v.kind != kind) {
throw UserException.validationError()
.message(String.format("Option %s must be of type %s but you tried to set to %s.", getOptionName(),
kind.name(), v.kind.name()))
.build(logger);
}
- if (isAdminOption() && v.type != OptionType.SYSTEM) {
- throw UserException.validationError()
- .message("Admin related settings can only be set at SYSTEM level scope. Given scope '%s'.", v.type)
- .build(logger);
- }
}
@Override
public Kind getKind() {
return kind;
}
-
- protected void setDefaultValue(OptionValue defaultValue) {
- this.defaultValue = defaultValue;
- }
-
- public String getConfigProperty() {
- return OPTION_DEFAULTS_ROOT + getOptionName();
- }
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/6adeb986/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java
index 1caf10b..4042a81 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java
@@ -33,10 +33,14 @@ import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.SecurityContext;
import javax.xml.bind.annotation.XmlRootElement;
+import com.fasterxml.jackson.annotation.JsonProperty;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
+import org.apache.drill.exec.server.options.OptionList;
+import org.apache.drill.exec.server.options.OptionManager;
import org.apache.drill.exec.server.options.OptionValue;
import org.apache.drill.exec.server.options.OptionValue.Kind;
+import org.apache.drill.exec.server.options.SystemOptionManager;
import org.apache.drill.exec.server.rest.DrillRestServer.UserAuthEnabled;
import org.apache.drill.exec.server.rest.auth.DrillUserPrincipal;
import org.apache.drill.exec.work.WorkManager;
@@ -50,79 +54,128 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
public class StatusResources {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(StatusResources.class);
+ public static final String REST_API_SUFFIX = ".json";
+ public static final String PATH_STATUS_JSON = "/status" + REST_API_SUFFIX;
+ public static final String PATH_STATUS = "/status";
+ public static final String PATH_OPTIONS_JSON = "/options" + REST_API_SUFFIX;
+ public static final String PATH_INTERNAL_OPTIONS_JSON = "/internal_options" + REST_API_SUFFIX;
+ public static final String PATH_OPTIONS = "/options";
+ public static final String PATH_INTERNAL_OPTIONS = "/internal_options";
+
@Inject UserAuthEnabled authEnabled;
@Inject WorkManager work;
@Inject SecurityContext sc;
@GET
- @Path("/status.json")
+ @Path(StatusResources.PATH_STATUS_JSON)
@Produces(MediaType.APPLICATION_JSON)
public Pair<String, String> getStatusJSON() {
return new ImmutablePair<>("status", "Running!");
}
@GET
- @Path("/status")
+ @Path(StatusResources.PATH_STATUS)
@Produces(MediaType.TEXT_HTML)
public Viewable getStatus() {
return ViewableWithPermissions.create(authEnabled.get(), "/rest/status.ftl", sc, getStatusJSON());
}
- @GET
- @Path("/options.json")
- @RolesAllowed(DrillUserPrincipal.AUTHENTICATED_ROLE)
- @Produces(MediaType.APPLICATION_JSON)
- public List<OptionWrapper> getSystemOptionsJSON() {
+ private List<OptionWrapper> getSystemOptionsJSONHelper(boolean internal)
+ {
List<OptionWrapper> options = new LinkedList<>();
- for (OptionValue option : work.getContext().getOptionManager()) {
- options.add(new OptionWrapper(option.name, option.getValue(), option.type, option.kind));
+ OptionManager optionManager = work.getContext().getOptionManager();
+ OptionList optionList = internal ? optionManager.getInternalOptionList(): optionManager.getPublicOptionList();
+
+ for (OptionValue option : optionList) {
+ options.add(new OptionWrapper(option.name, option.getValue(), option.accessibleScopes, option.kind, option.scope));
}
+
return options;
}
@GET
- @Path("/options")
+ @Path(StatusResources.PATH_OPTIONS_JSON)
+ @RolesAllowed(DrillUserPrincipal.AUTHENTICATED_ROLE)
+ @Produces(MediaType.APPLICATION_JSON)
+ public List<OptionWrapper> getSystemPublicOptionsJSON() {
+ return getSystemOptionsJSONHelper(false);
+ }
+
+ @GET
+ @Path(StatusResources.PATH_INTERNAL_OPTIONS_JSON)
+ @RolesAllowed(DrillUserPrincipal.AUTHENTICATED_ROLE)
+ @Produces(MediaType.APPLICATION_JSON)
+ public List<OptionWrapper> getSystemInternalOptionsJSON() {
+ return getSystemOptionsJSONHelper(true);
+ }
+
+ private Viewable getSystemOptionsHelper(boolean internal) {
+ return ViewableWithPermissions.create(authEnabled.get(),
+ "/rest/options.ftl",
+ sc,
+ getSystemOptionsJSONHelper(internal));
+ }
+
+ @GET
+ @Path(StatusResources.PATH_OPTIONS)
+ @RolesAllowed(DrillUserPrincipal.AUTHENTICATED_ROLE)
+ @Produces(MediaType.TEXT_HTML)
+ public Viewable getSystemPublicOptions() {
+ return getSystemOptionsHelper(false);
+ }
+
+ @GET
+ @Path(StatusResources.PATH_INTERNAL_OPTIONS)
@RolesAllowed(DrillUserPrincipal.AUTHENTICATED_ROLE)
@Produces(MediaType.TEXT_HTML)
- public Viewable getSystemOptions() {
- return ViewableWithPermissions.create(authEnabled.get(), "/rest/options.ftl", sc, getSystemOptionsJSON());
+ public Viewable getSystemInternalOptions() {
+ return getSystemOptionsHelper(true);
}
@POST
- @Path("/option/{optionName}")
+ @Path("option/{optionName}")
@RolesAllowed(DrillUserPrincipal.ADMIN_ROLE)
@Consumes("application/x-www-form-urlencoded")
@Produces(MediaType.TEXT_HTML)
- public Viewable updateSystemOption(@FormParam("name") String name, @FormParam("value") String value,
+ public Viewable updateSystemOption(@FormParam("name") String name,
+ @FormParam("value") String value,
@FormParam("kind") String kind) {
+ SystemOptionManager optionManager = work.getContext()
+ .getOptionManager();
+
try {
- work.getContext()
- .getOptionManager()
- .setOption(OptionValue.createOption(
- OptionValue.Kind.valueOf(kind),
- OptionValue.OptionType.SYSTEM,
- name,
- value, OptionValue.OptionScope.SYSTEM));
+ optionManager.setLocalOption(OptionValue.Kind.valueOf(kind), name, value);
} catch (Exception e) {
logger.debug("Could not update.", e);
}
- return getSystemOptions();
+
+ if (optionManager.getOptionDefinition(name).getMetaData().isInternal()) {
+ return getSystemInternalOptions();
+ } else {
+ return getSystemPublicOptions();
+ }
}
@XmlRootElement
- public class OptionWrapper {
+ public static class OptionWrapper {
private String name;
private Object value;
- private OptionValue.OptionType type;
+ private OptionValue.AccessibleScopes accessibleScopes;
private String kind;
+ private String optionScope;
@JsonCreator
- public OptionWrapper(String name, Object value, OptionValue.OptionType type, Kind kind) {
+ public OptionWrapper(@JsonProperty("name") String name,
+ @JsonProperty("value") Object value,
+ @JsonProperty("accessibleScopes") OptionValue.AccessibleScopes type,
+ @JsonProperty("kind") Kind kind,
+ @JsonProperty("optionScope") OptionValue.OptionScope scope) {
this.name = name;
this.value = value;
- this.type = type;
+ this.accessibleScopes = type;
this.kind = kind.name();
+ this.optionScope = scope.name();
}
public String getName() {
@@ -138,13 +191,21 @@ public class StatusResources {
return value;
}
- public OptionValue.OptionType getType() {
- return type;
+ public OptionValue.AccessibleScopes getAccessibleScopes() {
+ return accessibleScopes;
}
public String getKind() {
return kind;
}
- }
+ public String getOptionScope() {
+ return optionScope;
+ }
+
+ @Override
+ public String toString() {
+ return "OptionWrapper{" + "name='" + name + '\'' + ", value=" + value + ", accessibleScopes=" + accessibleScopes + ", kind='" + kind + '\'' + ", scope='" + optionScope + '\'' +'}';
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/6adeb986/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebServer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebServer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebServer.java
index 8818349..0718fdd 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebServer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebServer.java
@@ -71,6 +71,7 @@ import javax.servlet.DispatcherType;
import javax.servlet.http.HttpSession;
import javax.servlet.http.HttpSessionEvent;
import javax.servlet.http.HttpSessionListener;
+import java.io.IOException;
import java.math.BigInteger;
import java.security.KeyPair;
import java.security.KeyPairGenerator;
@@ -91,16 +92,18 @@ import static org.apache.drill.exec.server.rest.auth.DrillUserPrincipal.AUTHENTI
public class WebServer implements AutoCloseable {
private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(WebServer.class);
+ private static final int PORT_HUNT_TRIES = 10;
+
private final DrillConfig config;
private final MetricRegistry metrics;
private final WorkManager workManager;
- private final Server embeddedJetty;
-
private final BootStrapContext context;
+ private Server embeddedJetty;
+
/**
* Create Jetty based web server.
*
@@ -112,12 +115,6 @@ public class WebServer implements AutoCloseable {
this.config = context.getConfig();
this.metrics = context.getMetrics();
this.workManager = workManager;
-
- if (config.getBoolean(ExecConstants.HTTP_ENABLE)) {
- embeddedJetty = new Server();
- } else {
- embeddedJetty = null;
- }
}
private static final String BASE_STATIC_PATH = "/rest/static/";
@@ -139,9 +136,10 @@ public class WebServer implements AutoCloseable {
* @throws Exception
*/
public void start() throws Exception {
- if (embeddedJetty == null) {
+ if (!config.getBoolean(ExecConstants.HTTP_ENABLE)) {
return;
}
+
final boolean authEnabled = config.getBoolean(ExecConstants.USER_AUTHENTICATION_ENABLED);
if (authEnabled && !context.getAuthProvider().containsFactory(PlainFactory.SIMPLE_NAME)) {
logger.warn("Not starting web server. Currently Drill supports web authentication only through " +
@@ -149,19 +147,6 @@ public class WebServer implements AutoCloseable {
return;
}
- final ServerConnector serverConnector;
- if (config.getBoolean(ExecConstants.HTTP_ENABLE_SSL)) {
- try {
- serverConnector = createHttpsConnector();
- }
- catch(DrillException e){
- throw new DrillbitStartupException(e.getMessage(), e);
- }
- } else {
- serverConnector = createHttpConnector();
- }
- embeddedJetty.addConnector(serverConnector);
-
// Add resources
final ErrorHandler errorHandler = new ErrorHandler();
errorHandler.setShowStacks(true);
@@ -216,8 +201,36 @@ public class WebServer implements AutoCloseable {
}
}
- embeddedJetty.setHandler(servletContextHandler);
- embeddedJetty.start();
+ int port = config.getInt(ExecConstants.HTTP_PORT);
+ boolean portHunt = config.getBoolean(ExecConstants.HTTP_PORT_HUNT);
+ int retry = 0;
+
+ System.out.println("Port hunting " + portHunt);
+
+ for (; retry < PORT_HUNT_TRIES; retry++) {
+ embeddedJetty = new Server();
+ embeddedJetty.setHandler(servletContextHandler);
+ embeddedJetty.addConnector(createConnector(port));
+
+ try {
+ embeddedJetty.start();
+ } catch (Exception e) {
+ if (portHunt) {
+ int nextPort = port++;
+ logger.info("Failed to start on port {}, trying port {}", port, nextPort);
+ logger.error("Attempted port failure:", e);
+ embeddedJetty.stop();
+ } else {
+ throw e;
+ }
+ }
+
+ break;
+ }
+
+ if (retry == PORT_HUNT_TRIES) {
+ throw new IOException("Failed to find a port");
+ }
}
/**
@@ -275,6 +288,22 @@ public class WebServer implements AutoCloseable {
return security;
}
+ private ServerConnector createConnector(int port) throws Exception {
+ final ServerConnector serverConnector;
+ if (config.getBoolean(ExecConstants.HTTP_ENABLE_SSL)) {
+ try {
+ serverConnector = createHttpsConnector(port);
+ }
+ catch(DrillException e){
+ throw new DrillbitStartupException(e.getMessage(), e);
+ }
+ } else {
+ serverConnector = createHttpConnector(port);
+ }
+
+ return serverConnector;
+ }
+
/**
* Create an HTTPS connector for given jetty server instance. If the admin has specified keystore/truststore settings
* they will be used else a self-signed certificate is generated and used.
@@ -282,7 +311,7 @@ public class WebServer implements AutoCloseable {
* @return Initialized {@link ServerConnector} for HTTPS connectios.
* @throws Exception
*/
- private ServerConnector createHttpsConnector() throws Exception {
+ private ServerConnector createHttpsConnector(int port) throws Exception {
logger.info("Setting up HTTPS connector for web server");
final SslContextFactory sslContextFactory = new SslContextFactory();
@@ -359,7 +388,7 @@ public class WebServer implements AutoCloseable {
final ServerConnector sslConnector = new ServerConnector(embeddedJetty,
new SslConnectionFactory(sslContextFactory, HttpVersion.HTTP_1_1.asString()),
new HttpConnectionFactory(httpsConfig));
- sslConnector.setPort(config.getInt(ExecConstants.HTTP_PORT));
+ sslConnector.setPort(port);
return sslConnector;
}
@@ -369,11 +398,11 @@ public class WebServer implements AutoCloseable {
* @return Initialized {@link ServerConnector} instance for HTTP connections.
* @throws Exception
*/
- private ServerConnector createHttpConnector() throws Exception {
+ private ServerConnector createHttpConnector(int port) throws Exception {
logger.info("Setting up HTTP connector for web server");
final HttpConfiguration httpConfig = new HttpConfiguration();
final ServerConnector httpConnector = new ServerConnector(embeddedJetty, new HttpConnectionFactory(httpConfig));
- httpConnector.setPort(config.getInt(ExecConstants.HTTP_PORT));
+ httpConnector.setPort(port);
return httpConnector;
}
http://git-wip-us.apache.org/repos/asf/drill/blob/6adeb986/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/ExtendedOptionIterator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/ExtendedOptionIterator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/ExtendedOptionIterator.java
index fdd5092..e329de3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/ExtendedOptionIterator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/ExtendedOptionIterator.java
@@ -30,10 +30,8 @@ import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.server.options.OptionManager;
import org.apache.drill.exec.server.options.OptionValue;
import org.apache.drill.exec.server.options.OptionValue.Kind;
-import org.apache.drill.exec.server.options.OptionValue.OptionType;
import org.apache.drill.exec.server.options.OptionValue.OptionScope;
-
/*
* Extends the original Option iterator. The idea is to hide the implementation details and present the
* user with the rows which have values set at the top level of hierarchy and exclude the values set
@@ -59,9 +57,15 @@ public class ExtendedOptionIterator implements Iterator<Object> {
private final OptionManager fragmentOptions;
private final Iterator<OptionValue> mergedOptions;
- public ExtendedOptionIterator(FragmentContext context) {
+ public ExtendedOptionIterator(FragmentContext context, boolean internal) {
fragmentOptions = context.getOptions();
- mergedOptions = sortOptions(fragmentOptions.iterator());
+ final Iterator<OptionValue> optionList;
+
+ if (!internal) {
+ mergedOptions = sortOptions(fragmentOptions.getPublicOptionList().iterator());
+ } else {
+ mergedOptions = sortOptions(fragmentOptions.getInternalOptionList().iterator());
+ }
}
/* *
* Remove the redundant rows for the same option based on the scope and return
@@ -118,7 +122,7 @@ public class ExtendedOptionIterator implements Iterator<Object> {
put(Kind.BOOLEAN,"BIT");
}};
- return new ExtendedOptionValueWrapper(value.name, typeMapping.get(value.kind), value.type,value.getValue().toString(), value.scope);
+ return new ExtendedOptionValueWrapper(value.name, typeMapping.get(value.kind), value.accessibleScopes,value.getValue().toString(), value.scope);
}
public enum Status {
@@ -132,15 +136,15 @@ public class ExtendedOptionIterator implements Iterator<Object> {
public final String name;
public final String kind;
- public final OptionType type;
+ public final OptionValue.AccessibleScopes accessibleScopes;
public final String val;
public final OptionScope optionScope;
- public ExtendedOptionValueWrapper(final String name, final String kind, final OptionType type, final String value, final OptionScope scope) {
+ public ExtendedOptionValueWrapper(final String name, final String kind, final OptionValue.AccessibleScopes type, final String value, final OptionScope scope) {
this.name = name;
this.kind = kind;
- this.type = type;
+ this.accessibleScopes = type;
this.val = value;
this.optionScope = scope;
}
http://git-wip-us.apache.org/repos/asf/drill/blob/6adeb986/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/OptionIterator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/OptionIterator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/OptionIterator.java
index 424ff95..48d8c10 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/OptionIterator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/OptionIterator.java
@@ -25,35 +25,41 @@ import com.google.common.collect.Iterators;
import com.google.common.collect.Lists;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.server.options.DrillConfigIterator;
-import org.apache.drill.exec.server.options.FragmentOptionManager;
import org.apache.drill.exec.server.options.OptionManager;
import org.apache.drill.exec.server.options.OptionValue;
import org.apache.drill.exec.server.options.OptionValue.Kind;
-import org.apache.drill.exec.server.options.OptionValue.OptionType;
-import org.apache.drill.exec.server.options.SystemOptionManager;
+import org.apache.drill.exec.server.options.OptionValue.OptionScope;
+import org.apache.drill.exec.server.options.OptionValue.AccessibleScopes;
public class OptionIterator implements Iterator<Object> {
// private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(OptionIterator.class);
enum Mode {
- BOOT, SYS_SESS, BOTH
+ BOOT,
+ // These represent System/Session options that are public. These are available for everyone to change and are considered safe to change.
+ SYS_SESS_PUBLIC,
+ // These represent System/Session options that are internal. These are segregated from the public options and are not visible in the same tables and rest endpoints as public
+ // options. These are technically available for anyone to change, but users are discouraged from doing so unless they absolutely know what they are doing. The general purpose
+ // for internal options is to enable support to easily debug.
+ SYS_SESS_INTERNAL
};
private final OptionManager fragmentOptions;
private final Iterator<OptionValue> mergedOptions;
- private FragmentContext context;
public OptionIterator(FragmentContext context, Mode mode){
final DrillConfigIterator configOptions = new DrillConfigIterator(context.getConfig());
fragmentOptions = context.getOptions();
- this.context = context;
final Iterator<OptionValue> optionList;
switch(mode){
case BOOT:
optionList = configOptions.iterator();
break;
- case SYS_SESS:
- optionList = fragmentOptions.iterator();
+ case SYS_SESS_PUBLIC:
+ optionList = fragmentOptions.getPublicOptionList().iterator();
+ break;
+ case SYS_SESS_INTERNAL:
+ optionList = fragmentOptions.getInternalOptionList().iterator();
break;
default:
optionList = Iterators.concat(configOptions.iterator(), fragmentOptions.iterator());
@@ -74,23 +80,22 @@ public class OptionIterator implements Iterator<Object> {
public OptionValueWrapper next() {
final OptionValue value = mergedOptions.next();
final Status status;
- final FragmentOptionManager fragmentOptionManager = (FragmentOptionManager) fragmentOptions;
- final SystemOptionManager systemOptionManager = (SystemOptionManager) fragmentOptionManager.getFallback();
- if (value.type == OptionType.BOOT) {
+
+ if (value.accessibleScopes == AccessibleScopes.BOOT) {
status = Status.BOOT;
} else {
- final OptionValue def = systemOptionManager.getValidator(value.name).getDefault();
+ final OptionValue def = fragmentOptions.getDefault(value.name);
if (value.equalsIgnoreType(def)) {
status = Status.DEFAULT;
} else {
status = Status.CHANGED;
}
}
- return new OptionValueWrapper(value.name, value.kind, value.type, value.num_val, value.string_val,
+ return new OptionValueWrapper(value.name, value.kind, value.accessibleScopes, value.scope, value.num_val, value.string_val,
value.bool_val, value.float_val, status);
}
- public static enum Status {
+ public enum Status {
BOOT, DEFAULT, CHANGED
}
@@ -101,19 +106,21 @@ public class OptionIterator implements Iterator<Object> {
public final String name;
public final Kind kind;
- public final OptionType type;
+ public final AccessibleScopes accessibleScopes;
+ public final OptionScope optionScope;
public final Status status;
public final Long num_val;
public final String string_val;
public final Boolean bool_val;
public final Double float_val;
- public OptionValueWrapper(final String name, final Kind kind, final OptionType type, final Long num_val,
- final String string_val, final Boolean bool_val, final Double float_val,
- final Status status) {
+ public OptionValueWrapper(final String name, final Kind kind, final OptionValue.AccessibleScopes type, final OptionScope scope, final Long num_val,
+ final String string_val, final Boolean bool_val, final Double float_val,
+ final Status status) {
this.name = name;
this.kind = kind;
- this.type = type;
+ this.accessibleScopes = type;
+ this.optionScope = scope;
this.num_val = num_val;
this.string_val = string_val;
this.bool_val = bool_val;
http://git-wip-us.apache.org/repos/asf/drill/blob/6adeb986/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTable.java
index fa6981b..ee798b9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTable.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTable.java
@@ -21,7 +21,6 @@ import java.util.Iterator;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.store.sys.OptionIterator.OptionValueWrapper;
-import org.apache.drill.exec.store.sys.ExtendedOptionIterator.ExtendedOptionValueWrapper;
/**
* An enumeration of all tables in Drill's system ("sys") schema.
@@ -32,18 +31,31 @@ import org.apache.drill.exec.store.sys.ExtendedOptionIterator.ExtendedOptionValu
* </p>
*/
public enum SystemTable {
-
OPTION("options", false, OptionValueWrapper.class) {
@Override
public Iterator<Object> getIterator(final FragmentContext context) {
- return new OptionIterator(context, OptionIterator.Mode.SYS_SESS);
+ return new OptionIterator(context, OptionIterator.Mode.SYS_SESS_PUBLIC);
+ }
+ },
+
+ OPTION_VAL("options_val", false, ExtendedOptionIterator.ExtendedOptionValueWrapper.class) {
+ @Override
+ public Iterator<Object> getIterator(final FragmentContext context) {
+ return new ExtendedOptionIterator(context, false);
+ }
+ },
+
+ INTERNAL_OPTIONS("internal_options", false, OptionValueWrapper.class) {
+ @Override
+ public Iterator<Object> getIterator(final FragmentContext context) {
+ return new OptionIterator(context, OptionIterator.Mode.SYS_SESS_INTERNAL);
}
},
- OPTION2("options2", false,ExtendedOptionIterator.ExtendedOptionValueWrapper.class ) {
+ INTERNAL_OPTIONS_VAL("internal_options_val", false, ExtendedOptionIterator.ExtendedOptionValueWrapper.class) {
@Override
public Iterator<Object> getIterator(final FragmentContext context) {
- return new ExtendedOptionIterator(context);
+ return new ExtendedOptionIterator(context, true);
}
},
http://git-wip-us.apache.org/repos/asf/drill/blob/6adeb986/exec/java-exec/src/main/java/org/apache/drill/exec/testing/ExecutionControls.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/testing/ExecutionControls.java b/exec/java-exec/src/main/java/org/apache/drill/exec/testing/ExecutionControls.java
index a046c77..82646ea 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/testing/ExecutionControls.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/testing/ExecutionControls.java
@@ -22,15 +22,14 @@ import com.fasterxml.jackson.annotation.JsonSubTypes.Type;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
-import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
import org.apache.drill.exec.server.options.OptionManager;
+import org.apache.drill.exec.server.options.OptionMetaData;
import org.apache.drill.exec.server.options.OptionSet;
import org.apache.drill.exec.server.options.OptionValue;
-import org.apache.drill.exec.server.options.OptionValue.OptionType;
import org.apache.drill.exec.server.options.TypeValidators.TypeValidator;
import org.apache.drill.exec.testing.InjectionSite.InjectionSiteKeyDeserializer;
import org.apache.drill.exec.util.AssertionUtil;
@@ -96,12 +95,7 @@ public final class ExecutionControls {
}
@Override
- public void validate(final OptionValue v, final OptionSet manager) {
- if (v.type != OptionType.SESSION) {
- throw UserException.validationError()
- .message("Controls can be set only at SESSION level.")
- .build(logger);
- }
+ public void validate(final OptionValue v, final OptionMetaData metaData, final OptionSet manager) {
final String jsonString = v.string_val;
try {
validateControlsString(jsonString);
@@ -111,11 +105,6 @@ public final class ExecutionControls {
.build(logger);
}
}
-
- public void loadDefault(DrillConfig bootConfig){
- OptionValue value = OptionValue.createString(OptionType.SYSTEM, getOptionName(), bootConfig.getString(getConfigProperty()), OptionValue.OptionScope.BOOT);
- setDefaultValue(value);
- }
}
/**
http://git-wip-us.apache.org/repos/asf/drill/blob/6adeb986/exec/java-exec/src/main/resources/drill-module.conf
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/resources/drill-module.conf b/exec/java-exec/src/main/resources/drill-module.conf
index 26f0722..4e7bbdb 100644
--- a/exec/java-exec/src/main/resources/drill-module.conf
+++ b/exec/java-exec/src/main/resources/drill-module.conf
@@ -122,6 +122,7 @@ drill.exec: {
http: {
enabled: true,
ssl_enabled: false,
+ porthunt: false,
port: 8047,
max_profiles: 100,
session_max_idle_secs: 3600, # Default value 1hr
http://git-wip-us.apache.org/repos/asf/drill/blob/6adeb986/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java b/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
index f7f41fc..e30c5e9 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
@@ -460,6 +460,18 @@ public class BaseTestQuery extends ExecTest {
return file.getPath();
}
+ protected static void setSessionOption(final String option, final boolean value) {
+ setSessionOption(option, Boolean.toString(value));
+ }
+
+ protected static void setSessionOption(final String option, final long value) {
+ setSessionOption(option, Long.toString(value));
+ }
+
+ protected static void setSessionOption(final String option, final double value) {
+ setSessionOption(option, Double.toString(value));
+ }
+
protected static void setSessionOption(final String option, final String value) {
try {
runSQL(String.format("alter session set `%s` = %s", option, value));
http://git-wip-us.apache.org/repos/asf/drill/blob/6adeb986/exec/java-exec/src/test/java/org/apache/drill/QueryTestUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/QueryTestUtil.java b/exec/java-exec/src/test/java/org/apache/drill/QueryTestUtil.java
index 8e767e4..26a0537 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/QueryTestUtil.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/QueryTestUtil.java
@@ -41,7 +41,7 @@ import org.apache.drill.exec.server.DrillbitContext;
import org.apache.drill.exec.server.RemoteServiceSet;
import org.apache.drill.exec.server.options.OptionManager;
import org.apache.drill.exec.server.options.OptionValue;
-import org.apache.drill.exec.server.options.OptionValue.OptionScope;
+import org.apache.drill.exec.server.options.SystemOptionManager;
import org.apache.drill.exec.util.VectorUtil;
/**
@@ -188,11 +188,9 @@ public class QueryTestUtil {
final Drillbit drillbit, final ClassTransformer.ScalarReplacementOption srOption) {
// set the system option
final DrillbitContext drillbitContext = drillbit.getContext();
- final OptionManager optionManager = drillbitContext.getOptionManager();
+ final SystemOptionManager optionManager = drillbitContext.getOptionManager();
final OptionValue originalOptionValue = optionManager.getOption(ClassTransformer.SCALAR_REPLACEMENT_OPTION);
- final OptionValue newOptionValue = OptionValue.createString(OptionValue.OptionType.SYSTEM,
- ClassTransformer.SCALAR_REPLACEMENT_OPTION, srOption.name().toLowerCase(), OptionScope.SYSTEM);
- optionManager.setOption(newOptionValue);
+ optionManager.setLocalOption(ClassTransformer.SCALAR_REPLACEMENT_OPTION, srOption.name().toLowerCase());
// flush the code cache
drillbitContext.getCompiler().flushCache();
@@ -209,12 +207,12 @@ public class QueryTestUtil {
* @param drillbit the drillbit
* @param srOption the scalar replacement option value to use
*/
- public static void restoreScalarReplacementOption(final Drillbit drillbit, final OptionValue srOption) {
+ public static void restoreScalarReplacementOption(final Drillbit drillbit, final String srOption) {
@SuppressWarnings("resource")
final DrillbitContext drillbitContext = drillbit.getContext();
@SuppressWarnings("resource")
final OptionManager optionManager = drillbitContext.getOptionManager();
- optionManager.setOption(srOption);
+ optionManager.setLocalOption(ClassTransformer.SCALAR_REPLACEMENT_OPTION, srOption);
// flush the code cache
drillbitContext.getCompiler().flushCache();
http://git-wip-us.apache.org/repos/asf/drill/blob/6adeb986/exec/java-exec/src/test/java/org/apache/drill/TestStarQueries.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestStarQueries.java b/exec/java-exec/src/test/java/org/apache/drill/TestStarQueries.java
index b9dcacd..cf3e9b9 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestStarQueries.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestStarQueries.java
@@ -244,7 +244,7 @@ public class TestStarQueries extends BaseTestQuery{
@Test // select star for a SchemaTable.
public void testSelStarSubQSchemaTable() throws Exception {
- test("select name, kind, type from (select * from sys.options);");
+ test("select name, kind, accessibleScopes from (select * from sys.options);");
}
@Test // Join a select star of SchemaTable, with a select star of Schema-less table.
http://git-wip-us.apache.org/repos/asf/drill/blob/6adeb986/exec/java-exec/src/test/java/org/apache/drill/exec/DrillSystemTestBase.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/DrillSystemTestBase.java b/exec/java-exec/src/test/java/org/apache/drill/exec/DrillSystemTestBase.java
index e63f085..2b3b475 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/DrillSystemTestBase.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/DrillSystemTestBase.java
@@ -40,7 +40,7 @@ public class DrillSystemTestBase extends TestWithZookeeper {
try {
ImmutableList.Builder<Drillbit> servers = ImmutableList.builder();
for (int i = 0; i < numServers; i++) {
- servers.add(Drillbit.start(getConfig()));
+ servers.add(Drillbit.start(zkHelper.getConfig()));
}
this.servers = servers.build();
} catch (DrillbitStartupException e) {
http://git-wip-us.apache.org/repos/asf/drill/blob/6adeb986/exec/java-exec/src/test/java/org/apache/drill/exec/TestWithZookeeper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/TestWithZookeeper.java b/exec/java-exec/src/test/java/org/apache/drill/exec/TestWithZookeeper.java
index f65c638..842a4e1 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/TestWithZookeeper.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/TestWithZookeeper.java
@@ -18,26 +18,24 @@
package org.apache.drill.exec;
import org.apache.drill.common.config.DrillConfig;
+import org.junit.After;
import org.junit.AfterClass;
+import org.junit.Before;
import org.junit.BeforeClass;
public class TestWithZookeeper extends ExecTest {
// private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestWithZookeeper.class);
- private static ZookeeperHelper zkHelper;
+ protected ZookeeperHelper zkHelper;
- @BeforeClass
- public static void setUp() throws Exception {
+ @Before
+ public void setUp() throws Exception {
zkHelper = new ZookeeperHelper();
zkHelper.startZookeeper(1);
}
- @AfterClass
- public static void tearDown() throws Exception {
+ @After
+ public void tearDown() throws Exception {
zkHelper.stopZookeeper();
}
-
- public static DrillConfig getConfig() {
- return zkHelper.getConfig();
- }
}
|