drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ar...@apache.org
Subject [drill] branch master updated: DRILL-6526: Refactor FileSystemConfig to disallow direct access from the code to its variables
Date Wed, 27 Jun 2018 16:14:23 GMT
This is an automated email from the ASF dual-hosted git repository.

arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git


The following commit(s) were added to refs/heads/master by this push:
     new c346859  DRILL-6526: Refactor FileSystemConfig to disallow direct access from the
code to its variables
c346859 is described below

commit c346859735e4eab3ad12e755e5643fceb9536f74
Author: Arina Ielchiieva <arina.yelchiyeva@gmail.com>
AuthorDate: Fri Jun 22 19:36:41 2018 +0300

    DRILL-6526: Refactor FileSystemConfig to disallow direct access from the code to its variables
---
 .../drill/exec/store/dfs/FileSystemConfig.java     | 44 ++++++++++++++++++---
 .../drill/exec/store/dfs/FileSystemPlugin.java     | 31 +++++++--------
 .../apache/drill/exec/store/dfs/FormatCreator.java | 11 +++---
 .../exec/store/easy/json/JSONFormatPlugin.java     | 14 ++++---
 .../exec/store/easy/text/TextFormatPlugin.java     | 26 ++++++------
 .../exec/store/parquet/ParquetFormatPlugin.java    | 10 ++---
 .../drill/exec/util/StoragePluginTestUtils.java    | 46 +++++++++++++++-------
 .../exec/impersonation/BaseTestImpersonation.java  | 21 ++++------
 .../java/org/apache/drill/exec/sql/TestCTTAS.java  | 17 +++++++-
 .../org/apache/drill/exec/sql/TestInfoSchema.java  |  2 +-
 .../java/org/apache/drill/test/ClusterFixture.java | 24 ++++++++---
 11 files changed, 158 insertions(+), 88 deletions(-)

diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemConfig.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemConfig.java
index 3c8f3a7..4eda955 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemConfig.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemConfig.java
@@ -19,6 +19,8 @@ package org.apache.drill.exec.store.dfs;
 
 import java.util.Map;
 
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
 import org.apache.drill.common.logical.FormatPluginConfig;
 import org.apache.drill.common.logical.StoragePluginConfig;
 
@@ -26,12 +28,44 @@ import com.fasterxml.jackson.annotation.JsonTypeName;
 
 @JsonTypeName(FileSystemConfig.NAME)
 public class FileSystemConfig extends StoragePluginConfig {
-  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FileSystemConfig.class);
+
   public static final String NAME = "file";
-  public String connection;
-  public Map<String, String> config;
-  public Map<String, WorkspaceConfig> workspaces;
-  public Map<String, FormatPluginConfig> formats;
+
+  private final String connection;
+  private final Map<String, String> config;
+  private final Map<String, WorkspaceConfig> workspaces;
+  private final Map<String, FormatPluginConfig> formats;
+
+  @JsonCreator
+  public FileSystemConfig(@JsonProperty("connection") String connection,
+                          @JsonProperty("config") Map<String, String> config,
+                          @JsonProperty("workspaces") Map<String, WorkspaceConfig>
workspaces,
+                          @JsonProperty("formats") Map<String, FormatPluginConfig>
formats) {
+    this.connection = connection;
+    this.config = config;
+    this.workspaces = workspaces;
+    this.formats = formats;
+  }
+
+  @JsonProperty
+  public String getConnection() {
+    return connection;
+  }
+
+  @JsonProperty
+  public Map<String, String> getConfig() {
+    return config;
+  }
+
+  @JsonProperty
+  public Map<String, WorkspaceConfig> getWorkspaces() {
+    return workspaces;
+  }
+
+  @JsonProperty
+  public Map<String, FormatPluginConfig> getFormats() {
+    return formats;
+  }
 
   @Override
   public int hashCode() {
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java
index 734ab73..e71e7e1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java
@@ -20,8 +20,11 @@ package org.apache.drill.exec.store.dfs;
 import static org.apache.drill.exec.store.dfs.FileSystemSchemaFactory.DEFAULT_WS_NAME;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 
 import org.apache.calcite.schema.SchemaPlus;
@@ -44,8 +47,6 @@ import org.apache.hadoop.fs.FileSystem;
 
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSet.Builder;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
 
 /**
  * A Storage engine associated with a Hadoop FileSystem Implementation. Examples include
HDFS, MapRFS, QuantacastFileSystem,
@@ -62,42 +63,38 @@ public class FileSystemPlugin extends AbstractStoragePlugin {
   private final Configuration fsConf;
   private final LogicalPlanPersistence lpPersistance;
 
-  public FileSystemPlugin(FileSystemConfig config, DrillbitContext context, String name)
-      throws ExecutionSetupException{
+  public FileSystemPlugin(FileSystemConfig config, DrillbitContext context, String name)
throws ExecutionSetupException {
     super(context, name);
     this.config = config;
     this.lpPersistance = context.getLpPersistence();
 
     try {
-
       fsConf = new Configuration();
-      if (config.config != null) {
-        for (String s : config.config.keySet()) {
-          fsConf.set(s, config.config.get(s));
-        }
-      }
-      fsConf.set(FileSystem.FS_DEFAULT_NAME_KEY, config.connection);
+      Optional.ofNullable(config.getConfig())
+          .ifPresent(c -> c.forEach(fsConf::set));
+
+      fsConf.set(FileSystem.FS_DEFAULT_NAME_KEY, config.getConnection());
       fsConf.set("fs.classpath.impl", ClassPathFileSystem.class.getName());
       fsConf.set("fs.drill-local.impl", LocalSyncableFileSystem.class.getName());
 
       formatCreator = newFormatCreator(config, context, fsConf);
-      List<FormatMatcher> matchers = Lists.newArrayList();
-      formatPluginsByConfig = Maps.newHashMap();
+      List<FormatMatcher> matchers = new ArrayList<>();
+      formatPluginsByConfig = new HashMap<>();
       for (FormatPlugin p : formatCreator.getConfiguredFormatPlugins()) {
         matchers.add(p.getMatcher());
         formatPluginsByConfig.put(p.getConfig(), p);
       }
 
-      final boolean noWorkspace = config.workspaces == null || config.workspaces.isEmpty();
-      List<WorkspaceSchemaFactory> factories = Lists.newArrayList();
+      boolean noWorkspace = config.getWorkspaces() == null || config.getWorkspaces().isEmpty();
+      List<WorkspaceSchemaFactory> factories = new ArrayList<>();
       if (!noWorkspace) {
-        for (Map.Entry<String, WorkspaceConfig> space : config.workspaces.entrySet())
{
+        for (Map.Entry<String, WorkspaceConfig> space : config.getWorkspaces().entrySet())
{
           factories.add(new WorkspaceSchemaFactory(this, space.getKey(), name, space.getValue(),
matchers, context.getLpPersistence(), context.getClasspathScan()));
         }
       }
 
       // if the "default" workspace is not given add one.
-      if (noWorkspace || !config.workspaces.containsKey(DEFAULT_WS_NAME)) {
+      if (noWorkspace || !config.getWorkspaces().containsKey(DEFAULT_WS_NAME)) {
         factories.add(new WorkspaceSchemaFactory(this, DEFAULT_WS_NAME, name, WorkspaceConfig.DEFAULT,
matchers, context.getLpPersistence(), context.getClasspathScan()));
       }
 
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatCreator.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatCreator.java
index fe9014b..b981adf 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatCreator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatCreator.java
@@ -21,6 +21,7 @@ import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.Map;
 
 import org.apache.drill.common.exceptions.UserException;
@@ -31,8 +32,6 @@ import org.apache.drill.common.util.ConstructorChecker;
 import org.apache.drill.exec.server.DrillbitContext;
 import org.apache.hadoop.conf.Configuration;
 
-import com.google.common.collect.Maps;
-
 /**
  * Responsible for instantiating format plugins
  */
@@ -51,7 +50,7 @@ public class FormatCreator {
    * @return a map of type to constructor that taks the config
    */
   private static Map<Class<?>, Constructor<?>> initConfigConstructors(Collection<Class<?
extends FormatPlugin>> pluginClasses) {
-    Map<Class<?>, Constructor<?>> constructors = Maps.newHashMap();
+    Map<Class<?>, Constructor<?>> constructors = new HashMap<>();
     for (Class<? extends FormatPlugin> pluginClass: pluginClasses) {
       for (Constructor<?> c : pluginClass.getConstructors()) {
         try {
@@ -91,8 +90,8 @@ public class FormatCreator {
     this.pluginClasses = classpathScan.getImplementations(FormatPlugin.class);
     this.configConstructors = initConfigConstructors(pluginClasses);
 
-    Map<String, FormatPlugin> plugins = Maps.newHashMap();
-    if (storageConfig.formats == null || storageConfig.formats.isEmpty()) {
+    Map<String, FormatPlugin> plugins = new HashMap<>();
+    if (storageConfig.getFormats() == null || storageConfig.getFormats().isEmpty()) {
       for (Class<? extends FormatPlugin> pluginClass: pluginClasses) {
         for (Constructor<?> c : pluginClass.getConstructors()) {
           try {
@@ -107,7 +106,7 @@ public class FormatCreator {
         }
       }
     } else {
-      for (Map.Entry<String, FormatPluginConfig> e : storageConfig.formats.entrySet())
{
+      for (Map.Entry<String, FormatPluginConfig> e : storageConfig.getFormats().entrySet())
{
         Constructor<?> c = configConstructors.get(e.getValue().getClass());
         if (c == null) {
           logger.warn("Unable to find constructor for storage config named '{}' of type '{}'.",
e.getKey(), e.getValue().getClass().getName());
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java
index 095e09a..5eec5cc 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java
@@ -18,10 +18,10 @@
 package org.apache.drill.exec.store.easy.json;
 
 import java.io.IOException;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-import org.apache.drill.common.exceptions.ExecutionSetupException;
 import org.apache.drill.common.expression.SchemaPath;
 import org.apache.drill.common.logical.FormatPluginConfig;
 import org.apache.drill.common.logical.StoragePluginConfig;
@@ -44,7 +44,6 @@ import org.apache.hadoop.fs.FileSystem;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonTypeName;
 import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Maps;
 
 public class JSONFormatPlugin extends EasyFormatPlugin<JSONFormatConfig> {
 
@@ -60,14 +59,17 @@ public class JSONFormatPlugin extends EasyFormatPlugin<JSONFormatConfig>
{
   }
 
   @Override
-  public RecordReader getRecordReader(FragmentContext context, DrillFileSystem dfs, FileWork
fileWork,
-      List<SchemaPath> columns, String userName) throws ExecutionSetupException {
+  public RecordReader getRecordReader(FragmentContext context,
+                                      DrillFileSystem dfs,
+                                      FileWork fileWork,
+                                      List<SchemaPath> columns,
+                                      String userName) {
     return new JSONRecordReader(context, fileWork.getPath(), dfs, columns);
   }
 
   @Override
   public RecordWriter getRecordWriter(FragmentContext context, EasyWriter writer) throws
IOException {
-    Map<String, String> options = Maps.newHashMap();
+    Map<String, String> options = new HashMap<>();
 
     options.put("location", writer.getLocation());
 
@@ -76,7 +78,7 @@ public class JSONFormatPlugin extends EasyFormatPlugin<JSONFormatConfig>
{
     options.put("prefix", fragmentId);
 
     options.put("separator", " ");
-    options.put(FileSystem.FS_DEFAULT_NAME_KEY, ((FileSystemConfig)writer.getStorageConfig()).connection);
+    options.put(FileSystem.FS_DEFAULT_NAME_KEY, ((FileSystemConfig) writer.getStorageConfig()).getConnection());
 
     options.put("extension", "json");
     options.put("extended", Boolean.toString(context.getOptions().getOption(ExecConstants.JSON_EXTENDED_TYPES)));
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/TextFormatPlugin.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/TextFormatPlugin.java
index 8209252..b61ce30 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/TextFormatPlugin.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/TextFormatPlugin.java
@@ -19,10 +19,10 @@ package org.apache.drill.exec.store.easy.text;
 
 import java.io.IOException;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-import org.apache.drill.common.exceptions.ExecutionSetupException;
 import org.apache.drill.common.expression.SchemaPath;
 import org.apache.drill.common.logical.FormatPluginConfig;
 import org.apache.drill.common.logical.StoragePluginConfig;
@@ -60,14 +60,13 @@ import com.fasterxml.jackson.annotation.JsonInclude.Include;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonTypeName;
 import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Maps;
 
 public class TextFormatPlugin extends EasyFormatPlugin<TextFormatPlugin.TextFormatConfig>
{
   private final static String DEFAULT_NAME = "text";
 
   public TextFormatPlugin(String name, DrillbitContext context, Configuration fsConf, StoragePluginConfig
storageConfig) {
     super(name, context, fsConf, storageConfig, new TextFormatConfig(), true, false, true,
true,
-        Collections.<String>emptyList(), DEFAULT_NAME);
+        Collections.emptyList(), DEFAULT_NAME);
   }
 
   public TextFormatPlugin(String name, DrillbitContext context, Configuration fsConf, StoragePluginConfig
config,
@@ -78,17 +77,20 @@ public class TextFormatPlugin extends EasyFormatPlugin<TextFormatPlugin.TextForm
 
 
   @Override
-  public RecordReader getRecordReader(FragmentContext context, DrillFileSystem dfs, FileWork
fileWork,
-      List<SchemaPath> columns, String userName) throws ExecutionSetupException {
+  public RecordReader getRecordReader(FragmentContext context,
+                                      DrillFileSystem dfs,
+                                      FileWork fileWork,
+                                      List<SchemaPath> columns,
+                                      String userName) {
     Path path = dfs.makeQualified(new Path(fileWork.getPath()));
     FileSplit split = new FileSplit(path, fileWork.getStart(), fileWork.getLength(), new
String[]{""});
 
-    if (context.getOptions().getOption(ExecConstants.ENABLE_NEW_TEXT_READER_KEY).bool_val
== true) {
+    if (context.getOptions().getBoolean(ExecConstants.ENABLE_NEW_TEXT_READER_KEY)) {
       TextParsingSettings settings = new TextParsingSettings();
-      settings.set((TextFormatConfig)formatConfig);
+      settings.set(formatConfig);
       return new CompliantTextRecordReader(split, dfs, settings, columns);
     } else {
-      char delim = ((TextFormatConfig)formatConfig).getFieldDelimiter();
+      char delim = formatConfig.getFieldDelimiter();
       return new DrillTextRecordReader(split, dfs.getConf(), context, delim, columns);
     }
   }
@@ -112,7 +114,7 @@ public class TextFormatPlugin extends EasyFormatPlugin<TextFormatPlugin.TextForm
 
   @Override
   public RecordWriter getRecordWriter(final FragmentContext context, final EasyWriter writer)
throws IOException {
-    final Map<String, String> options = Maps.newHashMap();
+    final Map<String, String> options = new HashMap<>();
 
     options.put("location", writer.getLocation());
 
@@ -120,10 +122,10 @@ public class TextFormatPlugin extends EasyFormatPlugin<TextFormatPlugin.TextForm
     String fragmentId = String.format("%d_%d", handle.getMajorFragmentId(), handle.getMinorFragmentId());
     options.put("prefix", fragmentId);
 
-    options.put("separator", ((TextFormatConfig)getConfig()).getFieldDelimiterAsString());
-    options.put(FileSystem.FS_DEFAULT_NAME_KEY, ((FileSystemConfig)writer.getStorageConfig()).connection);
+    options.put("separator", getConfig().getFieldDelimiterAsString());
+    options.put(FileSystem.FS_DEFAULT_NAME_KEY, ((FileSystemConfig) writer.getStorageConfig()).getConnection());
 
-    options.put("extension", ((TextFormatConfig)getConfig()).getExtensions().get(0));
+    options.put("extension", getConfig().getExtensions().get(0));
 
     RecordWriter recordWriter = new DrillTextRecordWriter(context.getAllocator(), writer.getStorageStrategy());
     recordWriter.init(options);
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java
index 94760da..7d5959b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java
@@ -18,6 +18,7 @@
 package org.apache.drill.exec.store.parquet;
 
 import java.io.IOException;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -65,7 +66,6 @@ import org.apache.parquet.hadoop.ParquetFileWriter;
 
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
 
 public class ParquetFormatPlugin implements FormatPlugin{
   private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MockStorageEngine.class);
@@ -126,12 +126,12 @@ public class ParquetFormatPlugin implements FormatPlugin{
   }
 
   @Override
-  public AbstractWriter getWriter(PhysicalOperator child, String location, List<String>
partitionColumns) throws IOException {
+  public AbstractWriter getWriter(PhysicalOperator child, String location, List<String>
partitionColumns) {
     return new ParquetWriter(child, location, partitionColumns, this);
   }
 
   public RecordWriter getRecordWriter(FragmentContext context, ParquetWriter writer) throws
IOException, OutOfMemoryException {
-    Map<String, String> options = Maps.newHashMap();
+    Map<String, String> options = new HashMap<>();
 
     options.put("location", writer.getLocation());
 
@@ -139,7 +139,7 @@ public class ParquetFormatPlugin implements FormatPlugin{
     String fragmentId = String.format("%d_%d", handle.getMajorFragmentId(), handle.getMinorFragmentId());
     options.put("prefix", fragmentId);
 
-    options.put(FileSystem.FS_DEFAULT_NAME_KEY, ((FileSystemConfig)writer.getStorageConfig()).connection);
+    options.put(FileSystem.FS_DEFAULT_NAME_KEY, ((FileSystemConfig) writer.getStorageConfig()).getConnection());
 
     options.put(ExecConstants.PARQUET_BLOCK_SIZE, context.getOptions().getOption(ExecConstants.PARQUET_BLOCK_SIZE).num_val.toString());
     options.put(ExecConstants.PARQUET_WRITER_USE_SINGLE_FS_BLOCK,
@@ -215,7 +215,7 @@ public class ParquetFormatPlugin implements FormatPlugin{
 
     private final ParquetFormatConfig formatConfig;
 
-    public ParquetFormatMatcher(ParquetFormatPlugin plugin, ParquetFormatConfig formatConfig)
{
+    ParquetFormatMatcher(ParquetFormatPlugin plugin, ParquetFormatConfig formatConfig) {
       super(plugin, PATTERNS, MAGIC_STRINGS);
       this.formatConfig = formatConfig;
     }
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/StoragePluginTestUtils.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/util/StoragePluginTestUtils.java
index 4a90a4e..16836c2 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/StoragePluginTestUtils.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/StoragePluginTestUtils.java
@@ -18,11 +18,13 @@
 package org.apache.drill.exec.util;
 
 import java.io.File;
+import java.util.HashMap;
 import java.util.Map;
+import java.util.Optional;
 
 import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Maps;
 import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.common.logical.FormatPluginConfig;
 import org.apache.drill.exec.store.StoragePluginRegistry;
 import org.apache.drill.exec.store.dfs.FileSystemConfig;
 import org.apache.drill.exec.store.dfs.FileSystemPlugin;
@@ -68,21 +70,27 @@ public class StoragePluginTestUtils {
     final FileSystemPlugin plugin = (FileSystemPlugin) pluginRegistry.getPlugin(pluginName);
     final FileSystemConfig pluginConfig = (FileSystemConfig) plugin.getConfig();
 
-    Map<String, WorkspaceConfig> workspaces = Maps.newHashMap();
+    Map<String, WorkspaceConfig> newWorkspaces = new HashMap<>();
+    Optional.ofNullable(pluginConfig.getWorkspaces())
+      .ifPresent(newWorkspaces::putAll);
 
     if (schemas.length == 0) {
       schemas = new String[]{TMP_SCHEMA};
     }
 
-    for (String schema: schemas) {
-      WorkspaceConfig workspaceConfig = pluginConfig.workspaces.get(schema);
-      String inputFormat = workspaceConfig == null ? null: workspaceConfig.getDefaultInputFormat();
+    for (String schema : schemas) {
+      WorkspaceConfig workspaceConfig = newWorkspaces.get(schema);
+      String inputFormat = workspaceConfig == null ? null : workspaceConfig.getDefaultInputFormat();
       WorkspaceConfig newWorkspaceConfig = new WorkspaceConfig(tmpDirPath.getAbsolutePath(),
true, inputFormat, false);
-      workspaces.put(schema, newWorkspaceConfig);
+      newWorkspaces.put(schema, newWorkspaceConfig);
     }
 
-    pluginConfig.workspaces.putAll(workspaces);
-    pluginRegistry.createOrUpdate(pluginName, pluginConfig, true);
+    FileSystemConfig newPluginConfig = new FileSystemConfig(
+        pluginConfig.getConnection(),
+        pluginConfig.getConfig(),
+        newWorkspaces,
+        pluginConfig.getFormats());
+    pluginRegistry.createOrUpdate(pluginName, newPluginConfig, true);
   }
 
   public static void configureFormatPlugins(StoragePluginRegistry pluginRegistry) throws
ExecutionSetupException {
@@ -94,32 +102,42 @@ public class StoragePluginTestUtils {
     FileSystemPlugin fileSystemPlugin = (FileSystemPlugin) pluginRegistry.getPlugin(storagePlugin);
     FileSystemConfig fileSystemConfig = (FileSystemConfig) fileSystemPlugin.getConfig();
 
+    Map<String, FormatPluginConfig> newFormats = new HashMap<>();
+    Optional.ofNullable(fileSystemConfig.getFormats())
+      .ifPresent(newFormats::putAll);
+
     TextFormatPlugin.TextFormatConfig textConfig = new TextFormatPlugin.TextFormatConfig();
     textConfig.extensions = ImmutableList.of("txt");
     textConfig.fieldDelimiter = '\u0000';
-    fileSystemConfig.formats.put("txt", textConfig);
+    newFormats.put("txt", textConfig);
 
     TextFormatPlugin.TextFormatConfig ssvConfig = new TextFormatPlugin.TextFormatConfig();
     ssvConfig.extensions = ImmutableList.of("ssv");
     ssvConfig.fieldDelimiter = ' ';
-    fileSystemConfig.formats.put("ssv", ssvConfig);
+    newFormats.put("ssv", ssvConfig);
 
     TextFormatPlugin.TextFormatConfig psvConfig = new TextFormatPlugin.TextFormatConfig();
     psvConfig.extensions = ImmutableList.of("tbl");
     psvConfig.fieldDelimiter = '|';
-    fileSystemConfig.formats.put("psv", psvConfig);
+    newFormats.put("psv", psvConfig);
 
     SequenceFileFormatConfig seqConfig = new SequenceFileFormatConfig();
     seqConfig.extensions = ImmutableList.of("seq");
-    fileSystemConfig.formats.put("sequencefile", seqConfig);
+    newFormats.put("sequencefile", seqConfig);
 
     TextFormatPlugin.TextFormatConfig csvhtestConfig = new TextFormatPlugin.TextFormatConfig();
     csvhtestConfig.extensions = ImmutableList.of("csvh-test");
     csvhtestConfig.fieldDelimiter = ',';
     csvhtestConfig.extractHeader = true;
     csvhtestConfig.skipFirstLine = true;
-    fileSystemConfig.formats.put("csvh-test", csvhtestConfig);
+    newFormats.put("csvh-test", csvhtestConfig);
+
+    FileSystemConfig newFileSystemConfig = new FileSystemConfig(
+        fileSystemConfig.getConnection(),
+        fileSystemConfig.getConfig(),
+        fileSystemConfig.getWorkspaces(),
+        newFormats);
 
-    pluginRegistry.createOrUpdate(storagePlugin, fileSystemConfig, true);
+    pluginRegistry.createOrUpdate(storagePlugin, newFileSystemConfig, true);
   }
 }
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/BaseTestImpersonation.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/BaseTestImpersonation.java
index de2ddc1..6a4452e 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/BaseTestImpersonation.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/BaseTestImpersonation.java
@@ -19,7 +19,6 @@ package org.apache.drill.exec.impersonation;
 
 import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
-import com.google.common.collect.ImmutableMap;
 import org.apache.commons.io.FileUtils;
 import org.apache.drill.PlanTestBase;
 import org.apache.drill.common.config.DrillConfig;
@@ -122,22 +121,16 @@ public class BaseTestImpersonation extends PlanTestBase {
     updateTestCluster(1, DrillConfig.create(props));
   }
 
-  protected static void addMiniDfsBasedStorage(final Map<String, WorkspaceConfig> workspaces)
-      throws Exception {
+  protected static void addMiniDfsBasedStorage(final Map<String, WorkspaceConfig> workspaces)
throws Exception {
     // Create a HDFS based storage plugin based on local storage plugin and add it to plugin
registry (connection string
     // for mini dfs is varies for each run).
-    final StoragePluginRegistry pluginRegistry = getDrillbitContext().getStorage();
-    final FileSystemConfig lfsPluginConfig = (FileSystemConfig) pluginRegistry.getPlugin("dfs").getConfig();
+    StoragePluginRegistry pluginRegistry = getDrillbitContext().getStorage();
+    FileSystemConfig lfsPluginConfig = (FileSystemConfig) pluginRegistry.getPlugin("dfs").getConfig();
 
-    final FileSystemConfig miniDfsPluginConfig = new FileSystemConfig();
-    miniDfsPluginConfig.connection = dfsConf.get(FileSystem.FS_DEFAULT_NAME_KEY);
-
-    createAndAddWorkspace("tmp", "/tmp", (short)0777, processUser, processUser, workspaces);
-
-    miniDfsPluginConfig.workspaces = workspaces;
-    miniDfsPluginConfig.formats = ImmutableMap.copyOf(lfsPluginConfig.formats);
-    miniDfsPluginConfig.setEnabled(true);
+    String connection = dfsConf.get(FileSystem.FS_DEFAULT_NAME_KEY);
+    createAndAddWorkspace("tmp", "/tmp", (short) 0777, processUser, processUser, workspaces);
 
+    FileSystemConfig miniDfsPluginConfig = new FileSystemConfig(connection, null, workspaces,
lfsPluginConfig.getFormats());
     pluginRegistry.createOrUpdate(MINIDFS_STORAGE_PLUGIN_NAME, miniDfsPluginConfig, true);
   }
 
@@ -150,7 +143,7 @@ public class BaseTestImpersonation extends PlanTestBase {
     workspaces.put(name, ws);
   }
 
-  protected static void stopMiniDfsCluster() throws Exception {
+  protected static void stopMiniDfsCluster() {
     if (dfsCluster != null) {
       dfsCluster.shutdown();
       dfsCluster = null;
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestCTTAS.java b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestCTTAS.java
index f2c0c82..ebf2cdd 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestCTTAS.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestCTTAS.java
@@ -32,7 +32,10 @@ import org.junit.rules.ExpectedException;
 
 import java.io.File;
 import java.nio.file.Paths;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
+import java.util.Optional;
 
 import static org.apache.drill.exec.util.StoragePluginTestUtils.DFS_PLUGIN_NAME;
 import static org.apache.drill.exec.util.StoragePluginTestUtils.DFS_TMP_SCHEMA;
@@ -52,8 +55,18 @@ public class TestCTTAS extends BaseTestQuery {
     File tmp2 = dirTestWatcher.makeSubDir(Paths.get("tmp2"));
     StoragePluginRegistry pluginRegistry = getDrillbitContext().getStorage();
     FileSystemConfig pluginConfig = (FileSystemConfig) pluginRegistry.getPlugin(DFS_PLUGIN_NAME).getConfig();
-    pluginConfig.workspaces.put(temp2_wk, new WorkspaceConfig(tmp2.getAbsolutePath(), true,
null, false));
-    pluginRegistry.createOrUpdate(DFS_PLUGIN_NAME, pluginConfig, true);
+
+    Map<String, WorkspaceConfig> newWorkspaces = new HashMap<>();
+    Optional.ofNullable(pluginConfig.getWorkspaces())
+      .ifPresent(newWorkspaces::putAll);
+    newWorkspaces.put(temp2_wk, new WorkspaceConfig(tmp2.getAbsolutePath(), true, null, false));
+
+    FileSystemConfig newPluginConfig = new FileSystemConfig(
+        pluginConfig.getConnection(),
+        pluginConfig.getConfig(),
+        newWorkspaces,
+        pluginConfig.getFormats());
+    pluginRegistry.createOrUpdate(DFS_PLUGIN_NAME, newPluginConfig, true);
   }
 
   @Test
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestInfoSchema.java b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestInfoSchema.java
index e0ed2fb..3932d7e 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestInfoSchema.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestInfoSchema.java
@@ -420,7 +420,7 @@ public class TestInfoSchema extends BaseTestQuery {
     assertEquals("file", configMap.get("type"));
 
     final FileSystemConfig testConfig = (FileSystemConfig) bits[0].getContext().getStorage().getPlugin("dfs").getConfig();
-    final String tmpSchemaLocation = testConfig.workspaces.get("tmp").getLocation();
+    final String tmpSchemaLocation = testConfig.getWorkspaces().get("tmp").getLocation();
     assertEquals(tmpSchemaLocation, configMap.get("location"));
 
     batch.release();
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixture.java b/exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixture.java
index 9ddcdb7..096c8cd 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixture.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixture.java
@@ -30,6 +30,7 @@ import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Properties;
 
 import org.apache.drill.test.DrillTestWrapper.TestServices;
@@ -498,13 +499,24 @@ public class ClusterFixture extends BaseFixture implements AutoCloseable
{
     final FileSystemConfig pluginConfig = (FileSystemConfig) plugin.getConfig();
     final WorkspaceConfig newTmpWSConfig = new WorkspaceConfig(path, true, defaultFormat,
false);
 
-    pluginConfig.workspaces.remove(schemaName);
-    pluginConfig.workspaces.put(schemaName, newTmpWSConfig);
-    if (format != null) {
-      pluginConfig.formats.put(defaultFormat, format);
-    }
+    Map<String, WorkspaceConfig> newWorkspaces = new HashMap<>();
+    Optional.ofNullable(pluginConfig.getWorkspaces())
+      .ifPresent(newWorkspaces::putAll);
+    newWorkspaces.put(schemaName, newTmpWSConfig);
+
+    Map<String, FormatPluginConfig> newFormats = new HashMap<>(pluginConfig.getFormats());
+    Optional.ofNullable(pluginConfig.getFormats())
+      .ifPresent(newFormats::putAll);
+    Optional.ofNullable(format)
+      .ifPresent(f -> newFormats.put(defaultFormat, f));
+
+    FileSystemConfig newPluginConfig = new FileSystemConfig(
+        pluginConfig.getConnection(),
+        pluginConfig.getConfig(),
+        newWorkspaces,
+        newFormats);
 
-    pluginRegistry.createOrUpdate(pluginName, pluginConfig, true);
+    pluginRegistry.createOrUpdate(pluginName, newPluginConfig, true);
   }
 
   public static final String EXPLAIN_PLAN_TEXT = "text";


Mime
View raw message