drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From prog...@apache.org
Subject drill git commit: DRILL-5781: Fix unit test failures to use tests config even if default config is available
Date Mon, 25 Sep 2017 16:30:21 GMT
Repository: drill
Updated Branches:
  refs/heads/master 6cb626d78 -> 9620e3482


DRILL-5781: Fix unit test failures to use tests config even if default config is available

closes #942


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/9620e348
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/9620e348
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/9620e348

Branch: refs/heads/master
Commit: 9620e34829a25d767f90b272b9b57ccc9e0e36ea
Parents: 6cb626d
Author: Volodymyr Vysotskyi <vvovyk@gmail.com>
Authored: Thu Sep 7 18:01:12 2017 +0000
Committer: Paul Rogers <progers@maprtech.com>
Committed: Sun Sep 24 22:21:20 2017 -0700

----------------------------------------------------------------------
 .../org/apache/drill/hbase/HBaseTestsSuite.java |  5 +-
 .../src/test/resources/hbase-site.xml           |  4 +-
 .../apache/drill/exec/coord/zk/PathUtils.java   | 17 ++++++-
 .../org/apache/drill/exec/ZookeeperHelper.java  |  5 +-
 .../apache/drill/exec/ZookeeperTestUtil.java    | 44 +++++++++++++++++
 .../drill/exec/coord/zk/TestEphemeralStore.java |  5 +-
 .../exec/coord/zk/TestZookeeperClient.java      |  3 ++
 .../exec/server/TestDrillbitResilience.java     |  3 ++
 exec/java-exec/src/test/resources/core-site.xml | 52 ++++++++++++++++++++
 exec/java-exec/src/test/resources/login.conf    | 25 ++++++++++
 10 files changed, 155 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/9620e348/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java
b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java
index 3dd3608..1a176ca 100644
--- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java
+++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -21,6 +21,7 @@ import java.io.IOException;
 import java.lang.management.ManagementFactory;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import org.apache.drill.exec.ZookeeperTestUtil;
 import org.apache.drill.exec.util.GuavaPatcher;
 import org.apache.drill.hbase.test.Drill2130StorageHBaseHamcrestConfigurationTest;
 import org.apache.hadoop.conf.Configuration;
@@ -95,6 +96,8 @@ public class HBaseTestsSuite {
 
   @BeforeClass
   public static void initCluster() throws Exception {
+    ZookeeperTestUtil.setJaasTestConfigFile();
+
     if (initCount.get() == 0) {
       synchronized (HBaseTestsSuite.class) {
         if (initCount.get() == 0) {

http://git-wip-us.apache.org/repos/asf/drill/blob/9620e348/contrib/storage-hbase/src/test/resources/hbase-site.xml
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/test/resources/hbase-site.xml b/contrib/storage-hbase/src/test/resources/hbase-site.xml
index 54425d4..f5c0fcb 100644
--- a/contrib/storage-hbase/src/test/resources/hbase-site.xml
+++ b/contrib/storage-hbase/src/test/resources/hbase-site.xml
@@ -66,15 +66,13 @@
     Default is 10.
     </description>
   </property>
-<!--
-   <property>
+  <property>
     <name>hbase.master.info.port</name>
     <value>-1</value>
     <description>The port for the hbase master web UI
     Set to -1 if you do not want the info server to run.
     </description>
   </property>
--->
   <property>
     <name>hbase.regionserver.info.port</name>
     <value>-1</value>

http://git-wip-us.apache.org/repos/asf/drill/blob/9620e348/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/PathUtils.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/PathUtils.java b/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/PathUtils.java
index f01b989..bc452a9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/PathUtils.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/PathUtils.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -20,6 +20,8 @@ package org.apache.drill.exec.coord.zk;
 import com.google.common.base.Preconditions;
 import org.apache.parquet.Strings;
 
+import java.net.URL;
+
 /**
  * A convenience class used to expedite zookeeper paths manipulations.
  */
@@ -70,4 +72,17 @@ public final class PathUtils {
     return builder.toString();
   }
 
+  /**
+   * Creates and returns path with the protocol at the beginning from specified {@code url}.
+   *
+   * @param url the source of path and protocol
+   * @return string with protocol and path divided by colon
+   */
+  public static String getPathWithProtocol(URL url) {
+    if (url.getProtocol() != null) {
+      return url.getProtocol() + ":" + url.getPath();
+    } else {
+      return url.getPath();
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/9620e348/exec/java-exec/src/test/java/org/apache/drill/exec/ZookeeperHelper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/ZookeeperHelper.java b/exec/java-exec/src/test/java/org/apache/drill/exec/ZookeeperHelper.java
index 8a07fd2..5c4f83f 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/ZookeeperHelper.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/ZookeeperHelper.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -25,7 +25,6 @@ import java.lang.management.ManagementFactory;
 import java.util.Properties;
 
 import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.util.FileUtils;
 import org.apache.drill.exec.util.MiniZooKeeperCluster;
 
 /**
@@ -94,6 +93,8 @@ public class ZookeeperHelper {
     }
 
     try {
+      ZookeeperTestUtil.setZookeeperSaslTestConfigProps();
+
       zkCluster = new MiniZooKeeperCluster();
       zkCluster.setDefaultClientPort(MiniZooKeeperCluster.DEFAULT_PORT);
 

http://git-wip-us.apache.org/repos/asf/drill/blob/9620e348/exec/java-exec/src/test/java/org/apache/drill/exec/ZookeeperTestUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/ZookeeperTestUtil.java b/exec/java-exec/src/test/java/org/apache/drill/exec/ZookeeperTestUtil.java
new file mode 100644
index 0000000..20d14de
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/ZookeeperTestUtil.java
@@ -0,0 +1,44 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to you under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.drill.exec;
+
+import org.apache.drill.exec.coord.zk.PathUtils;
+import org.apache.zookeeper.client.ZooKeeperSaslClient;
+import org.apache.zookeeper.server.ZooKeeperSaslServer;
+
+import static org.apache.zookeeper.Environment.JAAS_CONF_KEY;
+
+public class ZookeeperTestUtil {
+
+  private static final String LOGIN_CONF_RESOURCE_PATHNAME = "login.conf";
+
+  /**
+   * Sets zookeeper server and client SASL test config properties.
+   */
+  public static void setZookeeperSaslTestConfigProps() {
+    System.setProperty(ZooKeeperSaslServer.LOGIN_CONTEXT_NAME_KEY, "DrillTestServerForUnitTests");
+    System.setProperty(ZooKeeperSaslClient.LOGIN_CONTEXT_NAME_KEY, "DrillTestClientForUnitTests");
+  }
+
+  /**
+   * Specifies JAAS configuration file.
+   */
+  public static void setJaasTestConfigFile() {
+    String configPath = PathUtils.getPathWithProtocol(ClassLoader.getSystemResource(LOGIN_CONF_RESOURCE_PATHNAME));
+    System.setProperty(JAAS_CONF_KEY, configPath);
+  }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/9620e348/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestEphemeralStore.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestEphemeralStore.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestEphemeralStore.java
index 021a0b7..3e94589 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestEphemeralStore.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestEphemeralStore.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -27,6 +27,7 @@ import org.apache.curator.framework.recipes.cache.PathChildrenCache;
 import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener;
 import org.apache.curator.retry.RetryNTimes;
 import org.apache.curator.test.TestingServer;
+import org.apache.drill.exec.ZookeeperTestUtil;
 import org.apache.drill.exec.coord.store.TransientStoreConfig;
 import org.apache.drill.exec.serialization.InstanceSerializer;
 import org.junit.After;
@@ -61,6 +62,8 @@ public class TestEphemeralStore {
 
   @Before
   public void setUp() throws Exception {
+    ZookeeperTestUtil.setZookeeperSaslTestConfigProps();
+
     server = new TestingServer();
     final RetryPolicy policy = new RetryNTimes(2, 1000);
     curator = CuratorFrameworkFactory.newClient(server.getConnectString(), policy);

http://git-wip-us.apache.org/repos/asf/drill/blob/9620e348/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestZookeeperClient.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestZookeeperClient.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestZookeeperClient.java
index 88f1fcb..a43cee2 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestZookeeperClient.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestZookeeperClient.java
@@ -31,6 +31,7 @@ import org.apache.curator.retry.RetryNTimes;
 import org.apache.curator.test.TestingServer;
 import org.apache.drill.common.collections.ImmutableEntry;
 import org.apache.drill.common.exceptions.DrillRuntimeException;
+import org.apache.drill.exec.ZookeeperTestUtil;
 import org.apache.drill.exec.exception.VersionMismatchException;
 import org.apache.drill.exec.store.sys.store.DataChangeVersion;
 import org.apache.zookeeper.CreateMode;
@@ -71,6 +72,8 @@ public class TestZookeeperClient {
 
   @Before
   public void setUp() throws Exception {
+    ZookeeperTestUtil.setZookeeperSaslTestConfigProps();
+
     server = new TestingServer();
     final RetryPolicy policy = new RetryNTimes(1, 1000);
     curator = CuratorFrameworkFactory.newClient(server.getConnectString(), policy);

http://git-wip-us.apache.org/repos/asf/drill/blob/9620e348/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java
index 52f5fb9..a823820 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java
@@ -42,6 +42,7 @@ import org.apache.drill.common.types.TypeProtos.MinorType;
 import org.apache.drill.common.util.RepeatTestRule.Repeat;
 import org.apache.drill.exec.ExecConstants;
 import org.apache.drill.exec.ZookeeperHelper;
+import org.apache.drill.exec.ZookeeperTestUtil;
 import org.apache.drill.exec.client.DrillClient;
 import org.apache.drill.exec.exception.DrillbitStartupException;
 import org.apache.drill.exec.exception.SchemaChangeException;
@@ -185,6 +186,8 @@ public class TestDrillbitResilience extends DrillTest {
     // turn off the HTTP server to avoid port conflicts between the drill bits
     System.setProperty(ExecConstants.HTTP_ENABLE, "false");
 
+    ZookeeperTestUtil.setJaasTestConfigFile();
+
     // turn on error for failure in cancelled fragments
     zkHelper = new ZookeeperHelper(true, true);
     zkHelper.startZookeeper(1);

http://git-wip-us.apache.org/repos/asf/drill/blob/9620e348/exec/java-exec/src/test/resources/core-site.xml
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/core-site.xml b/exec/java-exec/src/test/resources/core-site.xml
new file mode 100644
index 0000000..f56d618
--- /dev/null
+++ b/exec/java-exec/src/test/resources/core-site.xml
@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<configuration>
+  <property>
+    <name>fs.hdfs.impl</name>
+    <value>org.apache.hadoop.hdfs.DistributedFileSystem</value>
+  </property>
+  <property>
+    <name>fs.AbstractFileSystem.hdfs.impl</name>
+    <value>org.apache.hadoop.hdfs.DistributedFileSystem</value>
+    <description>The FileSystem for hdfs: uris.</description>
+  </property>
+  <property>
+    <name>dfs.namenode.edits.dir</name>
+    <value>${dfs.namenode.name.dir}</value>
+    <description>Determines where on the local filesystem the DFS name node
+      should store the transaction (edits) file. If this is a comma-delimited list
+      of directories then the transaction file is replicated in all of the
+      directories, for redundancy. Default value is same as dfs.namenode.name.dir
+    </description>
+  </property>
+  <property>
+    <name>fs.defaultFS</name>
+    <value>file:///</value>
+    <description>The name of the default file system.  A URI whose
+      scheme and authority determine the FileSystem implementation.  The
+      uri's scheme determines the config property (fs.SCHEME.impl) naming
+      the FileSystem implementation class.  The uri's authority is used to
+      determine the host, port, etc. for a filesystem.</description>
+  </property>
+  <property>
+    <name>datanucleus.autoCreateTables</name>
+    <value>true</value>
+  </property>
+</configuration>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/drill/blob/9620e348/exec/java-exec/src/test/resources/login.conf
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/login.conf b/exec/java-exec/src/test/resources/login.conf
new file mode 100644
index 0000000..0916120
--- /dev/null
+++ b/exec/java-exec/src/test/resources/login.conf
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * simple login, just get OS creds
+ */
+hadoop_simple {
+  org.apache.hadoop.security.login.GenericOSLoginModule required;
+  org.apache.hadoop.security.login.HadoopLoginModule required;
+};
\ No newline at end of file


Mime
View raw message