drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ar...@apache.org
Subject [01/12] drill git commit: DRILL-1170: YARN integration for Drill
Date Sun, 04 Mar 2018 17:13:41 GMT
Repository: drill
Updated Branches:
  refs/heads/master cf2478f7a -> 36159e228


http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/test/java/org/apache/drill/yarn/scripts/TestScripts.java
----------------------------------------------------------------------
diff --git a/drill-yarn/src/test/java/org/apache/drill/yarn/scripts/TestScripts.java b/drill-yarn/src/test/java/org/apache/drill/yarn/scripts/TestScripts.java
new file mode 100644
index 0000000..e8f30a6
--- /dev/null
+++ b/drill-yarn/src/test/java/org/apache/drill/yarn/scripts/TestScripts.java
@@ -0,0 +1,1288 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.yarn.scripts;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.drill.yarn.scripts.ScriptUtils.DrillbitRun;
+import org.apache.drill.yarn.scripts.ScriptUtils.RunResult;
+import org.apache.drill.yarn.scripts.ScriptUtils.ScriptRunner;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+
+/**
+ * Unit tests to test the many ways that the Drill shell scripts can run.
+ * Since it would be difficult to test options using the actual Drillbit, the
+ * scripts make use of a special test fixture in runbit: the ability to pass
+ * a "wrapper" script to run in place of the Drillit. That script probes stderr,
+ * stdout and log files, and writes its arguments (which is the Drillbit launch
+ * command) to a file. As a result, we can capture this output and analyze it
+ * to ensure we are passing the right arguments to the Drillbit, and that output
+ * is going to the right destinations.
+ */
+
+// Turned of by default: works only in a developer setup
+@Ignore
+public class TestScripts {
+  static ScriptUtils context;
+
+  @BeforeClass
+  public static void initialSetup() throws IOException {
+    context = ScriptUtils.instance();
+    context.initialSetup();
+  }
+
+  /**
+   * Test the simplest case: use the $DRILL_HOME/conf directory and default log
+   * location. Non-existent drill-env.sh and drill-config.sh files. Everything
+   * is at its Drill-provided defaults. Then, try overriding each user-settable
+   * environment variable in the environment (which simulates what YARN might
+   * do.)
+   */
+
+  @Test
+  public void testStockCombined() throws IOException {
+    context.createMockDistrib();
+    File siteDir = new File(context.testDrillHome, "conf");
+    context.createMockConf(siteDir);
+
+    // No drill-env.sh, no distrib-env.sh
+
+    {
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN)
+          .run();
+      assertEquals(0, result.returnCode);
+      result.validateJava();
+      result.validateStockArgs();
+      result.validateClassPath(ScriptUtils.stdCp);
+      result.validateStdOut();
+      result.validateStdErr();
+      result.validateDrillLog();
+    }
+
+    // As above, but pass an argument.
+
+    {
+      String propArg = "-Dproperty=value";
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN)
+          .withArg(propArg).run();
+      assertEquals(0, result.returnCode);
+      result.validateStdOut();
+      result.validateArg(propArg);
+    }
+
+    // Custom Java opts to achieve the same result
+
+    {
+      String propArg = "-Dproperty=value";
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN)
+          .addEnv("DRILL_JAVA_OPTS", propArg).run();
+      assertEquals(0, result.returnCode);
+      result.validateStockArgs(); // Should not lose standard JVM args
+      result.validateStdOut();
+      result.validateArg(propArg);
+    }
+
+    // Custom Drillbit Java Opts to achieve the same result
+
+    {
+      String propArg = "-Dproperty2=value2";
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN)
+          .addEnv("DRILLBIT_JAVA_OPTS", propArg).run();
+      assertEquals(0, result.returnCode);
+      result.validateStockArgs(); // Should not lose standard JVM args
+      result.validateStdOut();
+      result.validateArg(propArg);
+    }
+
+    // Both sets of options
+
+    {
+      String propArg = "-Dproperty=value";
+      String propArg2 = "-Dproperty2=value2";
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN)
+          .addEnv("DRILL_JAVA_OPTS", propArg)
+          .addEnv("DRILLBIT_JAVA_OPTS", propArg2).run();
+      assertEquals(0, result.returnCode);
+      result.validateArgs(new String[] { propArg, propArg2 });
+    }
+
+    // Custom heap memory
+
+    {
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN)
+          .addEnv("DRILL_HEAP", "5G").run();
+      result.validateArgs(new String[] { "-Xms5G", "-Xmx5G" });
+    }
+
+    // Custom direct memory
+
+    {
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN)
+          .addEnv("DRILL_MAX_DIRECT_MEMORY", "7G").run();
+      result.validateArg("-XX:MaxDirectMemorySize=7G");
+    }
+
+    // Enable GC logging
+
+    {
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN)
+          .addEnv("SERVER_LOG_GC", "1").run();
+      String logTail = context.testLogDir.getName() + "/drillbit.gc";
+      result.validateArgRegex("-Xloggc:.*/" + logTail);
+    }
+
+    // Max Perm Size
+
+    {
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN)
+          .addEnv("DRILLBIT_MAX_PERM", "600M").run();
+      result.validateArg("-XX:MaxPermSize=600M");
+    }
+
+    // Code cache size
+
+    {
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN)
+          .addEnv("DRILLBIT_CODE_CACHE_SIZE", "2G").run();
+      result.validateArg("-XX:ReservedCodeCacheSize=2G");
+    }
+  }
+
+  /**
+   * Use the "stock" setup, but add each custom bit of the class path to ensure
+   * it is passed to the Drillbit.
+   *
+   * @throws IOException
+   */
+
+  @Test
+  public void testClassPath() throws IOException {
+    context.createMockDistrib();
+    File siteDir = new File(context.testDrillHome, "conf");
+    context.createMockConf(siteDir);
+
+    File extrasDir = context.createDir(new File(context.testDir, "extras"));
+    File hadoopJar = context.makeDummyJar(extrasDir, "hadoop");
+    File hbaseJar = context.makeDummyJar(extrasDir, "hbase");
+    File prefixJar = context.makeDummyJar(extrasDir, "prefix");
+    File cpJar = context.makeDummyJar(extrasDir, "cp");
+    File extnJar = context.makeDummyJar(extrasDir, "extn");
+    File toolsJar = context.makeDummyJar(extrasDir, "tools");
+
+    {
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN)
+          .addEnv("DRILL_CLASSPATH_PREFIX", prefixJar.getAbsolutePath()).run();
+      result.validateClassPath(prefixJar.getAbsolutePath());
+    }
+
+    {
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN)
+          .addEnv("DRILL_TOOL_CP", toolsJar.getAbsolutePath()).run();
+      result.validateClassPath(toolsJar.getAbsolutePath());
+    }
+
+    {
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN)
+          .addEnv("HADOOP_CLASSPATH", hadoopJar.getAbsolutePath()).run();
+      result.validateClassPath(hadoopJar.getAbsolutePath());
+    }
+
+    {
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN)
+          .addEnv("HBASE_CLASSPATH", hbaseJar.getAbsolutePath()).run();
+      result.validateClassPath(hbaseJar.getAbsolutePath());
+    }
+
+    {
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN)
+          .addEnv("EXTN_CLASSPATH", extnJar.getAbsolutePath()).run();
+      result.validateClassPath(extnJar.getAbsolutePath());
+    }
+
+    {
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN)
+          .addEnv("DRILL_CLASSPATH", cpJar.getAbsolutePath()).run();
+      result.validateClassPath(cpJar.getAbsolutePath());
+    }
+
+    // Site jars not on path if not created
+
+    File siteJars = new File(siteDir, "jars");
+    {
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN).run();
+      assertFalse(result.classPathContains(siteJars.getAbsolutePath()));
+    }
+
+    // Site/jars on path if exists
+
+    context.createDir(siteJars);
+    context.makeDummyJar(siteJars, "site");
+
+    {
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN).run();
+      result.validateClassPath(siteJars.getAbsolutePath() + "/*");
+    }
+  }
+
+  /**
+   * Create a custom log folder location.
+   */
+
+  @Test
+  public void testLogDir() throws IOException {
+    context.createMockDistrib();
+    File siteDir = new File(context.testDrillHome, "conf");
+    context.createMockConf(siteDir);
+    File logsDir = context.createDir(new File(context.testDir, "logs"));
+    context.removeDir(new File(context.testDrillHome, "log"));
+
+    {
+      String logPath = logsDir.getAbsolutePath();
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN)
+          .addEnv("DRILL_LOG_DIR", logPath).withLogDir(logsDir).run();
+      assertEquals(0, result.returnCode);
+      result.validateArgs(
+          new String[] { "-Dlog.path=" + logPath + "/drillbit.log",
+              "-Dlog.query.path=" + logPath + "/drillbit_queries.json", });
+      result.validateStdOut();
+      result.validateStdErr();
+      result.validateDrillLog();
+    }
+
+  }
+
+  /**
+   * Create a custom Java lib path. This uses the new DRILL_JAVA_LIB_PATH
+   * variable.
+   */
+
+  @Test
+  public void testLibPath() throws IOException {
+    context.createMockDistrib();
+    File siteDir = new File(context.testDrillHome, "conf");
+    context.createMockConf(siteDir);
+    File logsDir = context.createDir(new File(context.testDir, "logs"));
+    context.removeDir(new File(context.testDrillHome, "log"));
+
+    {
+      String logPath = logsDir.getAbsolutePath();
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN)
+          .addEnv("DRILL_LOG_DIR", logPath).withLogDir(logsDir).run();
+      assertEquals(0, result.returnCode);
+      result.validateArgs(
+          new String[] { "-Dlog.path=" + logPath + "/drillbit.log",
+              "-Dlog.query.path=" + logPath + "/drillbit_queries.json", });
+      result.validateStdOut();
+      result.validateStdErr();
+      result.validateDrillLog();
+    }
+
+  }
+
+  /**
+   * Try setting custom environment variable values in drill-env.sh in the
+   * $DRILL_HOME/conf location.
+   */
+
+  @Test
+  public void testDrillEnv() throws IOException {
+    doEnvFileTest("drill-env.sh");
+  }
+
+  /**
+   * Repeat the above test using distrib-env.sh in the $DRILL_HOME/conf
+   * location.
+   */
+
+  @Test
+  public void testDistribEnv() throws IOException {
+    doEnvFileTest("distrib-env.sh");
+  }
+
+  /**
+   * Implementation of the drill-env.sh and distrib-env.sh tests.
+   */
+
+  private void doEnvFileTest(String fileName) throws IOException {
+    context.createMockDistrib();
+    File siteDir = new File(context.testDrillHome, "conf");
+    context.createMockConf(siteDir);
+
+    /**
+     * Set all properties in the env file.
+     */
+
+    Map<String, String> drillEnv = new HashMap<>();
+    String propArg = "-Dproperty=value";
+    drillEnv.put("DRILL_JAVA_OPTS", propArg);
+    drillEnv.put("DRILL_HEAP", "5G");
+    drillEnv.put("DRILL_MAX_DIRECT_MEMORY", "7G");
+    drillEnv.put("SERVER_LOG_GC", "1");
+    drillEnv.put("DRILLBIT_MAX_PERM", "600M");
+    drillEnv.put("DRILLBIT_CODE_CACHE_SIZE", "2G");
+    context.createEnvFile(new File(siteDir, fileName), drillEnv);
+
+    {
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN).run();
+      assertEquals(0, result.returnCode);
+
+      String expectedArgs[] = {
+          propArg,
+          "-Xms5G", "-Xmx5G",
+          "-XX:MaxDirectMemorySize=7G",
+          "-XX:ReservedCodeCacheSize=2G",
+          "-XX:MaxPermSize=600M"
+      };
+
+      result.validateArgs(expectedArgs);
+      String logTail = context.testLogDir.getName() + "/drillbit.gc";
+      result.validateArgRegex("-Xloggc:.*/" + logTail);
+    }
+
+    // Change some drill-env.sh options in the environment.
+    // The generated drill-env.sh should allow overrides.
+    // (The generated form is the form that customers should use.)
+
+    {
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN)
+          .addEnv("SERVER_LOG_GC", "0")
+          .addEnv("DRILL_MAX_DIRECT_MEMORY", "9G")
+          .run();
+      assertEquals(0, result.returnCode);
+      result.validateArg("-XX:MaxDirectMemorySize=9G");
+      result.validateArg("-XX:MaxPermSize=600M");
+      String logTail = context.testDrillHome.getName() + "/log/drillbit.gc";
+      assertFalse(result.containsArgRegex("-Xloggc:.*/" + logTail));
+    }
+  }
+
+  /**
+   * Test that drill-env.sh overrides distrib-env.sh, and that the environment
+   * overrides both. Assumes the basics were tested above.
+   *
+   * @throws IOException
+   */
+
+  @Test
+  public void testDrillAndDistribEnv() throws IOException {
+    context.createMockDistrib();
+    File siteDir = new File(context.testDrillHome, "conf");
+    context.createMockConf(siteDir);
+
+    Map<String, String> distribEnv = new HashMap<>();
+    distribEnv.put("DRILL_HEAP", "5G");
+    distribEnv.put("DRILL_MAX_DIRECT_MEMORY", "7G");
+    distribEnv.put("DRILLBIT_MAX_PERM", "600M");
+    context.createEnvFile(new File(siteDir, "distrib-env.sh"), distribEnv);
+
+    Map<String, String> drillEnv = new HashMap<>();
+    drillEnv.put("DRILL_HEAP", "6G");
+    drillEnv.put("DRILL_MAX_DIRECT_MEMORY", "9G");
+    context.createEnvFile(new File(siteDir, "drill-env.sh"), drillEnv);
+
+    {
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN).run();
+      assertEquals(0, result.returnCode);
+      String expectedArgs[] = {
+          "-Xms6G", "-Xmx6G",
+          "-XX:MaxDirectMemorySize=9G",
+          "-XX:MaxPermSize=600M",
+          "-XX:ReservedCodeCacheSize=1G" // Default
+      };
+
+      result.validateArgs(expectedArgs);
+    }
+
+    {
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN)
+          .addEnv("DRILL_MAX_DIRECT_MEMORY", "5G").run();
+      assertEquals(0, result.returnCode);
+      String expectedArgs[] = {
+          "-Xms6G", "-Xmx6G",
+          "-XX:MaxDirectMemorySize=5G",
+          "-XX:MaxPermSize=600M",
+          "-XX:ReservedCodeCacheSize=1G" // Default
+      };
+
+      result.validateArgs(expectedArgs);
+    }
+  }
+
+  @Test
+  public void testBadSiteDir() throws IOException {
+    context.createMockDistrib();
+    File siteDir = new File(context.testDrillHome, "conf");
+    context.removeDir(siteDir);
+
+    // Directory does not exist.
+
+    {
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN)
+          .withSite(siteDir).run();
+      assertEquals(1, result.returnCode);
+      assertTrue(result.stderr.contains("Config dir does not exist"));
+    }
+
+    // Not a directory
+
+    context.writeFile(siteDir, "dummy");
+    {
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN)
+          .withSite(siteDir).run();
+      assertEquals(1, result.returnCode);
+      assertTrue(result.stderr.contains("Config dir does not exist"));
+    }
+
+    // Directory exists, but drill-override.conf does not
+
+    siteDir.delete();
+    context.createDir(siteDir);
+    {
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN)
+          .withSite(siteDir).run();
+      assertEquals(1, result.returnCode);
+      assertTrue(result.stderr.contains("Drill config file missing"));
+    }
+  }
+
+  /**
+   * Move configuration to a site folder out of $DRILL_HOME/conf. The site
+   * folder can contain code (which is why we call it "site" and not "config".)
+   * The site directory can be passed to the Drillbit in several ways.
+   *
+   * @throws IOException
+   */
+
+  @Test
+  public void testSiteDir() throws IOException {
+    context.createMockDistrib();
+    File confDir = new File(context.testDrillHome, "conf");
+    context.createDir(confDir);
+    File siteDir = new File(context.testDir, "site");
+    context.createMockConf(siteDir);
+
+    // Dummy drill-env.sh to simulate the shipped "example" file.
+
+    context.writeFile(new File(confDir, "drill-env.sh"),
+        "#!/usr/bin/env bash\n" + "# Example file");
+    File siteJars = new File(siteDir, "jars");
+
+    Map<String, String> distribEnv = new HashMap<>();
+    distribEnv.put("DRILL_HEAP", "5G");
+    distribEnv.put("DRILL_MAX_DIRECT_MEMORY", "7G");
+    distribEnv.put("DRILLBIT_MAX_PERM", "600M");
+    context.createEnvFile(new File(confDir, "distrib-env.sh"), distribEnv);
+
+    Map<String, String> drillEnv = new HashMap<>();
+    drillEnv.put("DRILL_HEAP", "6G");
+    drillEnv.put("DRILL_MAX_DIRECT_MEMORY", "9G");
+    context.createEnvFile(new File(siteDir, "drill-env.sh"), drillEnv);
+
+    String expectedArgs[] = {
+        "-Xms6G", "-Xmx6G",
+        "-XX:MaxDirectMemorySize=9G",
+        "-XX:MaxPermSize=600M",
+        "-XX:ReservedCodeCacheSize=1G" // Default
+    };
+
+    // Site set using argument
+
+    {
+      // Use --config explicitly
+
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN)
+          .withArg("--config")
+          .withArg(siteDir.getAbsolutePath())
+          .run();
+      assertEquals(0, result.returnCode);
+      result.validateArgs(expectedArgs);
+      result.validateClassPath(siteDir.getAbsolutePath());
+    }
+
+    {
+      RunResult result = new DrillbitRun()
+          .withArg("--config")
+          .withArg(siteDir.getAbsolutePath())
+          .withArg(DrillbitRun.DRILLBIT_RUN)
+          .run();
+      assertEquals(0, result.returnCode);
+      result.validateArgs(expectedArgs);
+    }
+
+    {
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN)
+          .withSite(siteDir)
+          .run();
+      assertEquals(0, result.returnCode);
+      result.validateArgs(expectedArgs);
+    }
+
+    // Site argument and argument to Drillbit
+
+    {
+      String propArg = "-Dproperty=value";
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN)
+          .withSite(siteDir)
+          .withArg(propArg)
+          .run( );
+      assertEquals(0, result.returnCode);
+      result.validateArgs(expectedArgs);
+      result.validateArg(propArg);
+    }
+
+    // Set as an environment variable
+
+    {
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN)
+          .addEnv("DRILL_CONF_DIR",siteDir.getAbsolutePath())
+          .run();
+      assertEquals(0, result.returnCode);
+      result.validateArgs(expectedArgs);
+    }
+
+    // Site jars not on path if not created
+
+    {
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN)
+          .withSite(siteDir)
+          .run();
+      assertFalse(result.classPathContains(siteJars.getAbsolutePath()));
+    }
+
+    // Site/jars on path if exists
+
+    context.createDir(siteJars);
+    context.makeDummyJar(siteJars, "site");
+
+    {
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN)
+          .withSite(siteDir)
+          .run();
+      assertTrue(result.classPathContains(siteJars.getAbsolutePath() + "/*"));
+    }
+  }
+
+  /**
+   * Test the Java library path. Three sources:
+   * <ol>
+   * <li>DRILL_JAVA_LIB_PATH Set in drill-env.sh</li>
+   * <li>DOY_JAVA_LIB_PATH passed in from an env. var.</li>
+   * <li>$DRILL_SITE/lib, if it exists.</li>
+   * </ol>
+   *
+   * @throws IOException
+   */
+
+  @Test
+  public void testJavaLibDir() throws IOException {
+    context.createMockDistrib();
+    File siteDir = new File(context.testDrillHome, "conf");
+    context.createMockConf(siteDir);
+
+    // Stock run: no lib dir.
+
+    String prefix = "-Djava.library.path=";
+    {
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN)
+          .run();
+      assertFalse(result.containsArgRegex(prefix + ".*"));
+      assertNull(result.libPath);
+    }
+
+    // Old-style argument in DRILL_JAVA_OPTS
+
+    {
+      Map<String, String> env = new HashMap<>();
+      env.put("DRILL_JAVA_OPTS", prefix + "/foo/bar:/foo/mumble");
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN)
+          .withEnvironment(env)
+          .run();
+      assertTrue(result.containsArgRegex(prefix + ".*"));
+      assertNotNull(result.libPath);
+      assertEquals(2, result.libPath.length);
+      assertEquals("/foo/bar", result.libPath[0]);
+      assertEquals("/foo/mumble", result.libPath[1]);
+    }
+
+    // New-style argument in DRILL_JAVA_LIB_PATH
+
+    {
+      Map<String, String> env = new HashMap<>();
+      env.put("DRILL_JAVA_LIB_PATH", "/foo/bar:/foo/mumble");
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN)
+          .withEnvironment(env)
+          .run();
+      assertTrue(result.containsArgRegex(prefix + ".*"));
+      assertNotNull(result.libPath);
+      assertEquals(2, result.libPath.length);
+      assertEquals("/foo/bar", result.libPath[0]);
+      assertEquals("/foo/mumble", result.libPath[1]);
+    }
+
+    // YARN argument in DOY_JAVA_LIB_PATH
+
+    {
+      Map<String, String> env = new HashMap<>();
+      env.put("DOY_JAVA_LIB_PATH", "/foo/bar:/foo/mumble");
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN)
+          .withEnvironment(env)
+          .run();
+      assertTrue(result.containsArgRegex(prefix + ".*"));
+      assertNotNull(result.libPath);
+      assertEquals(2, result.libPath.length);
+      assertEquals("/foo/bar", result.libPath[0]);
+      assertEquals("/foo/mumble", result.libPath[1]);
+    }
+
+    // Both DRILL_JAVA_LIB_PATH and DOY_JAVA_LIB_PATH
+
+    {
+      Map<String, String> env = new HashMap<>();
+      env.put("DRILL_JAVA_LIB_PATH", "/foo/bar:/foo/mumble");
+      env.put("DOY_JAVA_LIB_PATH", "/doy/bar:/doy/mumble");
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN)
+          .withEnvironment(env)
+          .run();
+      assertTrue(result.containsArgRegex(prefix + ".*"));
+      assertNotNull(result.libPath);
+      assertEquals(4, result.libPath.length);
+      assertEquals("/doy/bar", result.libPath[0]);
+      assertEquals("/doy/mumble", result.libPath[1]);
+      assertEquals("/foo/bar", result.libPath[2]);
+      assertEquals("/foo/mumble", result.libPath[3]);
+    }
+
+    // Site directory with a lib folder
+
+    siteDir = new File(context.testDir, "site");
+    context.createMockConf(siteDir);
+    File libDir = new File(siteDir, "lib");
+    context.createDir(libDir);
+
+    {
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN)
+          .withSite(siteDir)
+          .run();
+      assertTrue(result.containsArgRegex(prefix + ".*"));
+      assertNotNull(result.libPath);
+      assertEquals(1, result.libPath.length);
+      assertEquals(libDir.getAbsolutePath(), result.libPath[0]);
+    }
+
+    // The whole enchilada: all three settings.
+
+    {
+      Map<String, String> env = new HashMap<>();
+      env.put("DRILL_JAVA_LIB_PATH", "/foo/bar:/foo/mumble");
+      env.put("DOY_JAVA_LIB_PATH", "/doy/bar:/doy/mumble");
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN)
+          .withSite(siteDir)
+          .withEnvironment(env)
+          .run();
+      assertTrue(result.containsArgRegex(prefix + ".*"));
+      assertNotNull(result.libPath);
+      assertEquals(5, result.libPath.length);
+      assertEquals(libDir.getAbsolutePath(), result.libPath[0]);
+      assertEquals("/doy/bar", result.libPath[1]);
+      assertEquals("/doy/mumble", result.libPath[2]);
+      assertEquals("/foo/bar", result.libPath[3]);
+      assertEquals("/foo/mumble", result.libPath[4]);
+    }
+  }
+
+  /**
+   * Test running a (simulated) Drillbit as a daemon with start, status, stop.
+   *
+   * @throws IOException
+   */
+
+  @Test
+  public void testStockDaemon() throws IOException {
+    context.createMockDistrib();
+    File siteDir = new File(context.testDrillHome, "conf");
+    context.createMockConf(siteDir);
+
+    // No drill-env.sh, no distrib-env.sh
+
+    File pidFile;
+    {
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_START).start();
+      assertEquals(0, result.returnCode);
+      result.validateJava();
+      result.validateStockArgs();
+      result.validateClassPath(ScriptUtils.stdCp);
+      assertTrue(result.stdout.contains("Starting drillbit, logging"));
+      assertTrue(result.log.contains("Starting drillbit on"));
+      assertTrue(result.log.contains("Drill Log Message"));
+      assertTrue(result.out.contains("Drill Stdout Message"));
+      assertTrue(result.out.contains("Stderr Message"));
+      pidFile = result.pidFile;
+    }
+
+    // Save the pid file for reuse.
+
+    assertTrue(pidFile.exists());
+    File saveDir = new File(context.testDir, "save");
+    context.createDir(saveDir);
+    File savedPidFile = new File(saveDir, pidFile.getName());
+    context.copyFile(pidFile, savedPidFile);
+
+    // Status should be running
+
+    {
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_STATUS).run( );
+      assertEquals(0, result.returnCode);
+      assertTrue(result.stdout.contains("drillbit is running"));
+    }
+
+    // Start should refuse to start a second Drillbit.
+
+    {
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_START).start();
+      assertEquals(1, result.returnCode);
+      assertTrue(
+          result.stdout.contains("drillbit is already running as process"));
+    }
+
+    // Normal start, allow normal shutdown
+
+    {
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_STOP).run();
+      assertEquals(0, result.returnCode);
+      assertTrue(result.log.contains("Terminating drillbit pid"));
+      assertTrue(result.stdout.contains("Stopping drillbit"));
+    }
+
+    // Status should report no drillbit (no pid file)
+
+    {
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_STATUS).run();
+      assertEquals(1, result.returnCode);
+      assertTrue(result.stdout.contains("drillbit is not running"));
+    }
+
+    // Stop should report no pid file
+
+    {
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_STOP).run();
+      assertEquals(1, result.returnCode);
+      assertTrue(
+          result.stdout.contains("No drillbit to stop because no pid file"));
+    }
+
+    // Get nasty. Put the pid file back. But, there is no process with that pid.
+
+    context.copyFile(savedPidFile, pidFile);
+
+    // Status should now complain.
+
+    {
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_STATUS).run();
+      assertEquals(1, result.returnCode);
+      assertTrue(result.stdout
+          .contains("file is present but drillbit is not running"));
+    }
+
+    // As should stop.
+
+    {
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_STOP).run();
+      assertEquals(1, result.returnCode);
+      assertTrue(
+          result.stdout.contains("No drillbit to stop because kill -0 of pid"));
+    }
+  }
+
+  @Test
+  public void testStockDaemonWithArg() throws IOException {
+    context.createMockDistrib();
+    File siteDir = new File(context.testDrillHome, "conf");
+    context.createMockConf(siteDir);
+
+    // As above, but pass an argument.
+
+    {
+      String propArg = "-Dproperty=value";
+      DrillbitRun runner = new DrillbitRun(DrillbitRun.DRILLBIT_START);
+      runner.withArg(propArg);
+      RunResult result = runner.start();
+      assertEquals(0, result.returnCode);
+      result.validateArg(propArg);
+    }
+
+    validateAndCloseDaemon(null);
+  }
+
+  private void validateAndCloseDaemon(File siteDir) throws IOException {
+    {
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_STATUS)
+          .withSite(siteDir)
+          .run();
+      assertEquals(0, result.returnCode);
+      assertTrue(result.stdout.contains("drillbit is running"));
+    }
+
+    {
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_STOP)
+          .withSite(siteDir)
+          .run();
+      assertEquals(0, result.returnCode);
+    }
+  }
+
+  /**
+   * The Daemon process creates a pid file. Verify that the DRILL_PID_DIR can be
+   * set to put the pid file in a custom location. The test is done with the
+   * site (conf) dir in the default location.
+   *
+   * @throws IOException
+   */
+
+  @Test
+  public void testPidDir() throws IOException {
+    context.createMockDistrib();
+    File siteDir = new File(context.testDrillHome, "conf");
+    context.createMockConf(siteDir);
+    File pidDir = context.createDir(new File(context.testDir, "pid"));
+    Map<String, String> drillEnv = new HashMap<>();
+    drillEnv.put("DRILL_PID_DIR", pidDir.getAbsolutePath());
+    context.createEnvFile(new File(siteDir, "drill-env.sh"), drillEnv);
+
+    {
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_START)
+          .withPidDir(pidDir)
+          .start();
+      assertEquals(0, result.returnCode);
+      assertTrue(result.pidFile.getParentFile().equals(pidDir));
+      assertTrue(result.pidFile.exists());
+    }
+
+    validateAndCloseDaemon(null);
+  }
+
+  /**
+   * Test a custom site directory with the Drill daemon process. The custom
+   * directory contains a drill-env.sh with a custom option. Verify that that
+   * option is picked up when starting Drill.
+   *
+   * @throws IOException
+   */
+
+  @Test
+  public void testSiteDirWithDaemon() throws IOException {
+    context.createMockDistrib();
+
+    File siteDir = new File(context.testDir, "site");
+    context.createMockConf(siteDir);
+
+    Map<String, String> drillEnv = new HashMap<>();
+    drillEnv.put("DRILL_MAX_DIRECT_MEMORY", "9G");
+    context.createEnvFile(new File(siteDir, "drill-env.sh"), drillEnv);
+
+    // Use the -site (--config) option.
+
+    {
+      DrillbitRun runner = new DrillbitRun(DrillbitRun.DRILLBIT_START);
+      runner.withSite(siteDir);
+      RunResult result = runner.start();
+      assertEquals(0, result.returnCode);
+      result.validateArg("-XX:MaxDirectMemorySize=9G");
+    }
+
+    validateAndCloseDaemon(siteDir);
+
+    // Set an env var.
+
+    {
+      DrillbitRun runner = new DrillbitRun(DrillbitRun.DRILLBIT_START);
+      runner.addEnv("DRILL_CONF_DIR", siteDir.getAbsolutePath());
+      RunResult result = runner.start();
+      assertEquals(0, result.returnCode);
+      result.validateArg("-XX:MaxDirectMemorySize=9G");
+    }
+
+    validateAndCloseDaemon(siteDir);
+  }
+
+  /**
+   * Launch the Drill daemon using a custom log file location. The config is in
+   * the default location.
+   *
+   * @throws IOException
+   */
+
+  @Test
+  public void testLogDirWithDaemon() throws IOException {
+    context.createMockDistrib();
+    File siteDir = new File(context.testDrillHome, "conf");
+    context.createMockConf(siteDir);
+    File logsDir = context.createDir(new File(context.testDir, "logs"));
+    context.removeDir(new File(context.testDrillHome, "log"));
+    Map<String, String> drillEnv = new HashMap<>();
+    drillEnv.put("DRILL_LOG_DIR", logsDir.getAbsolutePath());
+    context.createEnvFile(new File(siteDir, "drill-env.sh"), drillEnv);
+
+    {
+      DrillbitRun runner = new DrillbitRun(DrillbitRun.DRILLBIT_START);
+      runner.withLogDir(logsDir);
+      RunResult result = runner.start();
+      assertEquals(0, result.returnCode);
+      assertNotNull(result.logFile);
+      assertTrue(result.logFile.getParentFile().equals(logsDir));
+      assertTrue(result.logFile.exists());
+      assertNotNull(result.outFile);
+      assertTrue(result.outFile.getParentFile().equals(logsDir));
+      assertTrue(result.outFile.exists());
+    }
+
+    validateAndCloseDaemon(null);
+  }
+
+  /**
+   * Some distributions create symlinks to drillbit.sh in standard locations
+   * such as /usr/bin. Because drillbit.sh uses its own location to compute
+   * DRILL_HOME, it must handle symlinks. This test verifies that process.
+   *
+   * @throws IOException
+   */
+
+  @Test
+  public void testDrillbitSymlink() throws IOException {
+    context.createMockDistrib();
+    File siteDir = new File(context.testDrillHome, "conf");
+    context.createMockConf(siteDir);
+
+    File drillbitFile = new File(context.testDrillHome, "bin/drillbit.sh");
+    File linksDir = context.createDir(new File(context.testDir, "links"));
+    File link = new File(linksDir, drillbitFile.getName());
+    try {
+      Files.createSymbolicLink(link.toPath(), drillbitFile.toPath());
+    } catch (UnsupportedOperationException e) {
+      // Well. This is a system without symlinks, so we won't be testing
+      // syminks here...
+
+      return;
+    }
+
+    {
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_START).start();
+      assertEquals(0, result.returnCode);
+      assertEquals(result.pidFile.getParentFile(), context.testDrillHome);
+    }
+    validateAndCloseDaemon(null);
+  }
+
+  /**
+   * Test the restart command of drillbit.sh
+   *
+   * @throws IOException
+   */
+
+  @Test
+  public void testRestart() throws IOException {
+    context.createMockDistrib();
+    File siteDir = new File(context.testDrillHome, "conf");
+    context.createMockConf(siteDir);
+
+    int firstPid;
+    {
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_START).start();
+      assertEquals(0, result.returnCode);
+      firstPid = result.getPid();
+    }
+
+    // Make sure it is running.
+
+    {
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_STATUS)
+          .withSite(siteDir)
+          .run();
+      assertEquals(0, result.returnCode);
+      assertTrue(result.stdout.contains("drillbit is running"));
+    }
+
+    // Restart. Should get new pid.
+
+    {
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RESTART).start();
+      assertEquals(0, result.returnCode);
+      int secondPid = result.getPid();
+      assertNotEquals(firstPid, secondPid);
+    }
+
+    validateAndCloseDaemon(null);
+  }
+
+  /**
+   * Simulate a Drillbit that refuses to die. The stop script wait a while, then
+   * forces killing.
+   *
+   * @throws IOException
+   */
+
+  @Test
+  public void testForcedKill() throws IOException {
+    context.createMockDistrib();
+    File siteDir = new File(context.testDrillHome, "conf");
+    context.createMockConf(siteDir);
+
+    {
+      DrillbitRun runner = new DrillbitRun(DrillbitRun.DRILLBIT_START);
+      runner.addEnv("PRETEND_HUNG", "1");
+      RunResult result = runner.start();
+      assertEquals(0, result.returnCode);
+    }
+
+    {
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_STATUS)
+          .preserveLogs()
+          .run();
+      assertEquals(0, result.returnCode);
+      assertTrue(result.stdout.contains("drillbit is running"));
+    }
+
+    {
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_STOP)
+          .addEnv("DRILL_STOP_TIMEOUT","5")
+          .preserveLogs()
+          .run();
+      assertEquals(0, result.returnCode);
+      assertTrue(result.stdout.contains("drillbit did not complete after"));
+    }
+  }
+
+  /**
+   * Verify the basics of the sqlline script, including the env vars that can be
+   * customized. Also validate running in embedded mode (using drillbit memory
+   * and other options.)
+   *
+   * @throws IOException
+   */
+
+  @Test
+  public void testSqlline() throws IOException {
+    context.createMockDistrib();
+    File siteDir = new File(context.testDrillHome, "conf");
+    context.createMockConf(siteDir);
+
+    int stockArgCount;
+    {
+      // Out-of-the-box sqlline
+
+      RunResult result = new ScriptRunner("sqlline").run();
+      assertEquals(0, result.returnCode);
+      result.validateJava();
+      result.validateClassPath(ScriptUtils.stdCp);
+      assertTrue(result.containsArgsRegex(ScriptUtils.sqlLineArgs));
+      stockArgCount = result.echoArgs.size();
+    }
+
+    {
+      RunResult result = new ScriptRunner("sqlline")
+          .withArg("arg1")
+          .withArg("arg2")
+          .run( );
+      assertTrue(result.containsArg("arg1"));
+      assertTrue(result.containsArg("arg2"));
+    }
+    {
+      // Change drill memory and other drill-specific
+      // settings: should not affect sqlline
+
+      Map<String, String> drillEnv = new HashMap<>();
+      drillEnv.put("DRILL_JAVA_OPTS", "-Dprop=value");
+      drillEnv.put("DRILL_HEAP", "5G");
+      drillEnv.put("DRILL_MAX_DIRECT_MEMORY", "7G");
+      drillEnv.put("SERVER_LOG_GC", "1");
+      drillEnv.put("DRILLBIT_MAX_PERM", "600M");
+      drillEnv.put("DRILLBIT_CODE_CACHE_SIZE", "2G");
+      RunResult result = new ScriptRunner("sqlline")
+          .withEnvironment(drillEnv)
+          .run();
+      assertTrue(result.containsArgsRegex(ScriptUtils.sqlLineArgs));
+
+      // Nothing new should have been added
+
+      assertEquals(stockArgCount, result.echoArgs.size());
+    }
+    {
+      // Change client memory: should affect sqlline
+
+      Map<String, String> shellEnv = new HashMap<>();
+      shellEnv.put("CLIENT_GC_OPTS", "-XX:+UseG1GC");
+      shellEnv.put("SQLLINE_JAVA_OPTS", "-XX:MaxPermSize=256M");
+      RunResult result = new ScriptRunner("sqlline")
+          .withEnvironment(shellEnv)
+          .run();
+      assertTrue(result.containsArg("-XX:MaxPermSize=256M"));
+      assertTrue(result.containsArg("-XX:+UseG1GC"));
+    }
+    {
+      // Change drill memory and other drill-specific
+      // settings: then set the "magic" variable that says
+      // that Drill is embedded. The scripts should now use
+      // the Drillbit options.
+
+      Map<String, String> drillEnv = new HashMap<>();
+      drillEnv.put("DRILL_JAVA_OPTS", "-Dprop=value");
+      drillEnv.put("DRILL_HEAP", "5G");
+      drillEnv.put("DRILL_MAX_DIRECT_MEMORY", "7G");
+      drillEnv.put("SERVER_LOG_GC", "1");
+      drillEnv.put("DRILLBIT_MAX_PERM", "600M");
+      drillEnv.put("DRILLBIT_CODE_CACHE_SIZE", "2G");
+      drillEnv.put("DRILL_EMBEDDED", "1");
+      RunResult result = new ScriptRunner("sqlline")
+          .withEnvironment(drillEnv)
+          .run();
+
+      String expectedArgs[] = {
+          "-Dprop=value",
+          "-Xms5G", "-Xmx5G",
+          "-XX:MaxDirectMemorySize=7G",
+          "-XX:ReservedCodeCacheSize=2G",
+          "-XX:MaxPermSize=600M"
+      };
+
+      result.validateArgs(expectedArgs);
+      assertTrue(result.containsArg("sqlline.SqlLine"));
+    }
+  }
+
+  /**
+   * Verify that the sqlline client works with the --site option by customizing
+   * items in the site directory.
+   *
+   * @throws IOException
+   */
+
+  @Test
+  public void testSqllineSiteDir() throws IOException {
+    context.createMockDistrib();
+    File siteDir = new File(context.testDir, "site");
+    context.createMockConf(siteDir);
+
+    // Dummy drill-env.sh to simulate the shipped "example" file
+    // with some client-specific changes.
+
+    context.writeFile( new File( siteDir, "drill-env.sh" ),
+        "#!/usr/bin/env bash\n" +
+        "# Example file\n" +
+        "export SQLLINE_JAVA_OPTS=\"-XX:MaxPermSize=256M\"\n"
+        );
+    File siteJars = new File(siteDir, "jars");
+    context.createDir(siteJars);
+    context.makeDummyJar(siteJars, "site");
+    {
+      RunResult result = new ScriptRunner("sqlline").withSite(siteDir).run();
+      assertEquals(0, result.returnCode);
+      assertTrue(result.containsArg("-XX:MaxPermSize=256M"));
+      result.validateClassPath(siteJars.getAbsolutePath() + "/*");
+    }
+  }
+
+  /**
+   * Tests the three scripts that wrap sqlline for specific purposes:
+   * <ul>
+   * <li>drill-conf — Wrapper for sqlline, uses drill config to find Drill.
+   * Seems this one needs fixing to use a config other than the hard-coded
+   * $DRILL_HOME/conf location.</li>
+   * <li>drill-embedded — Starts a drill “embedded” in SqlLine, using a local
+   * ZK.</li>
+   * <li>drill-localhost — Wrapper for sqlline, uses a local ZK.</li>
+   * </ul>
+   *
+   * Of these, drill-embedded runs an embedded Drillbit and so should use the
+   * Drillbit memory options. The other two are clients, but with simple default
+   * options for finding the Drillbit.
+   * <p>
+   * Because the scripts are simple wrappers, all we do is verify that the right
+   * "extra" options are set, not the fundamentals (which were already covered
+   * in the sqlline tests.)
+   *
+   * @throws IOException
+   */
+
+  @Test
+  public void testSqllineWrappers() throws IOException {
+    context.createMockDistrib();
+    File siteDir = new File(context.testDrillHome, "conf");
+    context.createMockConf(siteDir);
+
+    {
+      // drill-conf: just adds a stub JDBC connect string.
+
+      RunResult result = new ScriptRunner("drill-conf")
+          .withArg("arg1")
+          .run();
+      assertEquals(0, result.returnCode);
+      result.validateJava();
+      result.validateClassPath(ScriptUtils.stdCp);
+      assertTrue(result.containsArgsRegex(ScriptUtils.sqlLineArgs));
+      assertTrue(result.containsArg("-u"));
+      assertTrue(result.containsArg("jdbc:drill:"));
+      assertTrue(result.containsArg("arg1"));
+    }
+
+    {
+      // drill-localhost: Adds a JDBC connect string to a drillbit
+      // on the localhost
+
+      RunResult result = new ScriptRunner("drill-localhost")
+          .withArg("arg1")
+          .run();
+      assertEquals(0, result.returnCode);
+      result.validateJava();
+      result.validateClassPath(ScriptUtils.stdCp);
+      assertTrue(result.containsArgsRegex(ScriptUtils.sqlLineArgs));
+      assertTrue(result.containsArg("-u"));
+      assertTrue(result.containsArg("jdbc:drill:drillbit=localhost"));
+      assertTrue(result.containsArg("arg1"));
+    }
+
+    {
+      // drill-embedded: Uses drillbit startup options and
+      // connects to the embedded drillbit.
+
+      RunResult result = new ScriptRunner("drill-embedded")
+          .withArg("arg1")
+          .run();
+      assertEquals(0, result.returnCode);
+      result.validateJava();
+      result.validateClassPath(ScriptUtils.stdCp);
+      assertTrue(result.containsArgsRegex(ScriptUtils.sqlLineArgs));
+      assertTrue(result.containsArg("-u"));
+      assertTrue(result.containsArg("jdbc:drill:zk=local"));
+      assertTrue(result.containsArg("-Xms4G"));
+      assertTrue(result.containsArg("-XX:MaxDirectMemorySize=8G"));
+      assertTrue(result.containsArg("arg1"));
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/test/java/org/apache/drill/yarn/zk/TestAmRegistration.java
----------------------------------------------------------------------
diff --git a/drill-yarn/src/test/java/org/apache/drill/yarn/zk/TestAmRegistration.java b/drill-yarn/src/test/java/org/apache/drill/yarn/zk/TestAmRegistration.java
new file mode 100644
index 0000000..31c6349
--- /dev/null
+++ b/drill-yarn/src/test/java/org/apache/drill/yarn/zk/TestAmRegistration.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.yarn.zk;
+
+import static org.junit.Assert.*;
+
+import org.apache.curator.test.TestingServer;
+import org.apache.drill.yarn.appMaster.AMRegistrar.AMRegistrationException;
+import org.junit.Test;
+
+public class TestAmRegistration {
+  private static final String TEST_CLUSTER_ID = "drillbits";
+  private static final String TEST_ZK_ROOT = "drill";
+  private static final String TEST_AM_HOST = "localhost";
+  private static final int TEST_AM_PORT = 8048;
+  private static final String TEST_APP_ID = "Application_001";
+
+  private static final String PROBE_AM_HOST = "somehost";
+  private static final String PROBE_APP_ID = "Application_002";
+  private static final int PROBE_AM_PORT = 8049;
+  private static final String PROBE_CLUSTER_ID = "second";
+  private static final String PROBE_ZK_ROOT = "myRoot";
+
+  @Test
+  public void testAMRegistry() throws Exception {
+    try (TestingServer server = new TestingServer()) {
+      server.start();
+      String connStr = server.getConnectString();
+      ZKClusterCoordinatorDriver driver = new ZKClusterCoordinatorDriver()
+          .setConnect(connStr, TEST_ZK_ROOT, TEST_CLUSTER_ID).build();
+
+      // Register an AM using the above.
+
+      driver.register(TEST_AM_HOST, TEST_AM_PORT, TEST_APP_ID);
+
+      // Simulate a second AM for the same cluster.
+
+      {
+        ZKClusterCoordinatorDriver driver2 = new ZKClusterCoordinatorDriver()
+            .setConnect(connStr, TEST_ZK_ROOT, TEST_CLUSTER_ID).build();
+
+        // Register an AM on the same (root, cluster id).
+
+        try {
+          driver.register(PROBE_AM_HOST, PROBE_AM_PORT, PROBE_APP_ID);
+          fail();
+        } catch (AMRegistrationException e) {
+          String msg = e.getMessage();
+          assertTrue(msg.contains("Application Master already exists"));
+          assertTrue(
+              msg.contains(" " + TEST_ZK_ROOT + "/" + TEST_CLUSTER_ID + " "));
+          assertTrue(msg.contains(" host: " + TEST_AM_HOST));
+          assertTrue(msg.contains(" Application ID: " + TEST_APP_ID));
+        }
+
+        driver2.close();
+      }
+
+      {
+        ZKClusterCoordinatorDriver driver2 = new ZKClusterCoordinatorDriver()
+            .setConnect(connStr, TEST_ZK_ROOT, PROBE_CLUSTER_ID).build();
+
+        // Register an AM on a different cluster id, same root.
+
+        try {
+          driver2.register(PROBE_AM_HOST, PROBE_AM_PORT, PROBE_APP_ID);
+        } catch (AMRegistrationException e) {
+          fail("Registration should be OK");
+        }
+
+        driver2.close();
+      }
+
+      {
+        ZKClusterCoordinatorDriver driver2 = new ZKClusterCoordinatorDriver()
+            .setConnect(connStr, PROBE_ZK_ROOT, TEST_CLUSTER_ID).build();
+
+        // Register an AM on a different root.
+
+        try {
+          driver2.register(PROBE_AM_HOST, PROBE_AM_PORT, PROBE_APP_ID);
+        } catch (AMRegistrationException e) {
+          fail("Registration should be OK");
+        }
+
+        driver2.close();
+      }
+
+      // First AM exits.
+
+      driver.close();
+
+      {
+        // Should be able to register an AM for the same cluster.
+
+        ZKClusterCoordinatorDriver driver2 = new ZKClusterCoordinatorDriver()
+            .setConnect(connStr, TEST_ZK_ROOT, TEST_CLUSTER_ID).build();
+
+        // Register an AM on the same (root, cluster id).
+
+        try {
+          driver2.register(PROBE_AM_HOST, PROBE_AM_PORT, PROBE_APP_ID);
+        } catch (AMRegistrationException e) {
+          fail("Registration should be OK");
+        }
+
+        driver2.close();
+      }
+
+      server.stop();
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/test/java/org/apache/drill/yarn/zk/TestZkRegistry.java
----------------------------------------------------------------------
diff --git a/drill-yarn/src/test/java/org/apache/drill/yarn/zk/TestZkRegistry.java b/drill-yarn/src/test/java/org/apache/drill/yarn/zk/TestZkRegistry.java
new file mode 100644
index 0000000..b6f6540
--- /dev/null
+++ b/drill-yarn/src/test/java/org/apache/drill/yarn/zk/TestZkRegistry.java
@@ -0,0 +1,459 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.yarn.zk;
+
+import static org.junit.Assert.*;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.RetryNTimes;
+import org.apache.curator.test.TestingServer;
+import org.apache.curator.x.discovery.ServiceInstance;
+import org.apache.drill.exec.coord.DrillServiceInstanceHelper;
+import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
+import org.apache.drill.exec.work.foreman.DrillbitStatusListener;
+import org.apache.drill.yarn.appMaster.EventContext;
+import org.apache.drill.yarn.appMaster.RegistryHandler;
+import org.apache.drill.yarn.appMaster.Task;
+import org.apache.drill.yarn.appMaster.TaskLifecycleListener.Event;
+import org.apache.drill.yarn.zk.ZKRegistry.DrillbitTracker;
+import org.junit.Test;
+
+/**
+ * Tests for the AM version of the cluster coordinator. The AM version has no
+ * dependencies on the DoY config system or other systems, making it easy to
+ * test in isolation using the Curator-provided test server.
+ */
+
+public class TestZkRegistry {
+  private static final String BARNEY_HOST = "barney";
+  private static final String WILMA_HOST = "wilma";
+  private static final String TEST_HOST = "host";
+  private static final String FRED_HOST = "fred";
+  public static final int TEST_USER_PORT = 123;
+  public static final int TEST_CONTROL_PORT = 456;
+  public static final int TEST_DATA_PORT = 789;
+  public static final String ZK_ROOT = "test-root";
+  public static final String CLUSTER_ID = "test-cluster";
+
+  /**
+   * Validate that the key format used for endpoint is the same as that
+   * generated for hosts coming from YARN.
+   */
+
+  @Test
+  public void testFormat() {
+    DrillbitEndpoint dbe = makeEndpoint(TEST_HOST);
+    assertEquals(makeKey(TEST_HOST), ZKClusterCoordinatorDriver.asString(dbe));
+
+    ZKClusterCoordinatorDriver driver = new ZKClusterCoordinatorDriver()
+        .setPorts(123, 456, 789);
+    assertEquals(makeKey(TEST_HOST), driver.toKey(TEST_HOST));
+
+    // Internal default ports (used mostly for testing.)
+
+    driver = new ZKClusterCoordinatorDriver();
+    assertEquals("fred:31010:31011:31012", driver.toKey(FRED_HOST));
+  }
+
+  public static String makeKey(String host) {
+    return host + ":" + TEST_USER_PORT + ":" + TEST_CONTROL_PORT + ":"
+        + TEST_DATA_PORT;
+  }
+
+  /**
+   * Basic setup: start a ZK and verify that the initial endpoint list is empty.
+   * Also validates the basics of the test setup (mock server, etc.)
+   *
+   * @throws Exception
+   */
+
+  @Test
+  public void testBasics() throws Exception {
+    try (TestingServer server = new TestingServer()) {
+      server.start();
+      String connStr = server.getConnectString();
+      ZKClusterCoordinatorDriver driver = new ZKClusterCoordinatorDriver()
+          .setConnect(connStr, "drill", "drillbits").build();
+      assertTrue(driver.getInitialEndpoints().isEmpty());
+      driver.close();
+      server.stop();
+    }
+  }
+
+  private class TestDrillbitStatusListener implements DrillbitStatusListener {
+    protected Set<DrillbitEndpoint> added;
+    protected Set<DrillbitEndpoint> removed;
+
+    @Override
+    public void drillbitUnregistered(
+        Set<DrillbitEndpoint> unregisteredDrillbits) {
+      removed = unregisteredDrillbits;
+    }
+
+    @Override
+    public void drillbitRegistered(Set<DrillbitEndpoint> registeredDrillbits) {
+      added = registeredDrillbits;
+    }
+
+    public void clear() {
+      added = null;
+      removed = null;
+    }
+  }
+
+  /**
+   * Test a typical life cycle: existing Drillbit on AM start, add a Drilbit
+   * (simulates a drillbit starting), and remove a drillbit (simulates a
+   * Drillbit ending).
+   *
+   * @throws Exception
+   */
+
+  @Test
+  public void testCycle() throws Exception {
+    TestingServer server = new TestingServer();
+    server.start();
+    String connStr = server.getConnectString();
+
+    CuratorFramework probeZk = connectToZk(connStr);
+    addDrillbit(probeZk, FRED_HOST);
+
+    ZKClusterCoordinatorDriver driver = new ZKClusterCoordinatorDriver()
+        .setConnect(connStr, ZK_ROOT, CLUSTER_ID).build();
+    List<DrillbitEndpoint> bits = driver.getInitialEndpoints();
+    assertEquals(1, bits.size());
+    assertEquals(makeKey(FRED_HOST),
+        ZKClusterCoordinatorDriver.asString(bits.get(0)));
+
+    TestDrillbitStatusListener listener = new TestDrillbitStatusListener();
+    driver.addDrillbitListener(listener);
+
+    addDrillbit(probeZk, WILMA_HOST);
+    Thread.sleep(50);
+    assertNull(listener.removed);
+    assertNotNull(listener.added);
+    assertEquals(1, listener.added.size());
+    for (DrillbitEndpoint dbe : listener.added) {
+      assertEquals(makeKey(WILMA_HOST),
+          ZKClusterCoordinatorDriver.asString(dbe));
+    }
+
+    listener.clear();
+    removeDrillbit(probeZk, FRED_HOST);
+    Thread.sleep(50);
+    assertNull(listener.added);
+    assertNotNull(listener.removed);
+    assertEquals(1, listener.removed.size());
+    for (DrillbitEndpoint dbe : listener.removed) {
+      assertEquals(makeKey(FRED_HOST),
+          ZKClusterCoordinatorDriver.asString(dbe));
+    }
+
+    probeZk.close();
+    driver.close();
+    server.stop();
+    server.close();
+  }
+
+  /**
+   * Make a Drill endpoint using the hard-coded test ports and the given host
+   * name.
+   *
+   * @param host
+   * @return
+   */
+
+  private DrillbitEndpoint makeEndpoint(String host) {
+    return DrillbitEndpoint.newBuilder().setAddress(host)
+        .setControlPort(TEST_CONTROL_PORT).setDataPort(TEST_DATA_PORT)
+        .setUserPort(TEST_USER_PORT).build();
+  }
+
+  /**
+   * Pretend to be a Drillbit creating its ZK entry. Real Drillbits use a GUID
+   * as the key, but we just use the host name, which is good enough for our
+   * purposes here.
+   *
+   * @param zk
+   * @param host
+   * @throws Exception
+   */
+
+  private void addDrillbit(CuratorFramework zk, String host) throws Exception {
+    DrillbitEndpoint dbe = makeEndpoint(host);
+    ServiceInstance<DrillbitEndpoint> si = ServiceInstance
+        .<DrillbitEndpoint> builder().name(CLUSTER_ID).payload(dbe).build();
+    byte data[] = DrillServiceInstanceHelper.SERIALIZER.serialize(si);
+    zk.create().forPath("/" + host, data);
+  }
+
+  private void removeDrillbit(CuratorFramework zk, String host)
+      throws Exception {
+    zk.delete().forPath("/" + host);
+  }
+
+  /**
+   * Connect to the test ZK for the simulated Drillbit side of the test. (The AM
+   * side of the test uses the actual AM code, which is what we're testing
+   * here...)
+   *
+   * @param connectString
+   * @return
+   */
+
+  public static CuratorFramework connectToZk(String connectString) {
+    CuratorFramework client = CuratorFrameworkFactory.builder()
+        .namespace(ZK_ROOT + "/" + CLUSTER_ID).connectString(connectString)
+        .retryPolicy(new RetryNTimes(3, 1000)).build();
+    client.start();
+    return client;
+  }
+
+  private static class TestRegistryHandler implements RegistryHandler {
+    String reserved;
+    String released;
+    Task start;
+    Task end;
+
+    public void clear() {
+      reserved = null;
+      released = null;
+      start = null;
+      end = null;
+    }
+
+    @Override
+    public void reserveHost(String hostName) {
+      assertNull(reserved);
+      reserved = hostName;
+    }
+
+    @Override
+    public void releaseHost(String hostName) {
+      assertNull(released);
+      released = hostName;
+    }
+
+    @Override
+    public void startAck(Task task, String propertyKey, Object value) {
+      start = task;
+    }
+
+    @Override
+    public void completionAck(Task task, String endpointProperty) {
+      end = task;
+    }
+
+    @Override
+    public void registryDown() {
+      // TODO Auto-generated method stub
+
+    }
+  }
+
+  public static class TestTask extends Task {
+    private String host;
+
+    public TestTask(String host) {
+      super(null, null);
+      this.host = host;
+    }
+
+    @Override
+    public String getHostName() {
+      return host;
+    }
+
+    @Override
+    public void resetTrackingState() {
+      trackingState = TrackingState.NEW;
+    }
+  }
+
+  @Test
+  public void testZKRegistry() throws Exception {
+    TestingServer server = new TestingServer();
+    server.start();
+    String connStr = server.getConnectString();
+
+    CuratorFramework probeZk = connectToZk(connStr);
+    addDrillbit(probeZk, FRED_HOST);
+
+    ZKClusterCoordinatorDriver driver = new ZKClusterCoordinatorDriver()
+        .setConnect(connStr, ZK_ROOT, CLUSTER_ID)
+        .setPorts(TEST_USER_PORT, TEST_CONTROL_PORT, TEST_DATA_PORT).build();
+    ZKRegistry registry = new ZKRegistry(driver);
+    TestRegistryHandler handler = new TestRegistryHandler();
+    registry.start(handler);
+
+    // We started with one "stray" drillbit that will be reported as unmanaged.
+
+    assertEquals(FRED_HOST, handler.reserved);
+    List<String> unmanaged = registry.listUnmanagedDrillits();
+    assertEquals(1, unmanaged.size());
+    String fredsKey = makeKey(FRED_HOST);
+    assertEquals(fredsKey, unmanaged.get(0));
+    Map<String, DrillbitTracker> trackers = registry.getRegistryForTesting();
+    assertEquals(1, trackers.size());
+    assertTrue(trackers.containsKey(fredsKey));
+    DrillbitTracker fredsTracker = trackers.get(fredsKey);
+    assertEquals(fredsKey, fredsTracker.key);
+    assertEquals(DrillbitTracker.State.UNMANAGED, fredsTracker.state);
+    assertNull(fredsTracker.task);
+    assertEquals(fredsKey,
+        ZKClusterCoordinatorDriver.asString(fredsTracker.endpoint));
+
+    // The handler should have been told about the initial stray.
+
+    assertEquals(FRED_HOST, handler.reserved);
+
+    // Pretend to start a new Drillbit.
+
+    Task wilmasTask = new TestTask(WILMA_HOST);
+    EventContext context = new EventContext(wilmasTask);
+
+    // Registry ignores the created event.
+
+    registry.stateChange(Event.CREATED, context);
+    assertEquals(1, registry.getRegistryForTesting().size());
+
+    // But, does care about the allocated event.
+
+    registry.stateChange(Event.ALLOCATED, context);
+    assertEquals(2, registry.getRegistryForTesting().size());
+    String wilmasKey = makeKey(WILMA_HOST);
+    DrillbitTracker wilmasTracker = registry.getRegistryForTesting()
+        .get(wilmasKey);
+    assertNotNull(wilmasTracker);
+    assertEquals(wilmasTask, wilmasTracker.task);
+    assertNull(wilmasTracker.endpoint);
+    assertEquals(wilmasKey, wilmasTracker.key);
+    assertEquals(DrillbitTracker.State.NEW, wilmasTracker.state);
+    handler.clear();
+
+    // Time goes on. The Drillbit starts and registers itself.
+
+    addDrillbit(probeZk, WILMA_HOST);
+    Thread.sleep(100);
+    assertEquals(wilmasTask, handler.start);
+    assertEquals(DrillbitTracker.State.REGISTERED, wilmasTracker.state);
+    assertEquals(handler.start, wilmasTask);
+
+    // Create another task: Barney
+
+    Task barneysTask = new TestTask(BARNEY_HOST);
+    context = new EventContext(barneysTask);
+    registry.stateChange(Event.CREATED, context);
+
+    // Start Barney, but assume a latency in Yarn, but not ZK.
+    // We get the ZK registration before the YARN launch confirmation.
+
+    handler.clear();
+    addDrillbit(probeZk, BARNEY_HOST);
+    Thread.sleep(100);
+    assertEquals(BARNEY_HOST, handler.reserved);
+    String barneysKey = makeKey(BARNEY_HOST);
+    DrillbitTracker barneysTracker = registry.getRegistryForTesting()
+        .get(barneysKey);
+    assertNotNull(barneysTracker);
+    assertEquals(DrillbitTracker.State.UNMANAGED, barneysTracker.state);
+    assertNull(barneysTracker.task);
+    assertEquals(2, registry.listUnmanagedDrillits().size());
+
+    handler.clear();
+    registry.stateChange(Event.ALLOCATED, context);
+    assertEquals(DrillbitTracker.State.REGISTERED, barneysTracker.state);
+    assertEquals(handler.start, barneysTask);
+    assertEquals(barneysTask, barneysTracker.task);
+    assertEquals(1, registry.listUnmanagedDrillits().size());
+
+    // Barney is having problems, it it drops out of ZK.
+
+    handler.clear();
+    removeDrillbit(probeZk, BARNEY_HOST);
+    Thread.sleep(100);
+    assertEquals(barneysTask, handler.end);
+    assertEquals(DrillbitTracker.State.DEREGISTERED, barneysTracker.state);
+
+    // Barney comes alive (presumably before the controller gives up and kills
+    // the Drillbit.)
+
+    handler.clear();
+    addDrillbit(probeZk, BARNEY_HOST);
+    Thread.sleep(100);
+    assertEquals(barneysTask, handler.start);
+    assertEquals(DrillbitTracker.State.REGISTERED, barneysTracker.state);
+
+    // Barney is killed by the controller.
+    // ZK entry drops. Tracker is removed, controller is notified.
+
+    handler.clear();
+    removeDrillbit(probeZk, BARNEY_HOST);
+    Thread.sleep(100);
+    assertNotNull(registry.getRegistryForTesting().get(barneysKey));
+    assertEquals(barneysTask, handler.end);
+
+    // The controller tells the registry to stop tracking the Drillbit.
+
+    handler.clear();
+    registry.stateChange(Event.ENDED, context);
+    assertNull(handler.end);
+    assertNull(registry.getRegistryForTesting().get(barneysKey));
+
+    // The stray drillbit deregisters from ZK. The tracker is removed.
+
+    handler.clear();
+    removeDrillbit(probeZk, FRED_HOST);
+    Thread.sleep(100);
+    assertNull(registry.getRegistryForTesting().get(fredsKey));
+    assertNull(handler.end);
+    assertEquals(FRED_HOST, handler.released);
+
+    // Wilma is killed by the controller.
+
+    handler.clear();
+    removeDrillbit(probeZk, WILMA_HOST);
+    Thread.sleep(100);
+    assertEquals(wilmasTask, handler.end);
+    assertNull(handler.released);
+    assertEquals(DrillbitTracker.State.DEREGISTERED, wilmasTracker.state);
+    assertNotNull(registry.getRegistryForTesting().get(wilmasKey));
+
+    handler.clear();
+    context = new EventContext(wilmasTask);
+    registry.stateChange(Event.ENDED, context);
+    assertNull(registry.getRegistryForTesting().get(wilmasKey));
+    assertNull(handler.released);
+    assertNull(handler.end);
+
+    // All drillbits should be gone.
+
+    assertTrue(registry.getRegistryForTesting().isEmpty());
+
+    probeZk.close();
+    driver.close();
+    server.stop();
+    server.close();
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/test/resources/doy-test-logback.xml
----------------------------------------------------------------------
diff --git a/drill-yarn/src/test/resources/doy-test-logback.xml b/drill-yarn/src/test/resources/doy-test-logback.xml
new file mode 100644
index 0000000..b00ff5f
--- /dev/null
+++ b/drill-yarn/src/test/resources/doy-test-logback.xml
@@ -0,0 +1,56 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+  license agreements. See the NOTICE file distributed with this work for additional
+  information regarding copyright ownership. The ASF licenses this file to
+  You under the Apache License, Version 2.0 (the "License"); you may not use
+  this file except in compliance with the License. You may obtain a copy of
+  the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+  by applicable law or agreed to in writing, software distributed under the
+  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+  OF ANY KIND, either express or implied. See the License for the specific
+  language governing permissions and limitations under the License. -->
+<configuration>
+
+<!-- 
+  <appender name="SOCKET" class="de.huxhorn.lilith.logback.appender.ClassicMultiplexSocketAppender">
+    <Compressing>true</Compressing>
+    <ReconnectionDelay>10000</ReconnectionDelay>
+    <IncludeCallerData>true</IncludeCallerData>
+    <RemoteHosts>${LILITH_HOSTNAME:-localhost}</RemoteHosts>
+  </appender> -->
+
+ <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+    <!-- encoders are assigned the type
+         ch.qos.logback.classic.encoder.PatternLayoutEncoder by default -->
+    <encoder>
+      <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
+    </encoder>
+  </appender>
+
+<!--
+  <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+    <file>/logs/test-common.log</file>
+    <encoder>
+      <pattern>%date %level [%thread] %logger{10} [%file:%line] %msg%n</pattern>
+    </encoder>
+    <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+      <fileNamePattern>/logs/test-common.%d{yyyy-MM-dd}.log</fileNamePattern>
+      <maxHistory>30</maxHistory>
+    </rollingPolicy>
+  </appender>
+  -->
+  <logger name="org.apache.drill" additivity="false">
+    <level value="error" />
+    <!--   <appender-ref ref="SOCKET" /> -->
+    <appender-ref ref="STDOUT" />
+<!--     <appender-ref ref="FILE" /> -->
+  </logger>
+
+  <root>
+    <level value="error" />
+    <!-- <appender-ref ref="SOCKET" /> -->
+    <appender-ref ref="STDOUT" />
+<!--     <appender-ref ref="FILE" /> -->
+  </root>
+
+</configuration>

http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/test/resources/second-test-config.conf
----------------------------------------------------------------------
diff --git a/drill-yarn/src/test/resources/second-test-config.conf b/drill-yarn/src/test/resources/second-test-config.conf
new file mode 100644
index 0000000..11c180f
--- /dev/null
+++ b/drill-yarn/src/test/resources/second-test-config.conf
@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file contains a test set of configuration settings used by the
+# TestConfig unit test. It has a benign name to prevent any accidental
+# inclusion into production code. The test class makes a copy of this
+# file with the desired name for the life of the test run only.
+
+drill.yarn: {
+  app-name: "My-App"
+  
+  drill-install: {
+    localize: false
+    drill-home: "/config/drill/home"
+    site-dir: "/config/drill/site"
+    client-path: "/foo/bar/drill-archive.tar.gz"
+  }
+  
+  yarn: {
+    queue: "my-queue"
+  }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/test/resources/test-doy-config.conf
----------------------------------------------------------------------
diff --git a/drill-yarn/src/test/resources/test-doy-config.conf b/drill-yarn/src/test/resources/test-doy-config.conf
new file mode 100644
index 0000000..b588530
--- /dev/null
+++ b/drill-yarn/src/test/resources/test-doy-config.conf
@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file contains a test set of configuration settings used by the
+# TestConfig unit test. It has a benign name to prevent any accidental
+# inclusion into production code. The test class makes a copy of this
+# file with the desired name for the life of the test run only.
+
+drill.yarn: {
+  app-name: "My-App"
+  
+  drill-install: {
+    drill-home: "/config/drill/home"
+    client-path: "/foo/bar/drill-archive.tar.gz"
+  }
+  
+  yarn: {
+    queue: "my-queue"
+  }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/test/resources/test-doy-distrib.conf
----------------------------------------------------------------------
diff --git a/drill-yarn/src/test/resources/test-doy-distrib.conf b/drill-yarn/src/test/resources/test-doy-distrib.conf
new file mode 100644
index 0000000..ff73ef3
--- /dev/null
+++ b/drill-yarn/src/test/resources/test-doy-distrib.conf
@@ -0,0 +1,30 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file contains a test set of configuration settings used by the
+# TestConfig unit test. It has a benign name to prevent any accidental
+# inclusion into production code. The test class makes a copy of this
+# file with the desired name for the life of the test run only.
+
+drill.yarn: {
+  app-name: "config-app-name"
+  
+  dfs: {
+    app-dir: "/opt/drill"
+  }
+  yarn: {
+    queue: "distrib-queue"
+  }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/test/resources/third-test-config.conf
----------------------------------------------------------------------
diff --git a/drill-yarn/src/test/resources/third-test-config.conf b/drill-yarn/src/test/resources/third-test-config.conf
new file mode 100644
index 0000000..f585e22
--- /dev/null
+++ b/drill-yarn/src/test/resources/third-test-config.conf
@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file contains a test set of configuration settings used by the
+# TestConfig unit test. It has a benign name to prevent any accidental
+# inclusion into production code. The test class makes a copy of this
+# file with the desired name for the life of the test run only.
+
+drill.yarn: {
+  app-name: "My-App"
+  
+  drill-install: {
+    localize: false
+    drill-home: "/config/drill/home"
+  }
+  
+  yarn: {
+    queue: "my-queue"
+  }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/test/resources/wrapper.sh
----------------------------------------------------------------------
diff --git a/drill-yarn/src/test/resources/wrapper.sh b/drill-yarn/src/test/resources/wrapper.sh
new file mode 100644
index 0000000..f436eb5
--- /dev/null
+++ b/drill-yarn/src/test/resources/wrapper.sh
@@ -0,0 +1,53 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+echo "Drill Stdout Message"
+echo "Stderr Message" 1>&2
+
+output="$DRILL_HOME/../output.txt"
+while [[ $# > 0 ]]
+do
+  if [[ $1 =~ ^-Dlog.path= ]]; then
+    thelog=${1/-Dlog.path=//}
+    echo "Drill Log Message" >> $thelog
+  fi
+  echo $1 >> $output
+  shift
+done
+
+function clean_up {
+  echo "Received SIGTERM"
+  if [ "$PRETEND_HUNG" == "1" ]; then
+    echo "Pretending to be hung."
+  else
+    echo "Exiting"
+    if [ "$PRETEND_FAIL" == "1" ]; then
+      exit 55
+    else
+      exit 0
+    fi
+  fi
+}
+
+trap clean_up SIGTERM
+
+if [ "$KEEP_RUNNING" == "1" ]; then
+  while [[ 1 > 0 ]]
+  do
+    sleep 1
+  done
+fi

http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 2c1f5fc..a163270 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1555,6 +1555,62 @@
             </exclusions>
           </dependency>
           <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-yarn-client</artifactId>
+            <version>${hadoop.version}</version>
+            <scope>compile</scope>
+            <exclusions>
+              <exclusion>
+                <groupId>com.sun.jersey</groupId>
+                <artifactId>jersey-core</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>com.sun.jersey</groupId>
+                <artifactId>jersey-server</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>com.sun.jersey</groupId>
+                <artifactId>jersey-json</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>com.sun.jersey</groupId>
+                <artifactId>jersey-client</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>org.codehaus.jackson</groupId>
+                <artifactId>jackson-core-asl</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>org.codehaus.jackson</groupId>
+                <artifactId>jackson-mapper-asl</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>org.codehaus.jackson</groupId>
+                <artifactId>jackson-xc</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>org.codehaus.jackson</groupId>
+                <artifactId>jackson-jaxrs</artifactId>
+              </exclusion>
+
+              <!-- Exclude the following to prevent enforcer failures. YARN uses them,
+                   but Drill forbids them. -->
+
+              <exclusion>
+                <groupId>log4j</groupId>
+                <artifactId>log4j</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>javax.servlet</groupId>
+                <artifactId>servlet-api</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>commons-logging</groupId>
+                <artifactId>commons-logging</artifactId>
+              </exclusion>
+            </exclusions>
+          </dependency>
+          <dependency>
             <groupId>org.apache.hbase</groupId>
             <artifactId>hbase-client</artifactId>
             <version>${hbase.version}</version>
@@ -2143,6 +2199,62 @@
             </exclusions>
           </dependency>
           <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-yarn-client</artifactId>
+            <version>${hadoop.version}</version>
+            <scope>compile</scope>
+            <exclusions>
+              <exclusion>
+                <groupId>com.sun.jersey</groupId>
+                <artifactId>jersey-core</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>com.sun.jersey</groupId>
+                <artifactId>jersey-server</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>com.sun.jersey</groupId>
+                <artifactId>jersey-json</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>com.sun.jersey</groupId>
+                <artifactId>jersey-client</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>org.codehaus.jackson</groupId>
+                <artifactId>jackson-core-asl</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>org.codehaus.jackson</groupId>
+                <artifactId>jackson-mapper-asl</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>org.codehaus.jackson</groupId>
+                <artifactId>jackson-xc</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>org.codehaus.jackson</groupId>
+                <artifactId>jackson-jaxrs</artifactId>
+              </exclusion>
+
+              <!-- Exclude the following to prevent enforcer failures. YARN uses them,
+                   but Drill forbids them. -->
+
+              <exclusion>
+                <groupId>log4j</groupId>
+                <artifactId>log4j</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>javax.servlet</groupId>
+                <artifactId>servlet-api</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>commons-logging</groupId>
+                <artifactId>commons-logging</artifactId>
+              </exclusion>
+            </exclusions>
+          </dependency>
+          <dependency>
             <groupId>org.antlr</groupId>
             <artifactId>antlr4-runtime</artifactId>
             <version>4.5</version>
@@ -2506,6 +2618,62 @@
               </exclusion>
             </exclusions>
           </dependency>
+          <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-yarn-client</artifactId>
+            <version>2.3.0-cdh5.0.3</version>
+            <scope>compile</scope>
+            <exclusions>
+              <exclusion>
+                <groupId>com.sun.jersey</groupId>
+                <artifactId>jersey-core</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>com.sun.jersey</groupId>
+                <artifactId>jersey-server</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>com.sun.jersey</groupId>
+                <artifactId>jersey-json</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>com.sun.jersey</groupId>
+                <artifactId>jersey-client</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>org.codehaus.jackson</groupId>
+                <artifactId>jackson-core-asl</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>org.codehaus.jackson</groupId>
+                <artifactId>jackson-mapper-asl</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>org.codehaus.jackson</groupId>
+                <artifactId>jackson-xc</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>org.codehaus.jackson</groupId>
+                <artifactId>jackson-jaxrs</artifactId>
+              </exclusion>
+
+              <!-- Exclude the following to prevent enforcer failures. YARN uses them,
+                   but Drill forbids them. -->
+
+              <exclusion>
+                <groupId>log4j</groupId>
+                <artifactId>log4j</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>javax.servlet</groupId>
+                <artifactId>servlet-api</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>commons-logging</groupId>
+                <artifactId>commons-logging</artifactId>
+              </exclusion>
+            </exclusions>
+          </dependency>
         </dependencies>
       </dependencyManagement>
       <repositories>
@@ -2580,6 +2748,62 @@
               </exclusion>
             </exclusions>
           </dependency>
+          <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-yarn-client</artifactId>
+            <version>2.4.0.2.1.3.0-563</version>
+            <scope>compile</scope>
+            <exclusions>
+              <exclusion>
+                <groupId>com.sun.jersey</groupId>
+                <artifactId>jersey-core</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>com.sun.jersey</groupId>
+                <artifactId>jersey-server</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>com.sun.jersey</groupId>
+                <artifactId>jersey-json</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>com.sun.jersey</groupId>
+                <artifactId>jersey-client</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>org.codehaus.jackson</groupId>
+                <artifactId>jackson-core-asl</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>org.codehaus.jackson</groupId>
+                <artifactId>jackson-mapper-asl</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>org.codehaus.jackson</groupId>
+                <artifactId>jackson-xc</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>org.codehaus.jackson</groupId>
+                <artifactId>jackson-jaxrs</artifactId>
+              </exclusion>
+
+              <!-- Exclude the following to prevent enforcer failures. YARN uses them,
+                   but Drill forbids them. -->
+
+              <exclusion>
+                <groupId>log4j</groupId>
+                <artifactId>log4j</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>javax.servlet</groupId>
+                <artifactId>servlet-api</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>commons-logging</groupId>
+                <artifactId>commons-logging</artifactId>
+              </exclusion>
+            </exclusions>
+          </dependency>
         </dependencies>
       </dependencyManagement>
       <repositories>
@@ -2723,6 +2947,7 @@
     <module>common</module>
     <module>logical</module>
     <module>exec</module>
+    <module>drill-yarn</module>
     <module>distribution</module>
   </modules>
 </project>


Mime
View raw message