drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ar...@apache.org
Subject [02/12] drill git commit: DRILL-1170: YARN integration for Drill
Date Sun, 04 Mar 2018 17:13:42 GMT
http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/resources/drill-am/manage.ftl
----------------------------------------------------------------------
diff --git a/drill-yarn/src/main/resources/drill-am/manage.ftl b/drill-yarn/src/main/resources/drill-am/manage.ftl
new file mode 100644
index 0000000..682530c
--- /dev/null
+++ b/drill-yarn/src/main/resources/drill-am/manage.ftl
@@ -0,0 +1,78 @@
+<#-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+  license agreements. See the NOTICE file distributed with this work for additional
+  information regarding copyright ownership. The ASF licenses this file to
+  You under the Apache License, Version 2.0 (the "License"); you may not use
+  this file except in compliance with the License. You may obtain a copy of
+  the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+  by applicable law or agreed to in writing, software distributed under the
+  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+  OF ANY KIND, either express or implied. See the License for the specific
+  language governing permissions and limitations under the License. -->
+
+<#include "*/generic.ftl">
+<#macro page_head>
+</#macro>
+
+<#macro page_body>
+  <h4>Manage Drill Cluster</h4>
+
+  Current Status: ${model.getLiveCount( )}
+  <#if model.getLiveCount( ) == 1 >Drillbit is
+  <#else>Drillbits are
+  </#if>running.
+  <p>
+  Free YARN nodes: Approximately ${model.getFreeNodeCount( )} 
+  <p><p>
+  
+  <table class="table table-hover" style="width: auto;">
+    <#-- Removed per user feedback. (Kept in REST API as client needs them.
+    <tr><td style="vertical-align: middle;">
+      <form action="/resize" method="POST" class="form-inline" role="form">
+        <div class="form-group">
+          <input hidden name="type" value="grow">
+          <label for="add">Add</label>
+          <input type="text" name="n" size="6" id="add" class="form-control"
+                  placeholder="+n" style="padding: 0 1em; margin: 0 1em;"/>
+          drillbits.
+          <button type="submit" class="btn btn-primary" style="margin: 0 1em;">Go</button>
+        </div>
+      </form>
+    </td></tr>
+    <tr><td>
+      <form action="/resize" method="POST" class="form-inline" role="form">
+        <div class="form-group">
+          <input hidden name="type" value="shrink">
+          <label for="shrink">Remove</label>
+          <input type="text" name="n" size="6" id="shrink" class="form-control"
+                  placeholder="-n" style="padding: 0 1em; margin: 0 1em;"/>
+          drillbits.
+          <button type="submit" class="btn btn-primary" style="margin: 0 1em;">Go</button>
+        </div>
+      </form>
+    </td></tr>
+    -->
+    <tr><td>
+      <form action="/resize" method="POST" class="form-inline" role="form">
+        <div class="form-group">
+          <input hidden name="type" value="resize">
+          <label for="resize">Resize to</label>
+          <input type="text" name="n" id="resize" size="6"
+                  placeholder="Size" class="form-control" style="padding: 0 1em; margin: 0 1em;"/>
+          drillbits.
+          <button type="submit" class="btn btn-primary" style="margin: 0 1em;">Go</button>
+        </div>
+      </form>
+    </td></tr>
+    <tr><td>
+      <form action="/stop" method="GET" class="form-inline" role="form">
+        <div class="form-group">
+          <label for="stop">Stop</label> the Drill cluster.
+          <button type="submit" id="stop" class="btn btn-primary" style="margin: 0 1em;">Go</button>
+        </div>
+      </form>
+    </td></tr>
+  </table>
+
+</#macro>
+
+<@page_html/>

http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/resources/drill-am/redirect.ftl
----------------------------------------------------------------------
diff --git a/drill-yarn/src/main/resources/drill-am/redirect.ftl b/drill-yarn/src/main/resources/drill-am/redirect.ftl
new file mode 100644
index 0000000..b38d10c
--- /dev/null
+++ b/drill-yarn/src/main/resources/drill-am/redirect.ftl
@@ -0,0 +1,33 @@
+<#-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+  license agreements. See the NOTICE file distributed with this work for additional
+  information regarding copyright ownership. The ASF licenses this file to
+  You under the Apache License, Version 2.0 (the "License"); you may not use
+  this file except in compliance with the License. You may obtain a copy of
+  the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+  by applicable law or agreed to in writing, software distributed under the
+  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+  OF ANY KIND, either express or implied. See the License for the specific
+  language governing permissions and limitations under the License. -->
+
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+    <title>Apache Drill - Application Master</title>
+    <META http-equiv="refresh" content="0;URL=${amLink}">
+    <style>
+      body { font-family: sans-serif;
+             text-align: center; }
+    </style> 
+  </head>
+  <body>
+    <h3>YARN Application Master &ndash; ${clusterName}</h3>   
+    <h4>Redirect</h4>
+    The Drill Application Master UI does not work correctly inside the proxy page
+    provided by YARN.
+    <p>Click
+    <a href="${amLink}">here</a> to go to the Application Master directly
+    if this page does not automatically redirect you.
+  </body>
+</html>
+  
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/resources/drill-am/shrink-warning.ftl
----------------------------------------------------------------------
diff --git a/drill-yarn/src/main/resources/drill-am/shrink-warning.ftl b/drill-yarn/src/main/resources/drill-am/shrink-warning.ftl
new file mode 100644
index 0000000..dded46a
--- /dev/null
+++ b/drill-yarn/src/main/resources/drill-am/shrink-warning.ftl
@@ -0,0 +1,58 @@
+<#-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+  license agreements. See the NOTICE file distributed with this work for additional
+  information regarding copyright ownership. The ASF licenses this file to
+  You under the Apache License, Version 2.0 (the "License"); you may not use
+  this file except in compliance with the License. You may obtain a copy of
+  the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+  by applicable law or agreed to in writing, software distributed under the
+  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+  OF ANY KIND, either express or implied. See the License for the specific
+  language governing permissions and limitations under the License. -->
+
+<#include "*/generic.ftl">
+<#macro page_head>
+</#macro>
+
+<#macro page_body>
+  <h4><#if model.isStop( )>
+  Confirm Cluster Shutdown
+  <#else>
+  Confirm Stopping of Drillbits
+  </#if></h4>
+
+  <div class="alert alert-danger">
+    <strong>Warning!</strong> You have requested to
+    <#if model.isStop()>
+    stop the Drill cluster.
+    <#elseif model.isCancel()>
+    cancel Drillbit ${model.getId()}.
+    <#elseif model.isKill()>
+    kill Drillbit ${model.getId()}.
+    <#else>
+    remove ${model.getCount( )}
+    <#if model.getCount() == 1>Drillbit<#else>Drillbits</#if>.
+    </#if>
+    <#if ! model.isCancel()>
+    In this version of Drill, stopping Drillbits will
+    cause in-flight queries to fail.
+    </#if>
+  </div>
+  <#if model.isStop( )>
+    <form method="POST" action="/stop">
+  <#elseif model.isCancel( )>
+    <form method="POST" action="/cancel?id=${model.getId( )}">
+  <#elseif model.isKill( )>
+    <form method="POST" action="/cancel?id=${model.getId( )}">
+  <#else>
+    <form method="POST" action="/resize">
+  </#if>
+  <#if model.isShrink( )>
+    <input type="hidden" name="n" value="${model.getCount( )}">
+    <input type="hidden" name="type" value="force-shrink">
+  </#if>
+  <input type="submit" value="Confirm"> or
+  <a href="/">Cancel</a>.
+  </form>
+</#macro>
+
+<@page_html/>

http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/resources/drill-am/static/css/drill-am.css
----------------------------------------------------------------------
diff --git a/drill-yarn/src/main/resources/drill-am/static/css/drill-am.css b/drill-yarn/src/main/resources/drill-am/static/css/drill-am.css
new file mode 100644
index 0000000..a58fad1
--- /dev/null
+++ b/drill-yarn/src/main/resources/drill-am/static/css/drill-am.css
@@ -0,0 +1,20 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more contributor
+  license agreements. See the NOTICE file distributed with this work for additional
+  information regarding copyright ownership. The ASF licenses this file to
+  You under the Apache License, Version 2.0 (the "License"); you may not use
+  this file except in compliance with the License. You may obtain a copy of
+  the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+  by applicable law or agreed to in writing, software distributed under the
+  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+  OF ANY KIND, either express or implied. See the License for the specific
+  language governing permissions and limitations under the License. */
+  
+body {
+  padding-top: 50px;
+}
+.drill-am {
+  padding: 0 15px 20px 15px;
+}
+h3 {
+  text-align: center;
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/resources/drill-am/static/img/apache-drill-logo.png
----------------------------------------------------------------------
diff --git a/drill-yarn/src/main/resources/drill-am/static/img/apache-drill-logo.png b/drill-yarn/src/main/resources/drill-am/static/img/apache-drill-logo.png
new file mode 100644
index 0000000..bce39c0
Binary files /dev/null and b/drill-yarn/src/main/resources/drill-am/static/img/apache-drill-logo.png differ

http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/resources/drill-am/static/img/drill.ico
----------------------------------------------------------------------
diff --git a/drill-yarn/src/main/resources/drill-am/static/img/drill.ico b/drill-yarn/src/main/resources/drill-am/static/img/drill.ico
new file mode 100644
index 0000000..0f9654e
Binary files /dev/null and b/drill-yarn/src/main/resources/drill-am/static/img/drill.ico differ

http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/resources/drill-am/tasks.ftl
----------------------------------------------------------------------
diff --git a/drill-yarn/src/main/resources/drill-am/tasks.ftl b/drill-yarn/src/main/resources/drill-am/tasks.ftl
new file mode 100644
index 0000000..c6725b1
--- /dev/null
+++ b/drill-yarn/src/main/resources/drill-am/tasks.ftl
@@ -0,0 +1,113 @@
+<#-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+  license agreements. See the NOTICE file distributed with this work for additional
+  information regarding copyright ownership. The ASF licenses this file to
+  You under the Apache License, Version 2.0 (the "License"); you may not use
+  this file except in compliance with the License. You may obtain a copy of
+  the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+  by applicable law or agreed to in writing, software distributed under the
+  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+  OF ANY KIND, either express or implied. See the License for the specific
+  language governing permissions and limitations under the License. -->
+
+<#include "*/generic.ftl">
+<#macro page_head>
+  <meta http-equiv="refresh" content="${refreshSecs}" >
+</#macro>
+
+<#macro page_body>
+  <h4>Drillbit Status</h4>
+  <p>&nbsp;
+
+  <#if model.hasTasks( ) >
+    <div class="table-responsive">
+      <table class="table table-hover">
+        <tr>
+          <th><span data-toggle="tooltip" title="Internal AM ID for the Drillbit.">ID</span></th>
+          <th><span data-toggle="tooltip" title="Cluster group from config file">Group</span></th>
+          <th><span data-toggle="tooltip"
+                    title="Host name or IP running the Drillbit and ink to Drillbit web UI.">
+              Host</span></th>
+          <th><span data-toggle="tooltip" title="State of the Drillbit process, hover for details.">State</span></th>
+          <th><span data-toggle="tooltip" title="ZooKeeper tracking state for the Drillbit, hover for details.">ZK State</span></th>
+          <th><span data-toggle="tooltip"
+                    title="YARN Container allocated to the Drillbit and link to the YARN Node Manager container UI.">
+              Container</span></th>
+          <th><span data-toggle="tooltip" title="Memory granted by YARN to the Drillbit.">Memory (MB)</span></th>
+          <th><span data-toggle="tooltip" title="Virtual cores granted by YARN to the Drillbit.">Virtual Cores</span></th>
+          <#if showDisks >
+            <th><span data-toggle="tooltip" title="Disk resources granted by YARN to the Drillbit.">Disks</span></th>
+          </#if>
+          <th><span data-toggle="tooltip" title="Start time in the AM server time zone.">Start Time</span></th>
+        </th>
+        <#list tasks as task>
+          <tr>
+            <td><b>${task.getTaskId( )}</b></td>
+            <td>${task.getGroupName( )}</td>
+            <td>
+            <#if task.isLive( )>
+              <a href="${task.getLink( )}" data-toggle="tooltip" title="Link to the Drillbit Web UI"></#if>
+            ${task.getHost( )}
+            <#if task.isLive( )></a></#if>
+            </td>
+            <td><span data-toggle="tooltip" title="${task.getStateHint( )}">${task.getState( )}</span>
+            <#if task.isCancelled( )><br/>(Cancelled)</#if>
+            <#if task.isCancellable( )>
+              <a href="/cancel?id=${task.getTaskId( )}" data-toggle="tooltip" title="Kill this Drillbit">[x]</a>
+            </#if>
+            </td>
+            <td><span data-toggle="tooltip" title="${task.getTrackingStateHint( )}">${task.getTrackingState( )}</span></td>
+            <td><#if task.hasContainer( )>
+              <a href="${task.getNmLink( )}" data-toggle="tooltip" title="Node Manager UI for Drillbit container">${task.getContainerId()}</a>
+            <#else>&nbsp;</#if></td>
+            <td>${task.getMemory( )}</td>
+            <td>${task.getVcores( )}</td>
+            <#if showDisks >
+              <td>${task.getDisks( )}</td>
+            </#if>
+            <td>${task.getStartTime( )}</td>
+          </tr>
+        </#list>
+      </table>
+    <#else>
+      <div class="alert alert-danger">
+        No drillbits are running.
+      </div>
+    </#if>
+    <#if model.hasUnmanagedDrillbits( ) >
+      <hr>
+      <div class="alert alert-danger">
+        <strong>Warning:</strong> ZooKeeper reports that
+        ${model.getUnmanagedDrillbitCount( )} Drillbit(s) are running that were not
+        started by the YARN Application Master. Perhaps they were started manually.
+      </div>
+      <table class="table table-hover" style="width: auto;">
+        <tr><th>Drillbit Host</th><th>Ports</th></tr>
+        <#list strays as stray >
+          <tr>
+            <td>${stray.getHost( )}</td>
+            <td>${stray.getPorts( )}</td>
+          </tr>
+        </#list>
+      </table>
+    </#if>
+    <#if model.hasBlacklist( ) >
+      <hr>
+      <div class="alert alert-danger">
+        <strong>Warning:</strong> 
+        ${model.getBlacklistCount( )} nodes have been black-listed due to
+        repeated Drillbit launch failures. Perhaps the nodes or Drill are
+        improperly configured.
+      </div>
+      <table class="table table-hover" style="width: auto;">
+        <tr><th>Blacklisted Host</th></tr>
+        <#list blacklist as node >
+          <tr>
+            <td>${node}</td>
+          </tr>
+        </#list>
+      </table>
+    </#if>
+  </div>
+</#macro>
+
+<@page_html/>

http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/resources/org/apache/drill/yarn/core/drill-on-yarn-defaults.conf
----------------------------------------------------------------------
diff --git a/drill-yarn/src/main/resources/org/apache/drill/yarn/core/drill-on-yarn-defaults.conf b/drill-yarn/src/main/resources/org/apache/drill/yarn/core/drill-on-yarn-defaults.conf
new file mode 100644
index 0000000..bda8152
--- /dev/null
+++ b/drill-yarn/src/main/resources/org/apache/drill/yarn/core/drill-on-yarn-defaults.conf
@@ -0,0 +1,275 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ---------------------------------------------------------------------------
+# Configuration for the Drill-on-YARN feature.
+# See drill-on-yarn-example.conf for details.
+# Some properties are private; the are meant for use by the Drill
+# developers. Those are marked as private here, and are explained here
+# rather than in the example config.
+
+drill.yarn: {
+  app-name: "Drill-on-YARN"
+
+  # Settings here support a default single-node cluster on the local host,
+  # using the default HDFS connection obtained from the Hadoop config files.
+
+  dfs: {
+    connection: ""
+    app-dir: "/user/drill"
+  }
+
+  drill-install: {
+    site-dir: ""
+    client-path: ""
+    dir-name: "<base>"
+
+    # Drill-on-YARN uses localization by default; it is the simplest and
+    # most reliable for production systems. However, localization can be
+    # disabled for development, testing or other special needs.
+
+    localize: true
+
+    # Localized runs typically provide a site directory. For various reasons,
+    # some users may wish to avoid using a site directory and instead just
+    # use their own $DRILL_HOME/conf directory as the site directory. In
+    # this case, $DRILL_HOME/conf must contain all site-specific content,
+    # including jars. This setting says to treat $DRILL_HOME/conf as the
+    # site directory. Disable this option if the Drill archive is rebuilt
+    # to contain the $DRILL_HOME/conf files. This flag is also automatically
+    # assumed if the Drill archive is *inside* $DRILL_HOME.
+
+    conf-as-site: true
+
+    # When localization is disabled, the path to Drill software defaults
+    # to the same location as on the client node. However, if the path
+    # on worker nodes is different, set the actual path here.
+
+    drill-home: ""
+
+    # Drill provides the option to store site-specific configuration outside
+    # of the Drill home directory. The site directory contains configuration
+    # files and custom Java code (in the jars subdirectory.) The site dir
+    # normally is localized by the client. However, in a non-localized
+    # run, use this setting to identify the location of the site directory
+    # on each worker node if the location is different than the node
+    # that runs the client.
+
+    site-dir: ""
+
+    # Key used for the Drill archive file in the AM launch config.
+    # Not usually changed by users.
+
+    drill-key: "drill"
+    site-key: "site"
+
+    # Set the Java java.library.path option to files that pre-exist on
+    # each Drillbit node. (This is not for libraries that are distributed
+    # by YARN.)
+
+    library-path: ""
+  }
+
+  yarn: {
+    queue: "default"
+    priority: 1
+    user: ""
+  }
+
+  hadoop: {
+    home: ""
+    class-path: ""
+    hbase-class-path: ""
+  }
+
+  client: {
+
+    # Time (in secs) between poll attempts on start and stop.
+
+    poll-sec: 1
+
+    # Maximum time to wait when starting the AM before asking the
+    # user to check status later.
+
+    start-wait-sec: 60
+
+    # Maximum time to wait when stopping the AM before skipping
+    # the confirmation step.
+
+    stop-wait-sec: 60
+  }
+
+  am: {
+    memory-mb: 512
+
+    # The AM is CPU-light, a single core will do.
+
+    vcores: 1
+
+    # The AM uses no disk. Available on selected YARN distributions.
+
+    disks: 0
+
+    # Heap memory for the AM. Estimate based on observation.
+
+    heap: "450M"
+
+    # Arguments passed to the AM JVM. (Private)
+
+    vm-args: ""
+
+    # AM-to-RM period in ms. (Private)
+
+    poll-ms: 1000
+
+    # AM clock tick period in ms. (Private)
+
+    tick-ms: 2000
+
+    # Set to true to dump launch variables and command to log.
+
+    debug-launch: false
+
+    # Extra class path for AM. The path must be valid on all nodes.
+    # Not normally used; provided only for special cases to avoid editing
+    # the launch script. This is a class path prefix to allow overrides
+    # of Drill classes.
+    # Analogous to DRILL_CLASSPATH_PREFIX in drill-config.sh
+    # Multiple entries should be separated by colons.
+
+    prefix-class-path: ""
+
+    # Extra class path for AM. The path must be valid on all nodes on which
+    # the AM may run.
+    # Not normally used; provided only for special cases to avoid editing
+    # the launch script.
+
+    class-path: ""
+
+    # Enables/disables auto shutdown when no Drillbits can run. Disable
+    # this when required for single-node testing.
+
+    auto-shutdown: true
+
+    # Specify nodes that can run the AM via a YARN node label expression.
+    # Blank means that node labele expression will be applied.
+
+    node-label-expr: ""
+  }
+
+  drillbit: {
+
+    # Default memory: 4096 (heap) + 8192 (direct) +  1024 (code cache) + 1024 overhead.
+
+    memory-mb: 14336
+    vcores: 4
+    disks: 1
+
+    heap: "4G"
+    max-direct-memory: "8G"
+    code-cache: "1G"
+
+    # Additional JVM arguments for the drillbit (Private)
+    # The value here is appended to any set in the launch script.
+    # Separate multiple arguments by spaces.
+
+    vm-args: ""
+
+    # Set to true to turn on garbage collection logging.
+    # Similar to SERVER_GC_OPTS in drillbit.sh. However under YARN
+    # all logging goes to YARN's log directory and the GC log is
+    # always called "gc.log".
+
+    log-gc: false
+
+    # Extra class path for Drill-bit. The path must be valid on all nodes.
+    # Not normally used; provided only for special cases to avoid editing
+    # the launch script. This is a class path prefix to allow overrides
+    # of Drill classes.
+    # Equivalent to DRILL_CLASSPATH_PREFIX in drill-config.sh
+    # Multiple entries should be separated by colons.
+
+    prefix-class-path: ""
+
+    # Extra class path for Drill extensions such as Hadoop or HBase.
+    # The path must be valid on all nodes. Inserted into Drill's
+    # class path before Drill's own third-party extensions; allows
+    # overriding of Drill's own included jars.
+    # Equivalent to HADOOP_CLASSPATH and HBASE_CLASSPATH
+    # in drill-config.sh
+
+    extn-class-path: ""
+
+    # Normal class path specification for jars added to Drill's own.
+    # Appears in the class path after Drill's jars. Useful for dependencies
+    # on custom data source implementations.
+    # The path must be valid on all nodes.
+    # Equivalent to DRILL_CLASSPATH in drill-config.sh
+
+    class-path: ""
+
+    # Maximum number of retries of each Drillbit before blacklisting the node.
+
+    max-retries: 3
+
+    # Set to true to dump launch variables and command to log.
+
+    debug-launch: false
+
+    # Disable normal YARN log location. Allows Drill to write to
+    # it's own log location as when running outside of YARN. However,
+    # under YARN, log aggregation is generally considered a "good thing."
+
+    disable-yarn-logs: false
+
+    # DoY maintains an approximate count of free nodes available under YARN.
+    # A common error is to ask for far more Drillbits than can be run given
+    # the available nodes. This setting limits the number of "extra" requests
+    # beyond the known free nodes. The reason this number is not zero is that
+    # the DoY count is approximate, so we want to provide so "wiggle room."
+
+    max-extra-nodes: 2
+
+    # Maximum time that the AM will wait when requesting a container to run
+    # the Drillbit. After this time, the request will timeout and the AM will
+    # decrease the desired cluster size to match the fact that no additional
+    # node appears to be available. The timeout is in seconds. 0 means no
+    # timeout
+
+    request-timeout-secs: 600
+  }
+
+  http: {
+    enabled: true
+    port: 8048
+    ssl-enabled: false
+    auth-type: "none"
+    # Default of one hour, as in the Drillbit
+    session-max-idle-secs: 3600
+    rest-key="secret"
+    docs-link: "http://drill.apache.org/docs/"
+    refresh-secs: 5
+  }
+
+  cluster: [
+
+    # Defined to support a demo single-node cluster.
+
+    {
+      name: "default"
+      type: "basic"
+      count: 1
+    }
+  ]
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/test/java/org/apache/drill/yarn/client/TestClient.java
----------------------------------------------------------------------
diff --git a/drill-yarn/src/test/java/org/apache/drill/yarn/client/TestClient.java b/drill-yarn/src/test/java/org/apache/drill/yarn/client/TestClient.java
new file mode 100644
index 0000000..7912bd5
--- /dev/null
+++ b/drill-yarn/src/test/java/org/apache/drill/yarn/client/TestClient.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.yarn.client;
+
+import static org.junit.Assert.*;
+
+import java.io.ByteArrayOutputStream;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.io.UnsupportedEncodingException;
+
+import org.apache.drill.yarn.client.ClientContext;
+import org.apache.drill.yarn.client.DrillOnYarn;
+import org.junit.Test;
+
+public class TestClient {
+
+  /**
+   * Unchecked exception to allow capturing "exit" events without actually
+   * exiting.
+   */
+
+  public static class SimulatedExitException extends RuntimeException {
+    private static final long serialVersionUID = 1L;
+    public int exitCode;
+
+    public SimulatedExitException(int exitCode) {
+      this.exitCode = exitCode;
+    }
+  }
+
+  public static class TestContext extends ClientContext {
+    public static ByteArrayOutputStream captureOut = new ByteArrayOutputStream();
+    public static ByteArrayOutputStream captureErr = new ByteArrayOutputStream();
+
+    public static void testInit() {
+      init(new TestContext());
+      resetOutput();
+    }
+
+    @Override
+    public void exit(int exitCode) {
+      throw new SimulatedExitException(exitCode);
+    }
+
+    public static void resetOutput() {
+      try {
+        out.flush();
+        captureOut.reset();
+        out = new PrintStream(captureOut, true, "UTF-8");
+        err.flush();
+        captureErr.reset();
+        err = new PrintStream(captureErr, true, "UTF-8");
+      } catch (UnsupportedEncodingException e) {
+        throw new IllegalStateException(e);
+      }
+    }
+
+    public static String getOut() {
+      out.flush();
+      try {
+        return captureOut.toString("UTF-8");
+      } catch (UnsupportedEncodingException e) {
+        throw new IllegalStateException(e);
+      }
+    }
+
+    public static String getErr() {
+      out.flush();
+      try {
+        return captureErr.toString("UTF-8");
+      } catch (UnsupportedEncodingException e) {
+        throw new IllegalStateException(e);
+      }
+    }
+  }
+
+  /**
+   * Test the basics of the DrillOnYarn app. Does not try any real commands, but
+   * does check for the basic error conditions.
+   */
+
+  @Test
+  public void testBasics() {
+    TestContext.testInit();
+
+    // No arguments provided.
+
+    try {
+      DrillOnYarn.run(new String[] {});
+      fail();
+    } catch (SimulatedExitException e) {
+      assert (e.exitCode == -1);
+      assertTrue(TestContext.getOut().contains("Usage: "));
+    }
+
+    // Bogus command
+
+    try {
+      DrillOnYarn.run(new String[] { "bogus" });
+      fail();
+    } catch (SimulatedExitException e) {
+      assert (e.exitCode == -1);
+      assertTrue(TestContext.getOut().contains("Usage: "));
+    }
+
+    // Help command
+
+    try {
+      DrillOnYarn.run(new String[] { "help" });
+      fail();
+    } catch (SimulatedExitException e) {
+      assert (e.exitCode == -1);
+      assertTrue(TestContext.getOut().contains("Usage: "));
+    }
+  }
+
+  // The idea here is to set up a simulated client environment, then
+  // test each command. This is a big project.
+
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/test/java/org/apache/drill/yarn/client/TestCommandLineOptions.java
----------------------------------------------------------------------
diff --git a/drill-yarn/src/test/java/org/apache/drill/yarn/client/TestCommandLineOptions.java b/drill-yarn/src/test/java/org/apache/drill/yarn/client/TestCommandLineOptions.java
new file mode 100644
index 0000000..f996c5b
--- /dev/null
+++ b/drill-yarn/src/test/java/org/apache/drill/yarn/client/TestCommandLineOptions.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.yarn.client;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+import org.apache.drill.yarn.client.CommandLineOptions;
+
+public class TestCommandLineOptions {
+  @Test
+  public void testOptions() {
+    CommandLineOptions opts = new CommandLineOptions();
+    opts.parse(new String[] {});
+    assertNull(opts.getCommand());
+
+    opts = new CommandLineOptions();
+    opts.parse(new String[] { "-h" });
+    assertEquals(CommandLineOptions.Command.HELP, opts.getCommand());
+
+    opts = new CommandLineOptions();
+    opts.parse(new String[] { "-?" });
+    assertEquals(CommandLineOptions.Command.HELP, opts.getCommand());
+
+    opts = new CommandLineOptions();
+    opts.parse(new String[] { "help" });
+    assertEquals(CommandLineOptions.Command.HELP, opts.getCommand());
+
+    opts = new CommandLineOptions();
+    opts.parse(new String[] { "start" });
+    assertEquals(CommandLineOptions.Command.START, opts.getCommand());
+
+    opts = new CommandLineOptions();
+    opts.parse(new String[] { "stop" });
+    assertEquals(CommandLineOptions.Command.STOP, opts.getCommand());
+
+    opts = new CommandLineOptions();
+    opts.parse(new String[] { "status" });
+    assertEquals(CommandLineOptions.Command.STATUS, opts.getCommand());
+
+    opts = new CommandLineOptions();
+    opts.parse(new String[] { "resize" });
+    assertNull(opts.getCommand());
+
+    opts = new CommandLineOptions();
+    opts.parse(new String[] { "resize", "10" });
+    assertEquals(CommandLineOptions.Command.RESIZE, opts.getCommand());
+    assertEquals("", opts.getResizePrefix());
+    assertEquals(10, opts.getResizeValue());
+
+    opts = new CommandLineOptions();
+    opts.parse(new String[] { "resize", "+2" });
+    assertEquals(CommandLineOptions.Command.RESIZE, opts.getCommand());
+    assertEquals("+", opts.getResizePrefix());
+    assertEquals(2, opts.getResizeValue());
+
+    opts = new CommandLineOptions();
+    opts.parse(new String[] { "resize", "-3" });
+    assertEquals(CommandLineOptions.Command.RESIZE, opts.getCommand());
+    assertEquals("-", opts.getResizePrefix());
+    assertEquals(3, opts.getResizeValue());
+
+    opts = new CommandLineOptions();
+    opts.parse(new String[] { "myDrill" });
+    assertNull(opts.getCommand());
+  }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/test/java/org/apache/drill/yarn/core/TestConfig.java
----------------------------------------------------------------------
diff --git a/drill-yarn/src/test/java/org/apache/drill/yarn/core/TestConfig.java b/drill-yarn/src/test/java/org/apache/drill/yarn/core/TestConfig.java
new file mode 100644
index 0000000..0519484
--- /dev/null
+++ b/drill-yarn/src/test/java/org/apache/drill/yarn/core/TestConfig.java
@@ -0,0 +1,267 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.yarn.core;
+
+import static org.junit.Assert.*;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.Test;
+
+import com.typesafe.config.Config;
+
+public class TestConfig {
+
+  /**
+   * Mock config that lets us tinker with loading and environment access for
+   * testing.
+   */
+
+  private static class DoYTestConfig extends DrillOnYarnConfig {
+    protected Map<String, String> mockEnv = new HashMap<>();
+    protected File configDir;
+
+    public DoYTestConfig(TestClassLoader cl, File configDir)
+        throws DoyConfigException {
+      doLoad(cl);
+      instance = this;
+      this.configDir = configDir;
+    }
+
+    @Override
+    protected String getEnv(String key) {
+      return mockEnv.get(key);
+    }
+  }
+
+  /**
+   * Mock class loader to let us add config files after the JVM starts. (In
+   * production code, the config file directories are added to the class path.)
+   */
+
+  private static class TestClassLoader extends ClassLoader {
+    private File configDir;
+
+    public TestClassLoader(ClassLoader parent, File configDir) {
+      super(parent);
+      this.configDir = configDir;
+    }
+
+    @Override
+    protected URL findResource(String name) {
+      File file = new File(configDir, name);
+      if (file.exists()) {
+        try {
+          return file.toURI().toURL();
+        } catch (MalformedURLException e) {
+          ;
+        }
+      }
+      return null;
+    }
+  }
+
+  /**
+   * Creates a stack of settings to test overrides.
+   * <table>
+   * <tr>
+   * <th>property</th>
+   * <th>default</th>
+   * <th>distrib</th>
+   * <th>user</th>
+   * <th>system</th>
+   * </tr>
+   * <tr>
+   * <td>drill-key</td>
+   * <td>"drill"</td>
+   * <td></td>
+   * <td></td>
+   * <td></td>
+   * </tr>
+   * <tr>
+   * <td>app-dir</td>
+   * <td>"/user/drill"</td>
+   * <td>"/opt/drill"</td>
+   * <td></td>
+   * <td></td>
+   * </tr>
+   * <tr>
+   * <td>app-name</td>
+   * <td>"Drill-on-YARN"</td>
+   * <td>"config-app-name"</td>
+   * <td>"My-App"</td>
+   * <td></td>
+   * </tr>
+   * <tr>
+   * <td>queue</td>
+   * <td>"default"</td>
+   * <td>"distrib-queue"</td>
+   * <td>"my-queue"</td>
+   * <td>"sys-queue"</td>
+   * </table>
+   * <p>
+   * Full property names:
+   * <ul>
+   * <li>drill.yarn.drill-install.drill-key</li>
+   * <li>drill.yarn.dfs.app-dir</li>
+   * <li>drill.yarn.app-name</li>
+   * <li>drill.yarn.zk.connect</li>
+   * </ul>
+   *
+   * @throws IOException
+   * @throws DoyConfigException
+   */
+  @Test
+  public void testLoad() throws IOException, DoyConfigException {
+
+    DoYTestConfig doyConfig = initConfig("test-doy-config.conf");
+    Config config = DrillOnYarnConfig.config();
+
+    assertEquals("drill",
+        config.getString(DrillOnYarnConfig.DRILL_ARCHIVE_KEY));
+    assertEquals("/opt/drill", config.getString(DrillOnYarnConfig.DFS_APP_DIR));
+    assertEquals("My-App", config.getString(DrillOnYarnConfig.APP_NAME));
+    // Commenting out for now, fails on VM.
+    //assertEquals("sys-queue", config.getString(DrillOnYarnConfig.YARN_QUEUE));
+
+    // Should also have access to Drill options.
+    // Does not test Drill's override mechanism because have not found a good
+    // way to add drill-override.conf to the class path in this test.
+
+    // assertEquals( "org.apache.drill.exec.opt.IdentityOptimizer",
+    // config.getString( "drill.exec.optimizer" ) );
+    assertEquals("drillbits1", config.getString(DrillOnYarnConfig.CLUSTER_ID));
+
+    // Drill home: with and without an env var.
+    // Must set the site env var. Class path testing can't be done here.
+    // No DRILL_HOME: will only occur during testing. In that case, we use
+    // the setting from the config file. Explicit site dir.
+
+    assertNull(doyConfig.mockEnv.get(DrillOnYarnConfig.DRILL_HOME_ENV_VAR));
+    doyConfig.mockEnv.put(DrillOnYarnConfig.DRILL_SITE_ENV_VAR, "/drill/site");
+    doyConfig.setClientPaths();
+    assertEquals("/config/drill/home",
+        doyConfig.getLocalDrillHome().getAbsolutePath());
+    assertTrue(doyConfig.hasSiteDir());
+    assertEquals("/drill/site", doyConfig.getLocalSiteDir().getAbsolutePath());
+
+    // Home set in an env var
+
+    doyConfig.mockEnv.put(DrillOnYarnConfig.DRILL_HOME_ENV_VAR, "/drill/home");
+    doyConfig.setClientPaths();
+    assertEquals("/drill/home",
+        doyConfig.getLocalDrillHome().getAbsolutePath());
+
+    // Remote site: localized case
+
+    assertTrue(config.getBoolean(DrillOnYarnConfig.LOCALIZE_DRILL));
+    assertEquals("/foo/bar/drill-archive.tar.gz",
+        config.getString(DrillOnYarnConfig.DRILL_ARCHIVE_PATH));
+    assertEquals("$PWD/drill/drill-archive", doyConfig.getRemoteDrillHome());
+    assertEquals("site", config.getString(DrillOnYarnConfig.SITE_ARCHIVE_KEY));
+    assertEquals("$PWD/site", doyConfig.getRemoteSiteDir());
+
+    // Localized, but no separate site directory
+
+    doyConfig.mockEnv.put(DrillOnYarnConfig.DRILL_SITE_ENV_VAR,
+        "/drill/home/conf");
+    doyConfig.setClientPaths();
+    // If $DRILL_HOME/conf is used, we still treat id as a site dir.
+//    assertFalse(doyConfig.hasSiteDir());
+//    assertNull(doyConfig.getRemoteSiteDir());
+
+    // Local app id file: composed from Drill home, ZK root and cluster id.
+    // (Turns out that there can be two different clusters sharing the same
+    // root...)
+    // With no site dir, app id is in parent of the drill directory.
+
+    assertEquals("/drill/home",
+        doyConfig.getLocalDrillHome().getAbsolutePath());
+    assertEquals("drill", config.getString(DrillOnYarnConfig.ZK_ROOT));
+    assertEquals("drillbits1", config.getString(DrillOnYarnConfig.CLUSTER_ID));
+    assertEquals("/drill/home/drill-drillbits1.appid",
+        doyConfig.getLocalAppIdFile().getAbsolutePath());
+
+    // Again, but with a site directory. App id is in parent of the site
+    // directory.
+
+    doyConfig.mockEnv.put(DrillOnYarnConfig.DRILL_SITE_ENV_VAR,
+        "/var/drill/site");
+    doyConfig.setClientPaths();
+    assertEquals("/var/drill/drill-drillbits1.appid",
+        doyConfig.getLocalAppIdFile().getAbsolutePath());
+  }
+
+  private DoYTestConfig initConfig(String configName)
+      throws IOException, DoyConfigException {
+    File tempDir = new File(System.getProperty("java.io.tmpdir"));
+    File configDir = new File(tempDir, "config");
+    if (configDir.exists()) {
+      FileUtils.forceDelete(configDir);
+    }
+    configDir.mkdirs();
+    configDir.deleteOnExit();
+
+    InputStream in = getClass().getResourceAsStream("/" + configName);
+    File dest = new File(configDir, "drill-on-yarn.conf");
+    Files.copy(in, dest.toPath(), StandardCopyOption.REPLACE_EXISTING);
+    in = getClass().getResourceAsStream("/test-doy-distrib.conf");
+    dest = new File(configDir, "doy-distrib.conf");
+    Files.copy(in, dest.toPath(), StandardCopyOption.REPLACE_EXISTING);
+
+    System.setProperty(DrillOnYarnConfig.YARN_QUEUE, "sys-queue");
+
+    TestClassLoader cl = new TestClassLoader(this.getClass().getClassLoader(),
+        configDir);
+
+    assertNotNull(cl.getResource(DrillOnYarnConfig.DISTRIB_FILE_NAME));
+
+    return new DoYTestConfig(cl, configDir);
+  }
+
+  @Test
+  public void testNonLocalized() throws IOException, DoyConfigException {
+    DoYTestConfig doyConfig = initConfig("second-test-config.conf");
+
+    // Test the non-localized case
+
+    doyConfig.mockEnv.put(DrillOnYarnConfig.DRILL_SITE_ENV_VAR, "/drill/site");
+    doyConfig.setClientPaths();
+    assertEquals("/config/drill/home", doyConfig.getRemoteDrillHome());
+    assertEquals("/config/drill/site", doyConfig.getRemoteSiteDir());
+  }
+
+  @Test
+  public void testNonLocalizedNonSite() throws IOException, DoyConfigException {
+    DoYTestConfig doyConfig = initConfig("third-test-config.conf");
+
+    // Test the non-localized case
+
+    assertEquals("/config/drill/home", doyConfig.getRemoteDrillHome());
+    assertNull(doyConfig.getRemoteSiteDir());
+  }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/test/java/org/apache/drill/yarn/scripts/README.md
----------------------------------------------------------------------
diff --git a/drill-yarn/src/test/java/org/apache/drill/yarn/scripts/README.md b/drill-yarn/src/test/java/org/apache/drill/yarn/scripts/README.md
new file mode 100644
index 0000000..3e4017b
--- /dev/null
+++ b/drill-yarn/src/test/java/org/apache/drill/yarn/scripts/README.md
@@ -0,0 +1,65 @@
+# Script Test Overview
+
+The tests here exercise the Drill shell scripts with a wide variety of options.
+The Drill scripts are designed to be run from the command line or from YARN.
+The scripts allow passing in values from a vendor-specific configuration file
+($DRILL_HOME/conf/distrib-env.sh), a user-specific configuration file
+($DRILL_SITE/drill-env.sh) or from environment variables set by YARN.
+
+Testing scripts is normally tedious because the scripts are designed to start
+a process, perhaps a Drillbit or Sqlline. To make automated tests possible,
+the scripts incorporate a "shim": an environment variable that, if set, is
+used to put a "wrapper" script around the Java execution line. The wrapper
+captures the environment and the command line, and generates stderr and
+stdout output. The test progams use this captured output to determine if
+the Java command line has the options we expect. (We boldly assume that
+if we give Java the right options, it will do the right thing with them.)
+
+Why are the script tests in the drill-yarn project? Because YARN is the most
+sensitive to any changes: YARN provides several levels of indirection between
+the user and the scripts; the scripts must work exactly as the Drill-on-YARN
+code expects (or visa-versa) or things tend to break.
+
+## Inputs
+
+The test program needs the following inputs:
+
+- The scripts, which are copied from the source tree. (Need details)
+- /src/test/resources/wrapper.sh which is the "magic" wrapper script
+for capturing the command line, etc.
+- A temporary directory where the program can build its mock Drill
+and site directories.
+
+## Running the Tests
+
+Simply run the tests. Each test sets up its distribution and
+optional site directory and required environment variables. Each
+test uses a builder to build up the required script launch and
+to analyze the results. Each test function usually does a single
+setup, then does a bunch of test runs against that environment.
+
+Each test uses "gobbler" threads to read the stdout and stderr
+from the test run. If you run the test in a debugger, you'll
+see a steady stream of threads come and go. This is a test, so
+we don't bother with a thread pool; we just brute-force create
+new threads as needed.
+
+## Extending the Tests
+
+You should extend the tests if you:
+
+- Add new environment variables to any script.
+- Change the logic of any script.
+- Find a bug that these tests somehow did not catch.
+
+Note that it is very important to ensure that each new enviornment
+variable or other option works when set in distrib-env.sh,
+drill-env.sh or in the environment. These tests are the only (sane)
+way to test the many combinations, and to do that on each
+subsequent change.
+
+## About the Code
+
+TestScripts.java are the tests, organized by functional area. ScriptUtils.java
+is a large number of (mostly ad-hoc) utilities needed to set up, run, tear down
+and analyze the results of each run.
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/test/java/org/apache/drill/yarn/scripts/ScriptUtils.java
----------------------------------------------------------------------
diff --git a/drill-yarn/src/test/java/org/apache/drill/yarn/scripts/ScriptUtils.java b/drill-yarn/src/test/java/org/apache/drill/yarn/scripts/ScriptUtils.java
new file mode 100644
index 0000000..3517cf8
--- /dev/null
+++ b/drill-yarn/src/test/java/org/apache/drill/yarn/scripts/ScriptUtils.java
@@ -0,0 +1,847 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.yarn.scripts;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.PrintWriter;
+import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Pattern;
+
+import org.apache.commons.io.FileUtils;
+
+public class ScriptUtils {
+
+  private static ScriptUtils instance = new ScriptUtils();
+  public File distribDir;
+  public File javaHome;
+  public File testDir;
+  public File testDrillHome;
+  public File testSiteDir;
+  public File testLogDir;
+  public boolean externalLogDir;
+
+  /**
+   * Out-of-the-box command-line arguments when launching sqlline.
+   * Order is not important here (though it is to Java.)
+   */
+
+  public static String sqlLineArgs[] = makeSqlLineArgs( );
+
+  private static String[] makeSqlLineArgs( ) {
+    String args[] = {
+      "-Dlog.path=/.*/drill/log/sqlline\\.log",
+      "-Dlog.query.path=/.*/drill/log/sqlline_queries\\.json",
+      "-XX:MaxPermSize=512M",
+      "sqlline\\.SqlLine",
+      "-d",
+      "org\\.apache\\.drill\\.jdbc\\.Driver",
+      "--maxWidth=10000",
+      "--color=true"
+    };
+
+    // Special handling if this machine happens to have the default
+    // /var/log/drill log location.
+
+    if ( new File( "/var/log/drill" ).exists() ) {
+      args[ 0 ] = "-Dlog\\.path=/var/log/drill/sqlline\\.log";
+      args[ 1 ] = "-Dlog\\.query\\.path=/var/log/drill/sqlline_queries\\.json";
+    }
+    return args;
+  }
+
+  public static final boolean USE_SOURCE = true;
+  public static final String TEMP_DIR = "/tmp";
+  public static boolean useSource = USE_SOURCE;
+
+  private ScriptUtils() {
+    String drillScriptsDir = System.getProperty("drillScriptDir");
+    assertNotNull(drillScriptsDir);
+    distribDir = new File(drillScriptsDir);
+    javaHome = new File(System.getProperty("java.home"));
+  }
+
+  public static ScriptUtils instance() {
+    return instance;
+  }
+
+  public ScriptUtils fromSource(String sourceDir) {
+    useSource = true;
+    return this;
+  }
+
+  public ScriptUtils fromDistrib(String distrib) {
+    distribDir = new File(distrib);
+    useSource = false;
+    return this;
+  }
+
+  /**
+   * Out-of-the-box command-line arguments when launching Drill.
+   * Order is not important here (though it is to Java.)
+   */
+
+  public static String stdArgs[] = buildStdArgs( );
+
+  private static String[] buildStdArgs( )
+  {
+    String args[] = {
+      "-Xms4G",
+      "-Xmx4G",
+      "-XX:MaxDirectMemorySize=8G",
+      "-XX:MaxPermSize=512M",
+      "-XX:ReservedCodeCacheSize=1G",
+      // Removed in Drill 1.8
+//      "-Ddrill\\.exec\\.enable-epoll=true",
+      "-XX:\\+CMSClassUnloadingEnabled",
+      "-XX:\\+UseG1GC",
+      "org\\.apache\\.drill\\.exec\\.server\\.Drillbit",
+      "-Dlog\\.path=/.*/script-test/drill/log/drillbit\\.log",
+      "-Dlog\\.query\\.path=/.*/script-test/drill/log/drillbit_queries\\.json",
+    };
+
+    // Special handling if this machine happens to have the default
+    // /var/log/drill log location.
+
+    if ( new File( "/var/log/drill" ).exists() ) {
+      args[ args.length-2 ] = "-Dlog\\.path=/var/log/drill/drillbit\\.log";
+      args[ args.length-1 ] = "-Dlog\\.query\\.path=/var/log/drill/drillbit_queries\\.json";
+    }
+    return args;
+  };
+
+  /**
+   * Out-of-the-box class-path before any custom additions.
+   */
+
+  static String stdCp[] =
+  {
+    "conf",
+    "jars/*",
+    "jars/ext/*",
+    "jars/3rdparty/*",
+    "jars/classb/*"
+  };
+
+  /**
+   * Directories to create to simulate a Drill distribution.
+   */
+
+  static String distribDirs[] = {
+      "bin",
+      "jars",
+      "jars/3rdparty",
+      "jars/ext",
+      "conf"
+  };
+
+  /**
+   * Out-of-the-box Jar directories.
+   */
+
+  static String jarDirs[] = {
+      "jars",
+      "jars/3rdparty",
+      "jars/ext",
+  };
+
+  /**
+   * Scripts we must copy from the source tree to create a simulated
+   * Drill bin directory.
+   */
+
+  public static String scripts[] = {
+      "drill-config.sh",
+      "drill-embedded",
+      "drill-localhost",
+      "drill-on-yarn.sh",
+      "drillbit.sh",
+      "drill-conf",
+      //dumpcat
+      //hadoop-excludes.txt
+      "runbit",
+      "sqlline",
+      //sqlline.bat
+      //submit_plan
+      "yarn-drillbit.sh"
+  };
+
+  /**
+   * Create the basic test directory. Tests add or remove details.
+   */
+
+  public void initialSetup() throws IOException {
+    File tempDir = new File(TEMP_DIR);
+    testDir = new File(tempDir, "script-test");
+    testDrillHome = new File(testDir, "drill");
+    testSiteDir = new File(testDir, "site");
+    File varLogDrill = new File( "/var/log/drill" );
+    if ( varLogDrill.exists() ) {
+      testLogDir = varLogDrill;
+      externalLogDir = true;
+    } else {
+      testLogDir = new File(testDrillHome, "log");
+    }
+    if (testDir.exists()) {
+      FileUtils.forceDelete(testDir);
+    }
+    testDir.mkdirs();
+    testSiteDir.mkdir();
+    testLogDir.mkdir();
+  }
+
+  public void createMockDistrib() throws IOException {
+    if (ScriptUtils.useSource) {
+      buildFromSource();
+    } else {
+      buildFromDistrib();
+    }
+  }
+
+  /**
+   * Build the Drill distribution directory directly from sources.
+   */
+
+  private void buildFromSource() throws IOException {
+    createMockDirs();
+    copyScripts(ScriptUtils.instance().distribDir);
+  }
+
+  /**
+   * Build the shell of a Drill distribution directory by creating the required
+   * directory structure.
+   */
+
+  private void createMockDirs() throws IOException {
+    if (testDrillHome.exists()) {
+      FileUtils.forceDelete(testDrillHome);
+    }
+    testDrillHome.mkdir();
+    for (String path : ScriptUtils.distribDirs) {
+      File subDir = new File(testDrillHome, path);
+      subDir.mkdirs();
+    }
+    for (String path : ScriptUtils.jarDirs) {
+      makeDummyJar(new File(testDrillHome, path), "dist");
+    }
+  }
+
+  /**
+   * The tests should not require jar files, but we simulate them to be a bit
+   * more realistic. Since we dont' run Java, the jar files can be simulated.
+   */
+
+  public File makeDummyJar(File dir, String prefix) throws IOException {
+    String jarName = "";
+    if (prefix != null) {
+      jarName += prefix + "-";
+    }
+    jarName += dir.getName() + ".jar";
+    File jarFile = new File(dir, jarName);
+    writeFile(jarFile, "Dummy jar");
+    return jarFile;
+  }
+
+  /**
+   * Create a simple text file with the given contents.
+   */
+
+  public void writeFile(File file, String contents) throws IOException {
+    try (PrintWriter out = new PrintWriter(new FileWriter(file))) {
+      out.println(contents);
+    }
+  }
+
+  /**
+   * Create a drill-env.sh or distrib-env.sh file with the given environment in
+   * the recommended format.
+   */
+
+  public void createEnvFile(File file, Map<String, String> env)
+      throws IOException {
+    try (PrintWriter out = new PrintWriter(new FileWriter(file))) {
+      out.println("#!/usr/bin/env bash");
+      for (String key : env.keySet()) {
+        String value = env.get(key);
+        out.print("export ");
+        out.print(key);
+        out.print("=${");
+        out.print(key);
+        out.print(":-\"");
+        out.print(value);
+        out.println("\"}");
+      }
+    }
+  }
+
+  /**
+   * Copy the standard scripts from source location to the mock distribution
+   * directory.
+   */
+
+  private void copyScripts(File sourceDir) throws IOException {
+    File binDir = new File(testDrillHome, "bin");
+    for (String script : ScriptUtils.scripts) {
+      File source = new File(sourceDir, script);
+      File dest = new File(binDir, script);
+      copyFile(source, dest);
+      dest.setExecutable(true);
+    }
+
+    // Create the "magic" wrapper script that simulates the Drillbit and
+    // captures the output we need for testing.
+
+    String wrapper = "wrapper.sh";
+    File dest = new File(binDir, wrapper);
+    try (InputStream is = getClass().getResourceAsStream("/" + wrapper)) {
+      Files.copy(is, dest.toPath(), StandardCopyOption.REPLACE_EXISTING);
+    }
+    dest.setExecutable(true);
+  }
+
+  private void buildFromDistrib() {
+    // TODO Auto-generated method stub
+
+  }
+
+  /**
+   * Consume the input from a stream, specifically the stderr or stdout stream
+   * from a process.
+   *
+   * @see http://stackoverflow.com/questions/14165517/processbuilder-forwarding-stdout-and-stderr-of-started-processes-without-blocki
+   */
+
+  private static class StreamGobbler extends Thread {
+    InputStream is;
+    public StringBuilder buf = new StringBuilder();
+
+    private StreamGobbler(InputStream is) {
+      this.is = is;
+    }
+
+    @Override
+    public void run() {
+      try {
+        InputStreamReader isr = new InputStreamReader(is);
+        BufferedReader br = new BufferedReader(isr);
+        String line = null;
+        while ((line = br.readLine()) != null) {
+          buf.append(line);
+          buf.append("\n");
+        }
+      } catch (IOException ioe) {
+        ioe.printStackTrace();
+      }
+    }
+  }
+
+  /**
+   * Handy run result class to capture the information we need for testing and
+   * to do various kinds of validation on it.
+   */
+
+  public static class RunResult {
+    File logDir;
+    File logFile;
+    String stdout;
+    String stderr;
+    List<String> echoArgs;
+    int returnCode;
+    String classPath[];
+    String libPath[];
+    String log;
+    public File pidFile;
+    public File outFile;
+    String out;
+
+    /**
+     * Split the class path into strings for easier validation.
+     */
+
+    public void analyze() {
+      if (echoArgs == null) {
+        return;
+      }
+      for (int i = 0; i < echoArgs.size(); i++) {
+        String arg = echoArgs.get(i);
+        if (arg.equals("-cp")) {
+          classPath = Pattern.compile(":").split((echoArgs.get(i + 1)));
+          break;
+        }
+      }
+      String probe = "-Djava.library.path=";
+      for (int i = 0; i < echoArgs.size(); i++) {
+        String arg = echoArgs.get(i);
+        if (arg.startsWith(probe)) {
+          assertNull(libPath);
+          libPath = Pattern.compile(":").split((arg.substring(probe.length())));
+          break;
+        }
+      }
+    }
+
+    /**
+     * Read the log file, if any, generated by the process.
+     */
+
+    public void loadLog() throws IOException {
+      log = loadFile(logFile);
+    }
+
+    private String loadFile(File file) throws IOException {
+      StringBuilder buf = new StringBuilder();
+      try ( BufferedReader reader = new BufferedReader(new FileReader(file)) ) {
+        String line;
+        while ((line = reader.readLine()) != null) {
+          buf.append(line);
+          buf.append("\n");
+        }
+        return buf.toString();
+      } catch (FileNotFoundException e) {
+        return null;
+      }
+    }
+
+    /**
+     * Validate that the first argument invokes Java correctly.
+     */
+
+    public void validateJava() {
+      assertNotNull(echoArgs);
+      String java = instance.javaHome + "/bin/java";
+      List<String> actual = echoArgs;
+      assertEquals(java, actual.get(0));
+    }
+
+    public boolean containsArg(String arg) {
+      for (String actual : echoArgs) {
+        if (actual.equals(arg)) {
+          return true;
+        }
+      }
+      return false;
+    }
+
+    public void validateStockArgs() {
+      for (String arg : ScriptUtils.stdArgs) {
+        assertTrue("Argument not found: " + arg + " in " + echoArgs, containsArgRegex(arg));
+      }
+    }
+
+    public void validateArg(String arg) {
+      validateArgs(Collections.singletonList(arg));
+    }
+
+    public void validateArgs(String args[]) {
+      validateArgs(Arrays.asList(args));
+    }
+
+    public void validateArgs(List<String> args) {
+      validateJava();
+      for (String arg : args) {
+        assertTrue(containsArg(arg));
+      }
+    }
+
+    public void validateArgRegex(String arg) {
+      assertTrue(containsArgRegex(arg));
+    }
+
+    public void validateArgsRegex(List<String> args) {
+      assertTrue(containsArgsRegex(args));
+    }
+
+    public boolean containsArgsRegex(List<String> args) {
+      for (String arg : args) {
+        if (!containsArgRegex(arg)) {
+          return false;
+        }
+      }
+      return true;
+    }
+
+    public boolean containsArgsRegex(String args[]) {
+      for (String arg : args) {
+        if (!containsArgRegex(arg)) {
+          return false;
+        }
+      }
+      return true;
+    }
+
+    public boolean containsArgRegex(String arg) {
+      for (String actual : echoArgs) {
+        if (actual.matches(arg)) {
+          return true;
+        }
+      }
+      return false;
+    }
+
+    public void validateClassPath(String expectedCP) {
+      assertTrue(classPathContains(expectedCP));
+    }
+
+    public void validateClassPath(String expectedCP[]) {
+      assertTrue(classPathContains(expectedCP));
+    }
+
+    public boolean classPathContains(String expectedCP[]) {
+      for (String entry : expectedCP) {
+        if (!classPathContains(entry)) {
+          return false;
+        }
+      }
+      return true;
+    }
+
+    public boolean classPathContains(String expectedCP) {
+      if (classPath == null) {
+        fail("No classpath returned");
+      }
+      String tail = "/" + instance.testDir.getName() + "/"
+          + instance.testDrillHome.getName() + "/";
+      String expectedPath;
+      if (expectedCP.startsWith("/")) {
+        expectedPath = expectedCP;
+      } else {
+        expectedPath = tail + expectedCP;
+      }
+      for (String entry : classPath) {
+        if (entry.endsWith(expectedPath)) {
+          return true;
+        }
+      }
+      return false;
+    }
+
+    public void loadOut() throws IOException {
+      out = loadFile(outFile);
+    }
+
+    /**
+     * Ensure that the Drill log file contains at least the sample message
+     * written by the wrapper.
+     */
+
+    public void validateDrillLog() {
+      assertNotNull(log);
+      assertTrue(log.contains("Drill Log Message"));
+    }
+
+    /**
+     * Validate that the stdout contained the expected message.
+     */
+
+    public void validateStdOut() {
+      assertTrue(stdout.contains("Starting drillbit on"));
+    }
+
+    /**
+     * Validate that the stderr contained the sample error message from the
+     * wrapper.
+     */
+
+    public void validateStdErr() {
+      assertTrue(stderr.contains("Stderr Message"));
+    }
+
+    public int getPid() throws IOException {
+      try (BufferedReader reader = new BufferedReader(new FileReader(pidFile))) {
+        return Integer.parseInt(reader.readLine());
+      }
+      finally { }
+    }
+
+  }
+
+  /**
+   * The "business end" of the tests: runs drillbit.sh and captures results.
+   */
+
+  public static class ScriptRunner {
+    // Drillbit commands
+
+    public static String DRILLBIT_RUN = "run";
+    public static String DRILLBIT_START = "start";
+    public static String DRILLBIT_STATUS = "status";
+    public static String DRILLBIT_STOP = "stop";
+    public static String DRILLBIT_RESTART = "restart";
+
+    public File cwd = instance.testDir;
+    public File drillHome = instance.testDrillHome;
+    public String script;
+    public List<String> args = new ArrayList<>();
+    public Map<String, String> env = new HashMap<>();
+    public File logDir;
+    public File pidFile;
+    public File outputFile;
+    public boolean preserveLogs;
+
+    public ScriptRunner(String script) {
+      this.script = script;
+    }
+
+    public ScriptRunner(String script, String cmd) {
+      this(script);
+      args.add(cmd);
+    }
+
+    public ScriptRunner(String script, String cmdArgs[]) {
+      this(script);
+      for (String arg : cmdArgs) {
+        args.add(arg);
+      }
+    }
+
+    public ScriptRunner withArg(String arg) {
+      args.add(arg);
+      return this;
+    }
+
+    public ScriptRunner withSite(File siteDir) {
+      if (siteDir != null) {
+        args.add("--site");
+        args.add(siteDir.getAbsolutePath());
+      }
+      return this;
+    }
+
+    public ScriptRunner withEnvironment(Map<String, String> env) {
+      if (env != null) {
+        this.env.putAll(env);
+      }
+      return this;
+    }
+
+    public ScriptRunner addEnv(String key, String value) {
+      env.put(key, value);
+      return this;
+    }
+
+    public ScriptRunner withLogDir(File logDir) {
+      this.logDir = logDir;
+      return this;
+    }
+
+    public ScriptRunner preserveLogs() {
+      preserveLogs = true;
+      return this;
+    }
+
+    public RunResult run() throws IOException {
+      File binDir = new File(drillHome, "bin");
+      File scriptFile = new File(binDir, script);
+      assertTrue(scriptFile.exists());
+      outputFile = new File(instance.testDir, "output.txt");
+      outputFile.delete();
+      if (logDir == null) {
+        logDir = instance.testLogDir;
+      }
+      if (!preserveLogs) {
+        cleanLogs(logDir);
+      }
+
+      Process proc = startProcess(scriptFile);
+      RunResult result = runProcess(proc);
+      if (result.returnCode == 0) {
+        captureOutput(result);
+        captureLog(result);
+      }
+      return result;
+    }
+
+    private void cleanLogs(File logDir) throws IOException {
+      if ( logDir == instance.testLogDir  &&  instance.externalLogDir ) {
+        return;
+      }
+      if (logDir.exists()) {
+        FileUtils.forceDelete(logDir);
+      }
+    }
+
+    private Process startProcess(File scriptFile) throws IOException {
+      outputFile.delete();
+      List<String> cmd = new ArrayList<>();
+      cmd.add(scriptFile.getAbsolutePath());
+      cmd.addAll(args);
+      ProcessBuilder pb = new ProcessBuilder().command(cmd).directory(cwd);
+      Map<String, String> pbEnv = pb.environment();
+      pbEnv.clear();
+      pbEnv.putAll(env);
+      File binDir = new File(drillHome, "bin");
+      File wrapperCmd = new File(binDir, "wrapper.sh");
+
+      // Set the magic wrapper to capture output.
+
+      pbEnv.put("_DRILL_WRAPPER_", wrapperCmd.getAbsolutePath());
+      pbEnv.put("JAVA_HOME", instance.javaHome.getAbsolutePath());
+      return pb.start();
+    }
+
+    private RunResult runProcess(Process proc) {
+      StreamGobbler errorGobbler = new StreamGobbler(proc.getErrorStream());
+      StreamGobbler outputGobbler = new StreamGobbler(proc.getInputStream());
+      outputGobbler.start();
+      errorGobbler.start();
+
+      try {
+        proc.waitFor();
+      } catch (InterruptedException e) {
+        // Won't occur.
+      }
+
+      RunResult result = new RunResult();
+      result.stderr = errorGobbler.buf.toString();
+      result.stdout = outputGobbler.buf.toString();
+      result.returnCode = proc.exitValue();
+      return result;
+    }
+
+    private void captureOutput(RunResult result) throws IOException {
+      // Capture the Java arguments which the wrapper script wrote to a file.
+
+      try (BufferedReader reader = new BufferedReader(new FileReader(outputFile))) {
+         result.echoArgs = new ArrayList<>();
+        String line;
+        while ((line = reader.readLine()) != null) {
+          result.echoArgs.add(line);
+        }
+        result.analyze();
+      } catch (FileNotFoundException e) {
+        ;
+      }
+    }
+
+    private void captureLog(RunResult result) throws IOException {
+      result.logDir = logDir;
+      result.logFile = new File(logDir, "drillbit.log");
+      if (result.logFile.exists()) {
+        result.loadLog();
+      } else {
+        result.logFile = null;
+      }
+    }
+  }
+
+  public static class DrillbitRun extends ScriptRunner {
+    public File pidDir;
+
+    public DrillbitRun() {
+      super("drillbit.sh");
+    }
+
+    public DrillbitRun(String cmd) {
+      super("drillbit.sh", cmd);
+    }
+
+    public DrillbitRun withPidDir(File pidDir) {
+      this.pidDir = pidDir;
+      return this;
+    }
+
+    public DrillbitRun asDaemon() {
+      addEnv("KEEP_RUNNING", "1");
+      return this;
+    }
+
+    public RunResult start() throws IOException {
+      if (pidDir == null) {
+        pidDir = drillHome;
+      }
+      pidFile = new File(pidDir, "drillbit.pid");
+      // pidFile.delete();
+      asDaemon();
+      RunResult result = run();
+      if (result.returnCode == 0) {
+        capturePidFile(result);
+        captureDrillOut(result);
+      }
+      return result;
+    }
+
+    private void capturePidFile(RunResult result) {
+      assertTrue(pidFile.exists());
+      result.pidFile = pidFile;
+    }
+
+    private void captureDrillOut(RunResult result) throws IOException {
+      // Drillbit.out
+
+      result.outFile = new File(result.logDir, "drillbit.out");
+      if (result.outFile.exists()) {
+        result.loadOut();
+      } else {
+        result.outFile = null;
+      }
+    }
+
+  }
+
+  /**
+   * Build a "starter" conf or site directory by creating a mock
+   * drill-override.conf file.
+   */
+
+  public void createMockConf(File siteDir) throws IOException {
+    createDir(siteDir);
+    File override = new File(siteDir, "drill-override.conf");
+    writeFile(override, "# Dummy override");
+  }
+
+  public void removeDir(File dir) throws IOException {
+    if (dir.exists()) {
+      FileUtils.forceDelete(dir);
+    }
+  }
+
+  /**
+   * Remove, then create a directory.
+   */
+
+  public File createDir(File dir) throws IOException {
+    removeDir(dir);
+    dir.mkdirs();
+    return dir;
+  }
+
+  public void copyFile(File source, File dest) throws IOException {
+    Files.copy(source.toPath(), dest.toPath(),
+        StandardCopyOption.REPLACE_EXISTING);
+  }
+
+}


Mime
View raw message