drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ar...@apache.org
Subject [05/12] drill git commit: DRILL-1170: YARN integration for Drill
Date Sun, 04 Mar 2018 17:13:45 GMT
http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/java/org/apache/drill/yarn/client/AMRunner.java
----------------------------------------------------------------------
diff --git a/drill-yarn/src/main/java/org/apache/drill/yarn/client/AMRunner.java b/drill-yarn/src/main/java/org/apache/drill/yarn/client/AMRunner.java
new file mode 100644
index 0000000..5252b88
--- /dev/null
+++ b/drill-yarn/src/main/java/org/apache/drill/yarn/client/AMRunner.java
@@ -0,0 +1,368 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.yarn.client;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.io.PrintWriter;
+import java.util.Map;
+
+import org.apache.drill.yarn.core.AppSpec;
+import org.apache.drill.yarn.core.DoYUtil;
+import org.apache.drill.yarn.core.DrillOnYarnConfig;
+import org.apache.drill.yarn.core.LaunchSpec;
+import org.apache.drill.yarn.core.YarnClientException;
+import org.apache.drill.yarn.core.YarnRMClient;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+
+import com.typesafe.config.Config;
+
+/**
+ * Launch the AM through YARN. Builds the launch description, then tracks
+ * the launch operation itself. Finally, provides the user with links to
+ * track the AM both through YARN and via the AM's own web UI.
+ */
+
+public class AMRunner {
+  private Config config;
+  private boolean verbose;
+  private ApplicationId appId;
+  public Map<String, LocalResource> resources;
+  public String drillArchivePath;
+  public String siteArchivePath;
+  public String remoteDrillHome;
+  public String remoteSiteDir;
+  private YarnRMClient client;
+  private GetNewApplicationResponse appResponse;
+  private boolean dryRun;
+
+  public AMRunner(Config config, boolean verbose, boolean dryRun) {
+    this.config = config;
+    this.verbose = verbose;
+    this.dryRun = dryRun;
+  }
+
+  public void run() throws ClientException {
+    connectToYarn();
+    if (dryRun) {
+      doDryRun();
+    } else {
+      doLaunch();
+    }
+  }
+
+  private void connectToYarn() {
+    System.out.print("Loading YARN Config...");
+    client = new YarnRMClient();
+    System.out.println(" Loaded.");
+  }
+
+  private void doDryRun() throws ClientException {
+    AppSpec master = buildSpec();
+    dump(master, System.out);
+  }
+
+  private void doLaunch() throws ClientException {
+    createApp();
+    AppSpec master = buildSpec();
+    if (verbose) {
+      dump(master, System.out);
+    }
+    validateResources(master);
+    launchApp(master);
+    waitForStartAndReport(master.appName);
+    writeAppIdFile();
+  }
+
+  private void dump(AppSpec master, PrintStream out) {
+    out.println("----------------------------------------------");
+    out.println("Application Master Launch Spec");
+    master.dump(out);
+    out.println("----------------------------------------------");
+  }
+
+  private AppSpec buildSpec() throws ClientException {
+    AppSpec master = new AppSpec();
+
+    // Heap memory
+
+    String heapMem = config.getString( DrillOnYarnConfig.AM_HEAP );
+    master.env.put( DrillOnYarnConfig.AM_HEAP_ENV_VAR, heapMem );
+
+    // Any additional VM arguments from the config file.
+
+    addIfSet( master, DrillOnYarnConfig.AM_VM_ARGS, DrillOnYarnConfig.AM_JAVA_OPTS_ENV_VAR );
+
+    // Any user specified override jars
+    // Not really needed by the AM.
+
+    addIfSet( master, DrillOnYarnConfig.AM_PREFIX_CLASSPATH, DrillOnYarnConfig.DRILL_CLASSPATH_PREFIX_ENV_VAR );
+
+    // Any user specified classpath.
+
+    addIfSet( master, DrillOnYarnConfig.AM_CLASSPATH, DrillOnYarnConfig.DRILL_CLASSPATH_ENV_VAR );
+
+    // Any user-specified library path
+
+    addIfSet( master, DrillOnYarnConfig.JAVA_LIB_PATH, DrillOnYarnConfig.DOY_LIBPATH_ENV_VAR );
+
+    // AM logs (of which there are none.
+    // Relies on the LOG_DIR_EXPANSION_VAR marker which is replaced by
+    // the container log directory.
+    // Must be set for the AM to prevent drill-config.sh from trying to create
+    // the log directory in $DRILL_HOME (which won't be writable under YARN.)
+
+    if (!config.getBoolean(DrillOnYarnConfig.DISABLE_YARN_LOGS)) {
+      master.env.put("DRILL_YARN_LOG_DIR",
+          ApplicationConstants.LOG_DIR_EXPANSION_VAR);
+    }
+
+    // AM launch script
+    // The drill home location is either a non-localized location,
+    // or, more typically, the expanded Drill directory under the
+    // container's working directory. When the localized directory,
+    // we rely on the fact that the current working directory is
+    // set to the container directory, so we just need the name
+    // of the Drill folder under the cwd.
+
+    master.command = remoteDrillHome + "/bin/drill-am.sh";
+
+    // If site dir, add that as an argument.
+
+    if ( remoteSiteDir != null ) {
+      master.cmdArgs.add( "--site" );
+      master.cmdArgs.add( remoteSiteDir );
+    }
+
+    // Strangely, YARN has no way to tell an AM what its app ID
+    // is. So, we pass it along here.
+
+    String appIdStr = dryRun ? "Unknown" : appId.toString();
+    master.env.put( DrillOnYarnConfig.APP_ID_ENV_VAR, appIdStr );
+
+    // Debug launch: dumps environment variables and other information
+    // in the launch script.
+
+    if ( config.getBoolean( DrillOnYarnConfig.AM_DEBUG_LAUNCH ) ) {
+      master.env.put( DrillOnYarnConfig.DRILL_DEBUG_ENV_VAR, "1" );
+    }
+
+    // If localized, add the drill and optionally site archive.
+
+    if ( config.getBoolean( DrillOnYarnConfig.LOCALIZE_DRILL) ) {
+
+      // Also, YARN has no way to tell an AM what localized resources are
+      // available, so we pass them along as environment variables.
+
+      master.env.put( DrillOnYarnConfig.DRILL_ARCHIVE_ENV_VAR, drillArchivePath );
+      if ( siteArchivePath != null ) {
+        master.env.put( DrillOnYarnConfig.SITE_ARCHIVE_ENV_VAR, siteArchivePath );
+      }
+    }
+
+    // Localized resources
+
+    master.resources.putAll( resources );
+
+    // Container specification.
+
+    master.memoryMb = config.getInt( DrillOnYarnConfig.AM_MEMORY );
+    master.vCores = config.getInt( DrillOnYarnConfig.AM_VCORES );
+    master.disks = config.getDouble( DrillOnYarnConfig.AM_DISKS );
+    master.appName = config.getString( DrillOnYarnConfig.APP_NAME );
+    master.queueName = config.getString( DrillOnYarnConfig.YARN_QUEUE );
+    master.priority = config.getInt( DrillOnYarnConfig.YARN_PRIORITY );
+    master.nodeLabelExpr = config.getString( DrillOnYarnConfig.AM_NODE_LABEL_EXPR );
+    return master;
+  }
+
+  private void addIfSet(LaunchSpec spec, String configParam, String envVar) {
+    String value = config.getString(configParam);
+    if (!DoYUtil.isBlank(value)) {
+      spec.env.put(envVar, value);
+    }
+  }
+
+  private void createApp() throws ClientException {
+    try {
+      appResponse = client.createAppMaster();
+    } catch (YarnClientException e) {
+      throw new ClientException("Failed to allocate Drill application master",
+          e);
+    }
+    appId = appResponse.getApplicationId();
+    System.out.println("Application ID: " + appId.toString());
+  }
+
+  private void validateResources( AppSpec master ) throws ClientException {
+
+    // Memory and core checks per YARN app specs.
+
+    int maxMemory = appResponse.getMaximumResourceCapability().getMemory();
+    int maxCores = appResponse.getMaximumResourceCapability().getVirtualCores();
+    if (verbose) {
+      System.out.println("Max Memory: " + maxMemory);
+      System.out.println("Max Cores: " + maxCores);
+    }
+
+    // YARN behaves very badly if we request a container larger than the
+    // maximum.
+
+    if (master.memoryMb > maxMemory) {
+     throw new ClientException( "YARN maximum memory is " + maxMemory
+         + " but the application master requests " + master.memoryMb );
+    }
+    if (master.vCores > maxCores) {
+      throw new ClientException("YARN maximum vcores is " + maxCores
+          + " but the application master requests " + master.vCores);
+    }
+
+    // Verify the limits for the Drillbit as well.
+
+    if (config.getInt(DrillOnYarnConfig.DRILLBIT_MEMORY) > maxMemory) {
+      throw new ClientException(
+          "YARN maximum memory is " + maxMemory + " but the Drillbit requests "
+              + config.getInt(DrillOnYarnConfig.DRILLBIT_MEMORY));
+    }
+    if (config.getInt(DrillOnYarnConfig.DRILLBIT_VCORES) > maxCores) {
+      throw new ClientException("YARN maximum vcores is " + maxCores
+          + " but the Drillbit requests "
+          + config.getInt(DrillOnYarnConfig.DRILLBIT_VCORES));
+    }
+  }
+
+  private void launchApp(AppSpec master) throws ClientException {
+    try {
+      client.submitAppMaster(master);
+    } catch (YarnClientException e) {
+      throw new ClientException("Failed to start Drill application master", e);
+    }
+  }
+
+  /**
+   * Write the app id file needed for subsequent commands. The app id file is
+   * the only way we know the YARN application associated with our Drill-on-YARN
+   * session. This file is ready by subsequent status, resize and stop commands
+   * so we can find our Drill AM on the YARN cluster.
+   *
+   * @throws ClientException
+   */
+
+  private void writeAppIdFile() throws ClientException {
+    // Write the appid file that lets us work with the app later
+    // (Analogous to a pid file.)
+    // File goes into the directory above Drill Home (which should be the
+    // folder that contains the localized archive) and is named for the
+    // ZK cluster (to ensure that the name is a valid file name.)
+
+    File appIdFile = ClientCommand.getAppIdFile();
+    try {
+      PrintWriter writer = new PrintWriter(new FileWriter(appIdFile));
+      writer.println(appId);
+      writer.close();
+    } catch (IOException e) {
+      throw new ClientException(
+          "Failed to write appid file: " + appIdFile.getAbsolutePath());
+    }
+  }
+
+  /**
+   * Poll YARN to track the launch process of the application so that we can
+   * wait until the AM is live before pointing the user to the AM's web UI.
+   */
+
+  private class StartMonitor {
+    StatusCommand.Reporter reporter;
+    private YarnApplicationState state;
+    private int pollWaitSec;
+    private int startupWaitSec;
+
+    public StartMonitor() {
+      pollWaitSec = config.getInt(DrillOnYarnConfig.CLIENT_POLL_SEC);
+      if (pollWaitSec < 1) {
+        pollWaitSec = 1;
+      }
+      startupWaitSec = config.getInt(DrillOnYarnConfig.CLIENT_START_WAIT_SEC);
+    }
+
+    void run(String appName) throws ClientException {
+      System.out.print("Launching " + appName + "...");
+      reporter = new StatusCommand.Reporter(client);
+      reporter.getReport();
+      if (!reporter.isStarting()) {
+        return;
+      }
+      updateState(reporter.getState());
+      try {
+        int attemptCount = startupWaitSec / pollWaitSec;
+        for (int attempt = 0; attempt < attemptCount; attempt++) {
+          if (!poll()) {
+            break;
+          }
+        }
+      } finally {
+        System.out.println();
+      }
+      reporter.display(verbose, true);
+      if (reporter.isStarting()) {
+        System.out.println(
+            "Application Master is slow to start, use the 'status' command later to check status.");
+      }
+    }
+
+    private boolean poll() throws ClientException {
+      try {
+        Thread.sleep(pollWaitSec * 1000);
+      } catch (InterruptedException e) {
+        return false;
+      }
+      reporter.getReport();
+      if (!reporter.isStarting()) {
+        return false;
+      }
+      YarnApplicationState newState = reporter.getState();
+      if (newState == state) {
+        System.out.print(".");
+        return true;
+      }
+      System.out.println();
+      updateState(newState);
+      return true;
+    }
+
+    private void updateState(YarnApplicationState newState) {
+      state = newState;
+      if (verbose) {
+        System.out.print("Application State: ");
+        System.out.println(state.toString());
+        System.out.print("Starting...");
+      }
+    }
+  }
+
+  private void waitForStartAndReport(String appName) throws ClientException {
+    StartMonitor monitor = new StartMonitor();
+    monitor.run(appName);
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/java/org/apache/drill/yarn/client/CleanCommand.java
----------------------------------------------------------------------
diff --git a/drill-yarn/src/main/java/org/apache/drill/yarn/client/CleanCommand.java b/drill-yarn/src/main/java/org/apache/drill/yarn/client/CleanCommand.java
new file mode 100644
index 0000000..1fcba2d
--- /dev/null
+++ b/drill-yarn/src/main/java/org/apache/drill/yarn/client/CleanCommand.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.yarn.client;
+
+import java.io.File;
+
+import org.apache.drill.yarn.core.DfsFacade;
+import org.apache.drill.yarn.core.DrillOnYarnConfig;
+
+import com.typesafe.config.Config;
+
+import org.apache.drill.yarn.core.DfsFacade.DfsFacadeException;
+
+public class CleanCommand extends ClientCommand {
+  private Config config;
+  private DfsFacade dfs;
+
+  @Override
+  public void run() throws ClientException {
+    config = DrillOnYarnConfig.config();
+    if (!isLocalized()) {
+      System.out.println("Not using localized files; nothing to clean.");
+      return;
+    }
+    connectToDfs();
+    removeDrillArchive();
+    removeSiteArchive();
+  }
+
+  public boolean isLocalized() {
+    return config.getBoolean(DrillOnYarnConfig.LOCALIZE_DRILL);
+  }
+
+  protected void connectToDfs() throws ClientException {
+    try {
+      System.out.print("Connecting to DFS...");
+      dfs = new DfsFacade(config);
+      dfs.connect();
+      System.out.println(" Connected.");
+    } catch (DfsFacadeException e) {
+      System.out.println("Failed.");
+      throw new ClientException("Failed to connect to DFS", e);
+    }
+  }
+
+  private void removeDrillArchive() {
+    String localArchivePath = config
+        .getString(DrillOnYarnConfig.DRILL_ARCHIVE_PATH);
+    String archiveName = new File(localArchivePath).getName();
+    removeArchive(archiveName);
+  }
+
+  private void removeArchive(String archiveName) {
+    System.out.print("Removing " + archiveName + " ...");
+    try {
+      dfs.removeDrillFile(archiveName);
+      System.out.println(" Removed");
+      ;
+    } catch (DfsFacadeException e) {
+      System.out.println();
+      System.err.println(e.getMessage());
+    }
+  }
+
+  private void removeSiteArchive() {
+    DrillOnYarnConfig doyConfig = DrillOnYarnConfig.instance();
+    if (!doyConfig.hasSiteDir()) {
+      return;
+    }
+    String archiveName = DrillOnYarnConfig.SITE_ARCHIVE_NAME;
+    removeArchive(archiveName);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/java/org/apache/drill/yarn/client/ClientCommand.java
----------------------------------------------------------------------
diff --git a/drill-yarn/src/main/java/org/apache/drill/yarn/client/ClientCommand.java b/drill-yarn/src/main/java/org/apache/drill/yarn/client/ClientCommand.java
new file mode 100644
index 0000000..469d04c
--- /dev/null
+++ b/drill-yarn/src/main/java/org/apache/drill/yarn/client/ClientCommand.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.yarn.client;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.IOException;
+
+import org.apache.drill.yarn.core.DrillOnYarnConfig;
+import org.apache.drill.yarn.core.YarnRMClient;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+
+public abstract class ClientCommand {
+  protected CommandLineOptions opts;
+
+  public void setOpts(CommandLineOptions opts) {
+    this.opts = opts;
+  }
+
+  public abstract void run() throws ClientException;
+
+  /**
+   * Return the path to the app id file. The file goes into the directory above
+   * Drill Home (which should be the folder that contains the localized archive)
+   * and is named for the ZK cluster (to ensure that the name is a valid file
+   * name.)
+   *
+   * @return
+   */
+  protected static File getAppIdFile() {
+    return DrillOnYarnConfig.instance().getLocalAppIdFile();
+  }
+
+  protected ApplicationId checkAppId() throws ClientException {
+    String appIdStr;
+    if (opts.appId != null) {
+      appIdStr = opts.appId;
+    } else {
+      File appIdFile = getAppIdFile();
+      appIdStr = loadAppId(appIdFile);
+      if (appIdStr == null) {
+        throw new ClientException(
+            "No Drill cluster is running (did not find file appid file: "
+                + appIdFile.toString() + ")");
+      }
+    }
+    return ConverterUtils.toApplicationId(appIdStr);
+  }
+
+  protected YarnRMClient getClient() throws ClientException {
+    return new YarnRMClient(checkAppId());
+  }
+
+  protected String loadAppId(File appIdFile) {
+    BufferedReader reader = null;
+    String appIdStr;
+    try {
+      reader = new BufferedReader(new FileReader(appIdFile));
+      appIdStr = reader.readLine();
+      if (appIdStr != null) {
+        appIdStr = appIdStr.trim();
+      }
+    } catch (FileNotFoundException e) {
+      return null;
+    } catch (IOException e) {
+      return null;
+    } finally {
+      try {
+        if (reader != null) {
+          reader.close();
+        }
+      } catch (IOException e) {
+        // Ignore
+      }
+    }
+    return appIdStr;
+  }
+
+  protected void removeAppIdFile() {
+    getAppIdFile().delete();
+  }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/java/org/apache/drill/yarn/client/ClientContext.java
----------------------------------------------------------------------
diff --git a/drill-yarn/src/main/java/org/apache/drill/yarn/client/ClientContext.java b/drill-yarn/src/main/java/org/apache/drill/yarn/client/ClientContext.java
new file mode 100644
index 0000000..377b3b3
--- /dev/null
+++ b/drill-yarn/src/main/java/org/apache/drill/yarn/client/ClientContext.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.yarn.client;
+
+import java.io.PrintStream;
+
+/**
+ * Provides a static set of contextual operations that can be configured one way
+ * for production, a separate way for unit tests.
+ */
+
+public class ClientContext {
+
+  private static ClientContext instance;
+  public static PrintStream err = System.err;
+  public static PrintStream out = System.out;
+
+  public static void init() {
+    init(new ClientContext());
+  }
+
+  protected static void init(ClientContext instance) {
+    ClientContext.instance = instance;
+  }
+
+  public static ClientContext instance() {
+    return instance;
+  }
+
+  public void exit(int exitCode) {
+    System.exit(exitCode);
+  }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/java/org/apache/drill/yarn/client/ClientException.java
----------------------------------------------------------------------
diff --git a/drill-yarn/src/main/java/org/apache/drill/yarn/client/ClientException.java b/drill-yarn/src/main/java/org/apache/drill/yarn/client/ClientException.java
new file mode 100644
index 0000000..24c062b
--- /dev/null
+++ b/drill-yarn/src/main/java/org/apache/drill/yarn/client/ClientException.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.yarn.client;
+
+public class ClientException extends Exception {
+  private static final long serialVersionUID = 1L;
+
+  public ClientException(String msg) {
+    super(msg);
+  }
+
+  public ClientException(String msg, Exception e) {
+    super(msg, e);
+  }
+
+  public ClientException(Exception e) {
+    super(e.getMessage(), e);
+  }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/java/org/apache/drill/yarn/client/CommandLineOptions.java
----------------------------------------------------------------------
diff --git a/drill-yarn/src/main/java/org/apache/drill/yarn/client/CommandLineOptions.java b/drill-yarn/src/main/java/org/apache/drill/yarn/client/CommandLineOptions.java
new file mode 100644
index 0000000..174265d
--- /dev/null
+++ b/drill-yarn/src/main/java/org/apache/drill/yarn/client/CommandLineOptions.java
@@ -0,0 +1,230 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.yarn.client;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Drill YARN client command line options.
+ * <p><pre>
+ * DrillYarnClient -h|--help |
+ *                 start |
+ *                 stop |
+ *                 status |
+ *                 resize [+|-]n
+ * </pre></p>
+ * <ul>
+ * <li>help: Prints command line usage</li>
+ * <li>start: starts the defined cluster</li>
+ * <li>stop: stops the defined cluster</li>
+ * <li>resize: adds (+n), removes (-n) or resizes (n) the cluster</li>
+ * <li>status: prints status about the cluster</li>
+ * </ul>
+ * <p>
+ * This is a do-it-yourself parser because the command line parser
+ * used by Drill does not accept parameters (arguments) without a dash,
+ * and does not accept arguments (such as resize -3) with a dash.
+ */
+
+public class CommandLineOptions {
+
+  public enum Command {
+    HELP( "help", "Provide description of usage."),
+
+    /**
+     * Primary command to upload the application archive and start the Drill cluster.
+     */
+
+    START( "start", "Start the cluster."),
+
+    // Removed at QA request. QA wants a "real" restart. Also, upload of the
+    // archive is fast enough that a "start without upload" option is not really
+    // needed.
+//    /**
+//     * Convenience method when debugging, testing. Restarts the cluster without the
+//     * archive upload; assumes the upload was already done.
+//     */
+//
+//    RESTART( "restart", "Restart the cluster (without archive upload)."),
+
+    /**
+     * Primary command to stop a running cluster.
+     */
+
+    STOP( "stop", "Stop the cluster."),
+
+    /**
+     * Primary command to get the status of a running cluster.
+     */
+
+    STATUS( "status", "Provide the status of the cluster."),
+
+    RESIZE( "resize", "Resize the cluster +n: add nodes, -n: remove nodes, n resize to given size."),
+
+    TEST( null, null ),
+
+    /**
+     * Convenience command to display the effective configuration settings to
+     * diagnose problems.
+     */
+
+    DESCRIBE( "describe", "Display and validate configuration." ),
+
+    /**
+     * Convenience command to upload the application archive to test the DFS
+     * settings without launching the Drill cluster.
+     */
+
+    UPLOAD( "upload", "Upload archives to validate DFS." ),
+
+    /**
+     * Convenience command to remove the Drill-on-YARN archive(s) from DFS.
+     * Note: doing this while a Drill cluster is running will cause subsequent
+     * Drillbit launches to fail.
+     */
+
+    CLEAN( "clean", "Remove archives stored in DFS." );
+
+    private String cmd;
+    private String descrip;
+
+    private Command(String cmd, String descrip) {
+      this.cmd = cmd;
+      this.descrip = descrip;
+    }
+
+    public boolean isMatch(String arg) {
+      String key = (cmd == null) ? toString() : cmd;
+      return key.equalsIgnoreCase(arg);
+    }
+
+    public boolean isHidden() {
+      return descrip == null;
+    }
+
+    public String getCommand( ) { return cmd; }
+    public String getDescription( ) { return descrip; }
+  }
+
+  Command command;
+  public String appId;
+  public boolean dryRun;
+  public String resizePrefix;
+  public int resizeValue;
+  public boolean verbose = false;
+  public boolean force = false;
+
+  /**
+   * Parse the command line. Invalid option combinations result in the
+   * error option being set.
+   */
+  public boolean parse(String args[]) {
+    for (int i = 0; i < args.length; i++) {
+      String arg = args[i];
+      if (arg.equals("-h") || arg.equals("-?")) {
+        command = Command.HELP;
+        break;
+      }
+      if (arg.equals("-v") || arg.equals("--verbose")) {
+        verbose = true;
+        continue;
+      }
+      if (arg.equals("-f") || arg.equals("--force")) {
+        force = true;
+        continue;
+      }
+      if (arg.equals("-d") || arg.equals("--dryrun")) {
+        dryRun = true;
+        continue;
+      }
+      if (arg.equals("-a") || arg.equals("--appid")) {
+        if (i + 1 == args.length) {
+          return false;
+        }
+        appId = args[++i];
+        continue;
+      }
+      if (command != null) {
+        command = null;
+        return false;
+      }
+
+      // Check if a command line word matches this command. Be nice,
+      // allow -foo and --foo in addition to the "proper" foo.
+
+      String cmdStr = arg;
+      if (cmdStr.startsWith("--")) {
+        cmdStr = arg.substring(2);
+      } else if (cmdStr.startsWith("-")) {
+        cmdStr = cmdStr.substring(1);
+      }
+      for (Command cmd : Command.values()) {
+        if (cmd.isMatch(cmdStr)) {
+          command = cmd;
+          if (command == Command.RESIZE) {
+            if (i + 1 == args.length) {
+              command = null;
+              break;
+            }
+            parseResizeOption(args[++i]);
+          }
+          break;
+        }
+      }
+    }
+    return true;
+  }
+
+  private void parseResizeOption(String resize) {
+    Pattern p = Pattern.compile("([+-]?)(\\d+)");
+    Matcher m = p.matcher(resize);
+    if (m.matches()) {
+      resizePrefix = m.group(1);
+      resizeValue = Integer.parseInt(m.group(2));
+    } else {
+      command = null;
+      return;
+    }
+  }
+
+  public Command getCommand() {
+    return command;
+  }
+
+  public String getResizePrefix() {
+    return resizePrefix;
+  }
+
+  public int getResizeValue() {
+    return resizeValue;
+  }
+
+  public void usage() {
+    ClientContext.out.println(
+        "Usage: drill-on-yarn.sh [--site site-dir] command [-v|--verbose][-a app-id]");
+    ClientContext.out.println("Where command is one of:");
+    for (Command cmd : Command.values()) {
+      if (cmd.isHidden()) {
+        continue;
+      }
+      ClientContext.out
+          .println("  " + cmd.getCommand() + " - " + cmd.getDescription());
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/java/org/apache/drill/yarn/client/DrillOnYarn.java
----------------------------------------------------------------------
diff --git a/drill-yarn/src/main/java/org/apache/drill/yarn/client/DrillOnYarn.java b/drill-yarn/src/main/java/org/apache/drill/yarn/client/DrillOnYarn.java
new file mode 100644
index 0000000..587766a
--- /dev/null
+++ b/drill-yarn/src/main/java/org/apache/drill/yarn/client/DrillOnYarn.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.yarn.client;
+
+
+import org.apache.drill.yarn.core.DoyConfigException;
+import org.apache.drill.yarn.core.DrillOnYarnConfig;
+import org.apache.log4j.BasicConfigurator;
+
+/**
+ * Client for the Drill-on-YARN integration. See YARN documentation
+ * for the role of a YARN client.
+ * <p>
+ * The client needs configuration information from drill-on-yarn.conf,
+ * the directory of which must be in the class path. It is put there
+ * by the drill-on-yarn.sh script.
+ * <p>
+ * The client also requires a debugging configuration file to be given
+ * explicitly as follows:<br>
+ * -Dlogback.configurationFile=/path/to/yarn-client-log.xml<br>
+ * The drillbit itself uses the default logging config file name of
+ * logback.xml; which contains references to system properties that are
+ * not defined in this client. The result of not including the log
+ * configuration file is that you'll see various var.name_IS_UNDEFINED
+ * files in the directory from which you launched the client.
+ * <p>
+ * The client accepts a command, creates a command object for that
+ * command, and executes it. There are a few main commands (start, stop),
+ * along with some management commands (status, resize), and a few commands
+ * mostly used for debugging and diagnosis (upload,etc.) Some commands
+ * are very similar, so a single command object may handle multiple user
+ * commands.
+ * <p>
+ * The client requires a working distributed file system (DFS), the
+ * configuration of which is given either implicitly, or in the Hadoop
+ * configuration files. Similarly, the client requires a working YARN
+ * deployment, again with either implicit configuration or configuration
+ * given in the Hadoop configuration. The Hadoop configuration must be
+ * on the class path when launching this client.
+ *
+ * <h3>Debugging</h3>
+ * <p>
+ * To debug this class, add two or three directories to your class path:
+ * <ul>
+ * <li>$DRILL_CONF_DIR (if using a separate site directory)</li>
+ * <li>$HADOOP_HOME/etc/hadoop</li>
+ * <li>$DRILL_HOME/conf</li>
+ * </ul>
+ * Note that these MUST be in the order listed since $DRILL_HOME/conf
+ * contains, by default, a version of core-site.xml that probably is
+ * NOT the one you want to use for YARN. For YARN, you want the one
+ * in $HADOOP_HOME/etc/hadoop.
+ * <p>
+ * Also, set the following VM argument:<br>
+ * -Dlogback.configurationFile=/path/to/drill/conf/yarn-client-log.xml<br>
+ * or<br>
+ * -Dlogback.configurationFile=/path/to/drill-site/yarn-client-log.xml<br>
+ */
+
+public class DrillOnYarn {
+  public static void main(String argv[]) {
+    BasicConfigurator.configure();
+    ClientContext.init();
+    run(argv);
+  }
+
+  public static void run(String argv[]) {
+    ClientContext context = ClientContext.instance();
+
+    // Parse command-line options.
+
+    CommandLineOptions opts = new CommandLineOptions();
+    if (!opts.parse(argv)) {
+      opts.usage();
+      context.exit(-1);
+    }
+    if (opts.getCommand() == null) {
+      opts.usage();
+      context.exit(-1);
+    }
+
+    // Load configuration.
+
+    try {
+      DrillOnYarnConfig.load().setClientPaths();
+    } catch (DoyConfigException e) {
+      ClientContext.err.println(e.getMessage());
+      context.exit(-1);
+    }
+
+    // Create the required command object.
+
+    ClientCommand cmd;
+    switch (opts.getCommand()) {
+    case UPLOAD:
+      cmd = new StartCommand(true, false);
+      break;
+    case START:
+      cmd = new StartCommand(true, true);
+      break;
+    // Removed at QA request. QA wants a "real" restart. Also, upload of the
+    // archive is fast enough that a "start without upload" option is not really
+    // needed.
+//    case RESTART:
+//      cmd = new StartCommand(false, true);
+//      break;
+    case DESCRIBE:
+      cmd = new PrintConfigCommand();
+      break;
+    case STATUS:
+      cmd = new StatusCommand();
+      break;
+    case STOP:
+      cmd = new StopCommand();
+      break;
+    case CLEAN:
+      cmd = new CleanCommand();
+      break;
+    case RESIZE:
+      cmd = new ResizeCommand();
+      break;
+    default:
+      cmd = new HelpCommand();
+    }
+
+    // Run the command.
+
+    cmd.setOpts(opts);
+    try {
+      cmd.run();
+    } catch (ClientException e) {
+      displayError(opts, e);
+      context.exit(1);
+    }
+  }
+
+  private static void displayError(CommandLineOptions opts, ClientException e) {
+
+    // Show the Drill-provided explanation of the error.
+
+    ClientContext.err.println(e.getMessage());
+
+    // Add the underlying exception information, if any.
+
+    Throwable parent = e;
+    Throwable cause = e.getCause();
+    while (cause != null && cause != parent) {
+      ClientContext.err.print("  Caused by: ");
+      ClientContext.err.println(cause.getMessage());
+      parent = cause;
+      cause = cause.getCause();
+    }
+
+    // Include the full stack trace if requested.
+
+    if (opts.verbose) {
+      ClientContext.err.println("Full stack trace:");
+      e.printStackTrace(ClientContext.err);
+    }
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/java/org/apache/drill/yarn/client/FileUploader.java
----------------------------------------------------------------------
diff --git a/drill-yarn/src/main/java/org/apache/drill/yarn/client/FileUploader.java b/drill-yarn/src/main/java/org/apache/drill/yarn/client/FileUploader.java
new file mode 100644
index 0000000..ace2d03
--- /dev/null
+++ b/drill-yarn/src/main/java/org/apache/drill/yarn/client/FileUploader.java
@@ -0,0 +1,551 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.yarn.client;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.PrintStream;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.drill.yarn.core.DfsFacade;
+import org.apache.drill.yarn.core.DoYUtil;
+import org.apache.drill.yarn.core.DoyConfigException;
+import org.apache.drill.yarn.core.DrillOnYarnConfig;
+import org.apache.drill.yarn.core.DfsFacade.DfsFacadeException;
+import org.apache.drill.yarn.core.DfsFacade.Localizer;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+
+import com.typesafe.config.Config;
+
+/**
+ * Performs the file upload portion of the operation by uploading an archive to
+ * the target DFS system and directory. Records the uploaded archive so it may
+ * be used for localizing Drill in the launch step.
+ * <p>
+ * Some of the code is a bit of a dance so we can get information early to
+ * display in status messages.
+ * <p>
+ * This class handles x cases:
+ * <ol>
+ * <li>Non-localized, config in $DRILL_HOME/conf.</li>
+ * <li>Non-localized, config in a site directory.</li>
+ * <li>Localized, config in $DRILL_HOME.</li>
+ * <li>Localized, config in a site directory.</li>
+ * </ol>
+ * <p>
+ * The non-localized case adds complexity, but is very handy when doing
+ * development as it avoids the wait for the archives to up- and down-load. The
+ * non-localized mode is not advertised to users as it defeats one of the main
+ * benefits of YARN.
+ * <p>
+ * In the localized case, YARN is incomplete; there is no API to inform the AM
+ * of the set of localized files, so we pass the information along in
+ * environment variables. Also, tar is a bit annoying because it includes the
+ * root directory name when unpacking, so that the drill.tar.gz archive unpacks
+ * to, say, apache-drill.x.y.z. So, we must pass along the directory name as
+ * well.
+ * <p>
+ * All of this is further complicated by the way YARN needs detailed information
+ * to localize resources, and that YARN uses a "key" to identify localized
+ * resources, which becomes the directory name in the task's working folder.
+ * Thus, Drill becomes, say<br>
+ * $PWD/drill/apache-drill.x.y.z/bin, conf, ...<br>
+ * YARN provides PWD. The Drillbit launch script needs to know the next two
+ * directory names.
+ * <p>
+ * For efficiency, we omit uploading the Drill archive if one already exists in
+ * dfs and is the same size as the one on the client. We always upload the
+ * config archive (if needed) because config changes are likely to be one reason
+ * that someone (re)starts the Drill cluster.
+ */
+
+public abstract class FileUploader {
+  protected DrillOnYarnConfig doyConfig;
+  protected Config config;
+  protected DfsFacade dfs;
+  protected boolean dryRun;
+  protected boolean verbose;
+  protected File localDrillHome;
+  protected File localSiteDir;
+  protected File localDrillArchivePath;
+
+  public Map<String, LocalResource> resources = new HashMap<>();
+  public String drillArchivePath;
+  public String siteArchivePath;
+  public String remoteDrillHome;
+  public String remoteSiteDir;
+
+  public static class NonLocalized extends FileUploader {
+    public NonLocalized(boolean dryRun, boolean verbose) {
+      super(dryRun, verbose);
+    }
+
+    @Override
+    public void run() throws ClientException {
+      setup();
+      prepareDrillHome();
+      if (hasSiteDir()) {
+        prepareSiteDir();
+      }
+      if (verbose || dryRun) {
+        dump(System.out);
+      }
+    }
+
+    private void prepareDrillHome() throws ClientException {
+      // We need the drill home property. The client can figure out the
+      // Drill home, but the AM must be told.
+
+      String drillHomeProp = config.getString(DrillOnYarnConfig.DRILL_HOME);
+      if (DoYUtil.isBlank(drillHomeProp)) {
+        System.out.println("Warning: non-localized run "
+            + DrillOnYarnConfig.DRILL_HOME + " is not set.");
+        System.out.println(
+            "Assuming remote Drill home is the same as the local location: "
+                + localDrillHome.getAbsolutePath());
+      }
+    }
+
+    private void prepareSiteDir() throws ClientException {
+      String siteDirProp = config.getString(DrillOnYarnConfig.SITE_DIR);
+      if (DoYUtil.isBlank(siteDirProp)) {
+        System.out.println("Warning: non-localized run "
+            + DrillOnYarnConfig.SITE_DIR + " is not set.");
+        System.out.println(
+            "Assuming remote Drill site is the same as the local location: "
+                + localSiteDir.getAbsolutePath());
+      }
+    }
+  }
+
+  public static class ReuseFiles extends FileUploader {
+    public ReuseFiles(boolean dryRun, boolean verbose) {
+      super(dryRun, verbose);
+    }
+
+    @Override
+    public void run() throws ClientException {
+      setup();
+      checkDrillArchive();
+      if (hasSiteDir()) {
+        checkSiteArchive();
+      }
+      if (verbose || dryRun) {
+        dump(System.out);
+      }
+    }
+
+    /**
+     * Upload the Drill archive if desired. Skip the upload if the file already
+     * exists in dfs and is the same size as the local file. However using the
+     * force option can force an upload even if the sizes match.
+     * <p>
+     * Prepares the information needed to tell YARN and the AM about the
+     * localized archive.
+     * <p>
+     * Note that the Drill archive is not created by this client; it must
+     * already exist on disk. Typically, it is just the archive downloaded from
+     * Apache or some other distribution. The uploaded archive retains the name
+     * of the archive in the client, which may be useful to check the version of
+     * the uploaded code based on the file name.
+     *
+     * @throws ClientException
+     */
+
+    private void checkDrillArchive() throws ClientException {
+      // Print the progress message here because doing the connect takes
+      // a while and the message makes it look like we're doing something.
+
+      DfsFacade.Localizer localizer = makeDrillLocalizer();
+      connectToDfs();
+      try {
+        if (!localizer.destExists()) {
+          throw new ClientException(
+              "Drill archive not found in DFS: " + drillArchivePath);
+        }
+      } catch (IOException e) {
+        throw new ClientException(
+            "Failed to check existence of " + drillArchivePath, e);
+      }
+      if (!localDrillArchivePath.exists()) {
+        return;
+      }
+      if (!localizer.filesMatch()) {
+        System.out.println(
+            "Warning: Drill archive on DFS does not match the local version.");
+      }
+      defineResources(localizer, DrillOnYarnConfig.DRILL_ARCHIVE_KEY);
+    }
+
+    private void checkSiteArchive() throws ClientException {
+      // Print the progress message here because doing the connect takes
+      // a while and the message makes it look like we're doing something.
+
+      DfsFacade.Localizer localizer = makeSiteLocalizer(null);
+      try {
+        if (!localizer.destExists()) {
+          throw new ClientException(
+              "Drill archive not found in DFS: " + drillArchivePath);
+        }
+      } catch (IOException e) {
+        throw new ClientException(
+            "Failed to check existence of " + drillArchivePath, e);
+      }
+      defineResources(localizer, DrillOnYarnConfig.SITE_ARCHIVE_KEY);
+    }
+  }
+
+  public static class UploadFiles extends FileUploader {
+    private boolean force;
+
+    public UploadFiles(boolean force, boolean dryRun, boolean verbose) {
+      super(dryRun, verbose);
+      this.force = force;
+    }
+
+    @Override
+    public void run() throws ClientException {
+      setup();
+      uploadDrillArchive();
+      if (hasSiteDir()) {
+        uploadSite();
+      }
+      if (verbose || dryRun) {
+        dump(System.out);
+      }
+    }
+
+    /**
+     * Create a temporary archive of the site directory and upload it to DFS. We
+     * always upload the site; we never reuse an existing one.
+     *
+     * @throws ClientException
+     */
+
+    private void uploadSite() throws ClientException {
+      File siteArchive = createSiteArchive();
+      try {
+        uploadSiteArchive(siteArchive);
+      } finally {
+        siteArchive.delete();
+      }
+    }
+
+    /**
+     * Upload the Drill archive if desired. Skip the upload if the file already
+     * exists in dfs and is the same size as the local file. However using the
+     * force option can force an upload even if the sizes match.
+     * <p>
+     * Prepares the information needed to tell YARN and the AM about the
+     * localized archive.
+     * <p>
+     * Note that the Drill archive is not created by this client; it must
+     * already exist on disk. Typically, it is just the archive downloaded from
+     * Apache or some other distribution. The uploaded archive retains the name
+     * of the archive in the client, which may be useful to check the version of
+     * the uploaded code based on the file name.
+     *
+     * @throws ClientException
+     */
+
+    private void uploadDrillArchive() throws ClientException {
+      // Print the progress message here because doing the connect takes
+      // a while and the message makes it look like we're doing something.
+
+      connectToDfs();
+      DfsFacade.Localizer localizer = makeDrillLocalizer();
+      boolean needsUpload = force || !localizer.filesMatch();
+
+      if (needsUpload) {
+        // Thoroughly check the Drill archive. Errors with the archive seem a
+        // likely source of confusion, so provide detailed error messages for
+        // common cases. Don't bother with these checks if no upload is needed.
+
+        if (!localDrillArchivePath.exists()) {
+          throw new ClientException(
+              "Drill archive not found: " + localDrillArchivePath.getAbsolutePath());
+        }
+        if (!localDrillArchivePath.canRead()) {
+          throw new ClientException(
+              "Drill archive is not readable: " + localDrillArchivePath.getAbsolutePath());
+        }
+        if (localDrillArchivePath.isDirectory()) {
+          throw new ClientException(
+              "Drill archive cannot be a directory: " + localDrillArchivePath.getAbsolutePath());
+        }
+      }
+
+      drillArchivePath = localizer.getDestPath();
+      if (needsUpload) {
+        if (dryRun) {
+          System.out.print(
+              "Upload " + localDrillArchivePath.getAbsolutePath() + " to " + drillArchivePath);
+        } else {
+          System.out.print("Uploading " + localDrillArchivePath.getAbsolutePath() + " to "
+              + drillArchivePath + " ... ");
+          upload(localizer);
+        }
+      } else {
+        System.out.println(
+            "Using existing Drill archive in DFS: " + drillArchivePath);
+      }
+
+      defineResources(localizer, DrillOnYarnConfig.DRILL_ARCHIVE_KEY);
+    }
+
+    /**
+     * Run the tar command to archive the site directory into a temporary
+     * archive which is then uploaded to DFS using a standardized name. The site
+     * directory is always uploaded since configuration is subject to frequent
+     * changes.
+     *
+     * @return
+     * @throws ClientException
+     */
+
+    private File createSiteArchive() throws ClientException {
+      File siteArchiveFile;
+      try {
+        siteArchiveFile = File.createTempFile("drill-site-", ".tar.gz");
+      } catch (IOException e) {
+        throw new ClientException("Failed to create site archive temp file", e);
+      }
+      String cmd[] = new String[] { "tar", "-C", localSiteDir.getAbsolutePath(),
+          "-czf", siteArchiveFile.getAbsolutePath(), "." };
+      List<String> cmdList = Arrays.asList(cmd);
+      String cmdLine = DoYUtil.join(" ", cmdList);
+      if (dryRun) {
+        System.out.print("Site archive command: ");
+        System.out.println(cmdLine);
+        return siteArchiveFile;
+      }
+
+      ProcessBuilder builder = new ProcessBuilder(cmdList);
+      builder.redirectErrorStream(true);
+      Process proc;
+      try {
+        proc = builder.start();
+      } catch (IOException e) {
+        throw new ClientException("Failed to launch tar process: " + cmdLine,
+            e);
+      }
+
+      // Should not be much output. But, we have to read it anyway to avoid
+      // blocking. We'll use the output if we encounter an error.
+
+      BufferedReader br = new BufferedReader(
+          new InputStreamReader(proc.getInputStream()));
+      StringBuilder buf = new StringBuilder();
+      try {
+        String line;
+        while ((line = br.readLine()) != null) {
+          buf.append(line);
+          buf.append("\n");
+        }
+        br.close();
+      } catch (IOException e) {
+        throw new ClientException("Failed to read output from tar command", e);
+      }
+      try {
+        proc.waitFor();
+      } catch (InterruptedException e) {
+        // Won't occur.
+      }
+      if (proc.exitValue() != 0) {
+        String msg = buf.toString().trim();
+        throw new ClientException("Tar of site directory failed: " + msg);
+      }
+      return siteArchiveFile;
+    }
+
+    /**
+     * Upload the site archive. For debugging, the client provides the option to
+     * use existing files, which users should not do in production.
+     *
+     * @param siteArchive
+     * @throws ClientException
+     */
+
+    private void uploadSiteArchive(File siteArchive) throws ClientException {
+      DfsFacade.Localizer localizer = makeSiteLocalizer(siteArchive);
+
+      if (dryRun) {
+        System.out.println("Upload site archive to " + siteArchivePath);
+      } else {
+        System.out
+        .print("Uploading site directory " + localSiteDir.getAbsolutePath() +
+               " to " + siteArchivePath + " ... ");
+        upload(localizer);
+      }
+      defineResources(localizer, DrillOnYarnConfig.SITE_ARCHIVE_KEY);
+    }
+  }
+
+  public FileUploader(boolean dryRun, boolean verbose) {
+    doyConfig = DrillOnYarnConfig.instance();
+    this.config = doyConfig.getConfig();
+    this.dryRun = dryRun;
+    this.verbose = verbose;
+  }
+
+  public abstract void run() throws ClientException;
+
+  /**
+   * Common setup of the Drill and site directories.
+   *
+   * @throws ClientException
+   */
+
+  protected void setup() throws ClientException {
+
+    // Local and remote Drill home locations.
+
+    localDrillHome = doyConfig.getLocalDrillHome();
+    try {
+      remoteDrillHome = doyConfig.getRemoteDrillHome();
+    } catch (DoyConfigException e) {
+      throw new ClientException(e);
+    }
+
+    // Site directory is optional. Local and remote locations, if provided.
+    // Check that the site directory is an existing directory.
+
+    localSiteDir = doyConfig.getLocalSiteDir();
+    if (hasSiteDir()) {
+      if (!localSiteDir.isDirectory()) {
+        throw new ClientException(
+            "Drill site dir not a directory: " + localSiteDir);
+      }
+      remoteSiteDir = doyConfig.getRemoteSiteDir();
+    }
+
+    // Disclaimer that this is just a dry run when that option is selected.
+
+    if (dryRun) {
+      System.out.println("Dry run only.");
+    }
+  }
+
+  public boolean hasSiteDir() {
+    return localSiteDir != null;
+  }
+
+  /**
+   * Report whether the user wants to localize (upload) Drill files, or just use
+   * files already on the worker nodes.
+   *
+   * @return
+   */
+
+  public boolean isLocalized() {
+    return config.getBoolean(DrillOnYarnConfig.LOCALIZE_DRILL);
+  }
+
+  protected void connectToDfs() throws ClientException {
+    try {
+      System.out.print("Connecting to DFS...");
+      dfs = new DfsFacade(config);
+      dfs.connect();
+      System.out.println(" Connected.");
+    } catch (DfsFacadeException e) {
+      System.out.println("Failed.");
+      throw new ClientException("Failed to connect to DFS", e);
+    }
+  }
+
+  protected Localizer makeDrillLocalizer() throws ClientException {
+    String localArchivePath = config
+        .getString(DrillOnYarnConfig.DRILL_ARCHIVE_PATH);
+    if (DoYUtil.isBlank(localArchivePath)) {
+      throw new ClientException("Drill archive path ("
+          + DrillOnYarnConfig.DRILL_ARCHIVE_PATH + ") is not set.");
+    }
+
+    // Archive is either absolute, or relative to $DRILL_HOME.
+
+    localDrillArchivePath = new File(localArchivePath);
+    if (!localDrillArchivePath.isAbsolute()) {
+      localDrillArchivePath = new File(
+          DrillOnYarnConfig.instance().getLocalDrillHome(), localArchivePath);
+    }
+    DfsFacade.Localizer localizer = new DfsFacade.Localizer(dfs,
+        localDrillArchivePath, "Drill");
+    drillArchivePath = localizer.getDestPath();
+    return localizer;
+  }
+
+  protected Localizer makeSiteLocalizer(File siteArchive) {
+    DfsFacade.Localizer localizer = new DfsFacade.Localizer(dfs, siteArchive,
+        DrillOnYarnConfig.SITE_ARCHIVE_NAME, "Site");
+    siteArchivePath = localizer.getDestPath();
+    return localizer;
+  }
+
+  protected void upload(Localizer localizer) throws ClientException {
+    try {
+      localizer.upload();
+    } catch (DfsFacadeException e) {
+      System.out.println("Failed.");
+      throw new ClientException(
+          "Failed to upload " + localizer.getLabel() + " archive", e);
+    }
+    System.out.println("Uploaded.");
+  }
+
+  protected void defineResources(Localizer localizer, String keyProp)
+      throws ClientException {
+    String key = config.getString(keyProp);
+    try {
+      localizer.defineResources(resources, key);
+    } catch (DfsFacadeException e) {
+      throw new ClientException(
+          "Failed to get DFS status for " + localizer.getLabel() + " archive",
+          e);
+    }
+  }
+
+  protected void dump(PrintStream out) {
+    out.print("Localized: ");
+    out.println((isLocalized()) ? "Yes" : "No");
+    out.print("Has Site Dir: ");
+    out.println((hasSiteDir()) ? "Yes" : "No");
+    out.print("Local Drill home: ");
+    out.println(localDrillHome.getAbsolutePath());
+    out.print("Remote Drill home: ");
+    out.println(remoteDrillHome);
+    if (hasSiteDir()) {
+      out.print("Local Site dir: ");
+      out.println(localSiteDir.getAbsolutePath());
+      out.print("Remote Site dir: ");
+      out.println(remoteSiteDir);
+    }
+    if (isLocalized()) {
+      out.print("Drill archive DFS path: ");
+      out.println(drillArchivePath);
+      if (hasSiteDir()) {
+        out.print("Site archive DFS path: ");
+        out.println(siteArchivePath);
+      }
+    }
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/java/org/apache/drill/yarn/client/HelpCommand.java
----------------------------------------------------------------------
diff --git a/drill-yarn/src/main/java/org/apache/drill/yarn/client/HelpCommand.java b/drill-yarn/src/main/java/org/apache/drill/yarn/client/HelpCommand.java
new file mode 100644
index 0000000..3e7e5d9
--- /dev/null
+++ b/drill-yarn/src/main/java/org/apache/drill/yarn/client/HelpCommand.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.yarn.client;
+
+public class HelpCommand extends ClientCommand {
+  @Override
+  public void run() {
+    opts.usage();
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/java/org/apache/drill/yarn/client/KillCommand.java
----------------------------------------------------------------------
diff --git a/drill-yarn/src/main/java/org/apache/drill/yarn/client/KillCommand.java b/drill-yarn/src/main/java/org/apache/drill/yarn/client/KillCommand.java
new file mode 100644
index 0000000..8b7914c
--- /dev/null
+++ b/drill-yarn/src/main/java/org/apache/drill/yarn/client/KillCommand.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.yarn.client;
+
+import org.apache.drill.yarn.core.YarnClientException;
+import org.apache.drill.yarn.core.YarnRMClient;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+
+public class KillCommand extends ClientCommand {
+
+  @Override
+  public void run() throws ClientException {
+    ApplicationId appId = checkAppId();
+    if (appId == null) {
+      System.exit(-1);
+    }
+    YarnRMClient client = new YarnRMClient(appId);
+    try {
+      client.killApplication();
+    } catch (YarnClientException e) {
+      throw new ClientException(e);
+    }
+    System.out.println("Kill request sent, waiting for shut-down.");
+    try {
+      client.waitForCompletion();
+    } catch (YarnClientException e) {
+      throw new ClientException(
+          "Wait for completion failed for app id: " + appId.toString(), e);
+    }
+    System.out.println("Application completed: " + appId.toString());
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/java/org/apache/drill/yarn/client/PrintConfigCommand.java
----------------------------------------------------------------------
diff --git a/drill-yarn/src/main/java/org/apache/drill/yarn/client/PrintConfigCommand.java b/drill-yarn/src/main/java/org/apache/drill/yarn/client/PrintConfigCommand.java
new file mode 100644
index 0000000..69cdf55
--- /dev/null
+++ b/drill-yarn/src/main/java/org/apache/drill/yarn/client/PrintConfigCommand.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.yarn.client;
+
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+
+import org.apache.drill.yarn.core.DrillOnYarnConfig;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+
+public class PrintConfigCommand extends ClientCommand {
+  @Override
+  public void run() {
+    // Dump configuration if requested for diagnostic use.
+
+    System.out.println("----------------------------------------------");
+    System.out.println("Effective Drill-on-YARN Configuration");
+    DrillOnYarnConfig.instance().dump();
+    System.out.println("----------------------------------------------");
+
+    // Dump YARN configuration.
+
+    System.out.println("YARN, DFS and Hadoop Configuration");
+    YarnConfiguration conf = new YarnConfiguration();
+    try {
+      YarnConfiguration.dumpConfiguration(conf,
+          new OutputStreamWriter(System.out));
+      System.out.println();
+    } catch (IOException e) {
+      // Ignore;
+    }
+    System.out.println("----------------------------------------------");
+  }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/java/org/apache/drill/yarn/client/ResizeCommand.java
----------------------------------------------------------------------
diff --git a/drill-yarn/src/main/java/org/apache/drill/yarn/client/ResizeCommand.java b/drill-yarn/src/main/java/org/apache/drill/yarn/client/ResizeCommand.java
new file mode 100644
index 0000000..43ae02c
--- /dev/null
+++ b/drill-yarn/src/main/java/org/apache/drill/yarn/client/ResizeCommand.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.yarn.client;
+
+import org.apache.drill.yarn.core.DoYUtil;
+import org.apache.drill.yarn.core.DrillOnYarnConfig;
+import org.apache.drill.yarn.core.YarnRMClient;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+import org.json.simple.parser.ParseException;
+
+import com.typesafe.config.Config;
+
+public class ResizeCommand extends ClientCommand {
+  private Config config;
+  private YarnRMClient client;
+
+  @Override
+  public void run() throws ClientException {
+    config = DrillOnYarnConfig.config();
+    client = getClient();
+    System.out.println(
+        "Resizing cluster for Application ID: " + client.getAppId().toString());
+
+    // First get an application report to ensure that the AM is,
+    // in fact, running, and to get the HTTP endpoint.
+
+    StatusCommand.Reporter reporter = new StatusCommand.Reporter(client);
+    try {
+      reporter.getReport();
+    } catch (ClientException e) {
+      reporter = null;
+    }
+    String prefix = opts.resizePrefix;
+    int quantity = opts.resizeValue;
+    String cmd;
+    if (prefix.equals("+")) {
+      cmd = "grow";
+      if (opts.verbose) {
+        System.out.println("Growing cluster by " + quantity + " nodes.");
+      }
+    } else if (prefix.equals("-")) {
+      cmd = "shrink";
+      if (opts.verbose) {
+        System.out.println("Shrinking cluster by " + quantity + " nodes.");
+      }
+    } else {
+      cmd = "resize";
+      if (opts.verbose) {
+        System.out.println("Resizing cluster to " + quantity + " nodes.");
+      }
+    }
+    if (sendResize(reporter.getAmUrl(), cmd, quantity)) {
+      System.out.println("Use web UI or status command to check progress.");
+    }
+  }
+
+  private boolean sendResize(String baseUrl, String cmd, int quantity) {
+    try {
+      if (DoYUtil.isBlank(baseUrl)) {
+        return false;
+      }
+      SimpleRestClient restClient = new SimpleRestClient();
+      String tail = "rest/" + cmd + "/" + quantity;
+      String masterKey = config.getString(DrillOnYarnConfig.HTTP_REST_KEY);
+      if (!DoYUtil.isBlank(masterKey)) {
+        tail += "?key=" + masterKey;
+      }
+      if (opts.verbose) {
+        System.out.println("Resizing with POST " + baseUrl + "/" + tail);
+      }
+      String result = restClient.send(baseUrl, tail, true);
+
+      JSONParser parser = new JSONParser();
+      Object response;
+      try {
+        response = parser.parse(result);
+      } catch (ParseException e) {
+        System.err.println("Invalid response received from AM");
+        if (opts.verbose) {
+          System.out.println(result);
+          System.out.println(e.getMessage());
+        }
+        return false;
+      }
+      JSONObject root = (JSONObject) response;
+
+      System.out.println("AM responded: " + root.get("message"));
+      if ("ok".equals(root.get("status"))) {
+        return true;
+      }
+      System.err.println("Failed to resize the application master.");
+      return false;
+    } catch (ClientException e) {
+      System.err.println("Resize failed: " + e.getMessage());
+      return false;
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/java/org/apache/drill/yarn/client/SimpleRestClient.java
----------------------------------------------------------------------
diff --git a/drill-yarn/src/main/java/org/apache/drill/yarn/client/SimpleRestClient.java b/drill-yarn/src/main/java/org/apache/drill/yarn/client/SimpleRestClient.java
new file mode 100644
index 0000000..e47fb58
--- /dev/null
+++ b/drill-yarn/src/main/java/org/apache/drill/yarn/client/SimpleRestClient.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.yarn.client;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+
+import org.apache.http.HttpResponse;
+import org.apache.http.client.ClientProtocolException;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpRequestBase;
+import org.apache.http.impl.client.DefaultHttpClient;
+
+public class SimpleRestClient {
+  public String send(String baseUrl, String resource, boolean isPost)
+      throws ClientException {
+    String url = baseUrl;
+    if (!url.endsWith("/")) {
+      url += "/";
+    }
+    url += resource;
+    try {
+      HttpClient client = new DefaultHttpClient();
+      HttpRequestBase request;
+      if (isPost) {
+        request = new HttpPost(url);
+      } else {
+        request = new HttpGet(url);
+      }
+
+      HttpResponse response = client.execute(request);
+      BufferedReader rd = new BufferedReader(
+          new InputStreamReader(response.getEntity().getContent()));
+      StringBuilder buf = new StringBuilder();
+      String line = null;
+      while ((line = rd.readLine()) != null) {
+        buf.append(line);
+      }
+      return buf.toString().trim();
+    } catch (ClientProtocolException e) {
+      throw new ClientException("Internal REST error", e);
+    } catch (IllegalStateException e) {
+      throw new ClientException("Internal REST error", e);
+    } catch (IOException e) {
+      throw new ClientException("REST request failed: " + url, e);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/java/org/apache/drill/yarn/client/StartCommand.java
----------------------------------------------------------------------
diff --git a/drill-yarn/src/main/java/org/apache/drill/yarn/client/StartCommand.java b/drill-yarn/src/main/java/org/apache/drill/yarn/client/StartCommand.java
new file mode 100644
index 0000000..fe505a2
--- /dev/null
+++ b/drill-yarn/src/main/java/org/apache/drill/yarn/client/StartCommand.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.yarn.client;
+
+import java.io.File;
+
+import org.apache.drill.yarn.client.StatusCommand.Reporter;
+import org.apache.drill.yarn.core.DrillOnYarnConfig;
+import org.apache.drill.yarn.core.YarnRMClient;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+
+import com.typesafe.config.Config;
+
+/**
+ * Launches a drill cluster by uploading the Drill archive then launching the
+ * Drill Application Master (AM). For testing, can also do just the upload or
+ * just the launch. Handles both a localized Drill and a non-localized launch
+ * (which uses a pre-installed Drill.)
+ * <p>
+ * This single operation combines upload and launch because the upload
+ * Information is needed by the launch.
+ * <p>
+ * On the surface, it would seem that uploading a file and launching an app
+ * should be simple operations. However, under YARN, we must handle a large
+ * number of details that must be gotten exactly right. Plus, both the upload
+ * and launch can be slow operations, so we provide feedback to the user that
+ * something is, indeed, happening.
+ */
+
+public class StartCommand extends ClientCommand {
+  private Config config;
+  private boolean upload;
+  private boolean launch;
+  private boolean dryRun;
+
+  public StartCommand(boolean upload, boolean launch) {
+    this.upload = upload;
+    this.launch = launch;
+  }
+
+  @Override
+  public void run() throws ClientException {
+    checkExistingApp();
+
+    dryRun = opts.dryRun;
+    config = DrillOnYarnConfig.config();
+    FileUploader uploader = upload();
+    if (launch) {
+      launch(uploader);
+    }
+  }
+
+  /**
+   * Check if an application ID file exists. If it does, check if an application
+   * is running. If an app is running, then we can't start a new one. If the app
+   * is not running, then clean up the "orphan" app id file.
+   *
+   * @throws ClientException
+   */
+
+  private void checkExistingApp() throws ClientException {
+    File appIdFile = getAppIdFile();
+    if (!appIdFile.exists()) {
+      return;
+    }
+
+    // File exists. Ask YARN about status.
+
+    Reporter reporter;
+    ApplicationId appId;
+    try {
+      System.out.println("Found app ID file: " + appIdFile.getAbsolutePath());
+      appId = checkAppId();
+      System.out.print("Checking application ID: " + appId.toString() + "...");
+      YarnRMClient client = new YarnRMClient(appId);
+      reporter = new Reporter(client);
+      reporter.getReport();
+    } catch (ClientException e) {
+      // This exception occurs when we ask for a report about an application
+      // that
+      // YARN does not know about. (YARN has likely been restarted.)
+
+      System.out.println(" Not running.");
+      appIdFile.delete();
+      return;
+    }
+
+    // YARN knows about the application. But, was it stopped, perhaps from the
+    // web UI?
+
+    if (reporter.isStopped()) {
+      System.out.println(" Completed with state " + reporter.getState());
+      appIdFile.delete();
+      return;
+    }
+
+    // The app (or another one with the same App ID) is running.
+
+    System.out.println(" Still running!");
+    throw new ClientException(
+        "Error: AM already running as Application ID: " + appId);
+  }
+
+  private FileUploader upload() throws ClientException {
+    FileUploader uploader;
+    if (!config.getBoolean(DrillOnYarnConfig.LOCALIZE_DRILL)) {
+      uploader = new FileUploader.NonLocalized(dryRun, opts.verbose);
+    } else if (upload) {
+      uploader = new FileUploader.UploadFiles(opts.force, dryRun, opts.verbose);
+    } else {
+      uploader = new FileUploader.ReuseFiles(dryRun, opts.verbose);
+    }
+    uploader.run();
+    return uploader;
+  }
+
+  private void launch(FileUploader uploader) throws ClientException {
+    AMRunner runner = new AMRunner(config, opts.verbose, dryRun);
+    runner.resources = uploader.resources;
+    runner.remoteDrillHome = uploader.remoteDrillHome;
+    runner.remoteSiteDir = uploader.remoteSiteDir;
+    if (uploader.isLocalized()) {
+      runner.drillArchivePath = uploader.drillArchivePath.toString();
+      if (uploader.hasSiteDir()) {
+        runner.siteArchivePath = uploader.siteArchivePath.toString();
+      }
+    }
+    runner.run();
+  }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/java/org/apache/drill/yarn/client/StatusCommand.java
----------------------------------------------------------------------
diff --git a/drill-yarn/src/main/java/org/apache/drill/yarn/client/StatusCommand.java b/drill-yarn/src/main/java/org/apache/drill/yarn/client/StatusCommand.java
new file mode 100644
index 0000000..863b700
--- /dev/null
+++ b/drill-yarn/src/main/java/org/apache/drill/yarn/client/StatusCommand.java
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.yarn.client;
+
+import org.apache.drill.yarn.core.DoYUtil;
+import org.apache.drill.yarn.core.YarnClientException;
+import org.apache.drill.yarn.core.YarnRMClient;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+import org.json.simple.parser.ParseException;
+
+public class StatusCommand extends ClientCommand {
+  public static class Reporter {
+    private YarnRMClient client;
+    ApplicationReport report;
+
+    public Reporter(YarnRMClient client) {
+      this.client = client;
+    }
+
+    public void getReport() throws ClientException {
+      try {
+        report = client.getAppReport();
+      } catch (YarnClientException e) {
+        throw new ClientException(
+            "Failed to get report for Drill application master", e);
+      }
+    }
+
+    public void display(boolean verbose, boolean isNew) {
+      YarnApplicationState state = report.getYarnApplicationState();
+      if (verbose || !isNew) {
+        System.out.println("Application State: " + state.toString());
+        System.out.println("Host: " + report.getHost());
+      }
+      if (verbose || !isNew) {
+        System.out.println("Queue: " + report.getQueue());
+        System.out.println("User: " + report.getUser());
+        long startTime = report.getStartTime();
+        System.out.println("Start Time: " + DoYUtil.toIsoTime(startTime));
+        System.out.println("Application Name: " + report.getName());
+      }
+      System.out.println("Tracking URL: " + report.getTrackingUrl());
+      if (isNew) {
+        System.out.println("Application Master URL: " + getAmUrl());
+      }
+      showFinalStatus();
+    }
+
+    public String getAmUrl() {
+      return StatusCommand.getAmUrl(report);
+    }
+
+    public void showFinalStatus() {
+      YarnApplicationState state = report.getYarnApplicationState();
+      if (state == YarnApplicationState.FAILED
+          || state == YarnApplicationState.FINISHED) {
+        FinalApplicationStatus status = report.getFinalApplicationStatus();
+        System.out.println("Final status: " + status.toString());
+        if (status != FinalApplicationStatus.SUCCEEDED) {
+          String diag = report.getDiagnostics();
+          if (!DoYUtil.isBlank(diag)) {
+            System.out.println("Diagnostics: " + diag);
+          }
+        }
+      }
+    }
+
+    public YarnApplicationState getState() {
+      return report.getYarnApplicationState();
+    }
+
+    public boolean isStarting() {
+      YarnApplicationState state = getState();
+      return state == YarnApplicationState.ACCEPTED
+          || state == YarnApplicationState.NEW
+          || state == YarnApplicationState.NEW_SAVING
+          || state == YarnApplicationState.SUBMITTED;
+    }
+
+    public boolean isStopped() {
+      YarnApplicationState state = getState();
+      return state == YarnApplicationState.FAILED
+          || state == YarnApplicationState.FINISHED
+          || state == YarnApplicationState.KILLED;
+    }
+
+    public boolean isRunning() {
+      YarnApplicationState state = getState();
+      return state == YarnApplicationState.RUNNING;
+    }
+  }
+
+  public static String getAmUrl(ApplicationReport report) {
+    return DoYUtil.unwrapAmUrl(report.getOriginalTrackingUrl());
+  }
+
+  @Override
+  public void run() throws ClientException {
+    YarnRMClient client = getClient();
+    System.out.println("Application ID: " + client.getAppId().toString());
+    Reporter reporter = new Reporter(client);
+    try {
+      reporter.getReport();
+    } catch (Exception e) {
+      removeAppIdFile();
+      System.out.println("Application is not running.");
+      return;
+    }
+    reporter.display(opts.verbose, false);
+    if (reporter.isRunning()) {
+      showAmStatus(reporter.report);
+    }
+  }
+
+  private void showAmStatus(ApplicationReport report) {
+    try {
+      String baseUrl = getAmUrl(report);
+      if (DoYUtil.isBlank(baseUrl)) {
+        return;
+      }
+      SimpleRestClient restClient = new SimpleRestClient();
+      String tail = "rest/status";
+      if (opts.verbose) {
+        System.out.println("Getting status with " + baseUrl + "/" + tail);
+      }
+      String result = restClient.send(baseUrl, tail, false);
+      formatResponse(result);
+      System.out.println("For more information, visit: " + baseUrl);
+    } catch (ClientException e) {
+      System.out.println("Failed to get AM status");
+      System.err.println(e.getMessage());
+    }
+  }
+
+  private void formatResponse(String result) {
+    JSONParser parser = new JSONParser();
+    Object status;
+    try {
+      status = parser.parse(result);
+    } catch (ParseException e) {
+      System.err.println("Invalid response received from AM");
+      if (opts.verbose) {
+        System.out.println(result);
+        System.out.println(e.getMessage());
+      }
+      return;
+    }
+    JSONObject root = (JSONObject) status;
+    showMetric("AM State", root, "state");
+    showMetric("Target Drillbit Count", root.get("summary"), "targetBitCount");
+    showMetric("Live Drillbit Count", root.get("summary"), "liveBitCount");
+    showMetric("Unmanaged Drillbit Count", root.get("summary"), "unmanagedCount");
+    showMetric("Blacklisted Node Count", root.get("summary"), "blackListCount");
+    showMetric("Free Node Count", root.get("summary"), "freeNodeCount");
+  }
+
+  private void showMetric(String label, Object object, String key) {
+    if (object == null) {
+      return;
+    }
+    if (!(object instanceof JSONObject)) {
+      return;
+    }
+    object = ((JSONObject) object).get(key);
+    if (object == null) {
+      return;
+    }
+    System.out.println(label + ": " + object.toString());
+  }
+}


Mime
View raw message