drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ar...@apache.org
Subject [06/12] drill git commit: DRILL-1170: YARN integration for Drill
Date Sun, 04 Mar 2018 17:13:46 GMT
http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AMSecurityManagerImpl.java
----------------------------------------------------------------------
diff --git a/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AMSecurityManagerImpl.java b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AMSecurityManagerImpl.java
new file mode 100644
index 0000000..d31690e
--- /dev/null
+++ b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AMSecurityManagerImpl.java
@@ -0,0 +1,221 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.yarn.appMaster.http;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.scanner.persistence.ScanResult;
+import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.exception.DrillbitStartupException;
+import org.apache.drill.exec.rpc.user.security.UserAuthenticationException;
+import org.apache.drill.exec.rpc.user.security.UserAuthenticator;
+import org.apache.drill.exec.rpc.user.security.UserAuthenticatorFactory;
+import org.apache.drill.exec.util.ImpersonationUtil;
+import org.apache.drill.yarn.appMaster.AMWrapperException;
+import org.apache.drill.yarn.core.DoYUtil;
+import org.apache.drill.yarn.core.DrillOnYarnConfig;
+
+import com.typesafe.config.Config;
+
+/**
+ * Implements the three supported AM security models: Drill,
+ * hard-coded user and password, and open access.
+ */
+
+public class AMSecurityManagerImpl implements AMSecurityManager {
+  private static final Log LOG = LogFactory.getLog(AMSecurityManagerImpl.class);
+
+  /**
+   * Thin layer around the Drill authentication system to adapt from
+   * Drill-on-YARN's environment to that expected by the Drill classes.
+   */
+  private static class DrillSecurityManager implements AMSecurityManager {
+    private UserAuthenticator authenticator;
+
+    @Override
+    public void init() {
+      try {
+        DrillOnYarnConfig doyConfig = DrillOnYarnConfig.instance();
+        DrillConfig config = doyConfig.getDrillConfig();
+        ScanResult classpathScan = doyConfig.getClassPathScan();
+        if (config.getBoolean(ExecConstants.USER_AUTHENTICATION_ENABLED)) {
+          authenticator = UserAuthenticatorFactory.createAuthenticator(config,
+              classpathScan);
+        } else {
+          authenticator = null;
+        }
+      } catch (DrillbitStartupException e) {
+        LOG.info("Authentication initialization failed", e);
+        throw new AMWrapperException("Security init failed", e);
+      }
+    }
+
+    @Override
+    public boolean login(String user, String password) {
+      if (authenticator == null) {
+        return true;
+      }
+      try {
+        authenticator.authenticate(user, password);
+      } catch (UserAuthenticationException e) {
+        LOG.info("Authentication failed for user " + user, e);
+        return false;
+      }
+      return ImpersonationUtil.getProcessUserName().equals(user);
+    }
+
+    @Override
+    public void close() {
+      try {
+        if (authenticator != null) {
+          authenticator.close();
+        }
+      } catch (IOException e) {
+        LOG.info("Ignoring error on authenticator close", e);
+      }
+    }
+
+    @Override
+    public boolean requiresLogin() {
+      return authenticator != null;
+    }
+  }
+
+  /**
+   * Simple security manager: user name and password reside in the DoY config
+   * file.
+   */
+
+  private static class SimpleSecurityManager implements AMSecurityManager {
+
+    private String userName;
+    private String password;
+
+    @Override
+    public void init() {
+      Config config = DrillOnYarnConfig.config();
+      userName = config.getString(DrillOnYarnConfig.HTTP_USER_NAME);
+      password = config.getString(DrillOnYarnConfig.HTTP_PASSWORD);
+      if (DoYUtil.isBlank(userName)) {
+        LOG.warn("Simple HTTP authentication is enabled, but "
+            + DrillOnYarnConfig.HTTP_USER_NAME + " is blank.");
+      }
+      if (DoYUtil.isBlank(userName)) {
+        LOG.warn("Simple HTTP authentication is enabled, but "
+            + DrillOnYarnConfig.HTTP_PASSWORD + " is blank.");
+      }
+    }
+
+    @Override
+    public boolean requiresLogin() {
+      return !DoYUtil.isBlank(userName);
+    }
+
+    @Override
+    public boolean login(String user, String pwd) {
+      if (!requiresLogin()) {
+        return true;
+      }
+      boolean ok = userName.equals(user) && password.equals(pwd);
+      if (!ok) {
+        LOG.info(
+            "Failed login attempt with simple authorization for user " + user);
+      }
+      return ok;
+    }
+
+    @Override
+    public void close() {
+      // Nothing to do
+    }
+
+  }
+
+  private static AMSecurityManagerImpl instance;
+
+  private AMSecurityManager managerImpl;
+
+  private AMSecurityManagerImpl() {
+  }
+
+  public static void setup() {
+    instance = new AMSecurityManagerImpl();
+    instance.init();
+  }
+
+  /**
+   * Look at the DoY config file to decide which security system (if any) to
+   * use.
+   */
+
+  @Override
+  public void init() {
+    Config config = DrillOnYarnConfig.config();
+    String authType = config.getString(DrillOnYarnConfig.HTTP_AUTH_TYPE);
+    if (DrillOnYarnConfig.AUTH_TYPE_DRILL.equals(authType)) {
+      // Drill authentication. Requires both DoY to select Drill
+      // auth, and for Drill's auth to be enabled.
+      if(config.getBoolean(ExecConstants.USER_AUTHENTICATION_ENABLED)) {
+        managerImpl = new DrillSecurityManager();
+        managerImpl.init();
+      }
+    } else if (DrillOnYarnConfig.AUTH_TYPE_SIMPLE.equals(authType)) {
+      managerImpl = new SimpleSecurityManager();
+      managerImpl.init();
+    } else if (DoYUtil.isBlank(authType)
+        || DrillOnYarnConfig.AUTH_TYPE_NONE.equals(authType)) {
+      ;
+    } else {
+      LOG.error("Unrecognized authorization type for "
+          + DrillOnYarnConfig.HTTP_AUTH_TYPE + ": " + authType
+          + " - assuming no auth.");
+    }
+  }
+
+  @Override
+  public boolean login(String user, String password) {
+    if (managerImpl == null) {
+      return true;
+    }
+    return managerImpl.login(user, password);
+  }
+
+  @Override
+  public void close() {
+    if (managerImpl != null) {
+      managerImpl.close();
+      managerImpl = null;
+    }
+  }
+
+  @Override
+  public boolean requiresLogin() {
+    return managerImpl != null;
+  }
+
+  public static AMSecurityManager instance() {
+    return instance;
+  }
+
+  public static boolean isEnabled() {
+    return instance != null && instance.managerImpl != null;
+  }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AbstractTasksModel.java
----------------------------------------------------------------------
diff --git a/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AbstractTasksModel.java b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AbstractTasksModel.java
new file mode 100644
index 0000000..bc4e8d6
--- /dev/null
+++ b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AbstractTasksModel.java
@@ -0,0 +1,380 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.yarn.appMaster.http;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
+import org.apache.drill.yarn.appMaster.ClusterController;
+import org.apache.drill.yarn.appMaster.ClusterControllerImpl;
+import org.apache.drill.yarn.appMaster.ControllerVisitor;
+import org.apache.drill.yarn.appMaster.Task;
+import org.apache.drill.yarn.appMaster.Task.TrackingState;
+import org.apache.drill.yarn.appMaster.TaskState;
+import org.apache.drill.yarn.appMaster.TaskVisitor;
+import org.apache.drill.yarn.core.DoYUtil;
+import org.apache.drill.yarn.core.DrillOnYarnConfig;
+import org.apache.drill.yarn.zk.ZKRegistry;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.Resource;
+
+public abstract class AbstractTasksModel {
+  public static class TaskModel {
+    public int id;
+    protected String groupName;
+    protected boolean isLive;
+    protected TaskState taskState;
+    protected String taskStateHint;
+    protected String state;
+    protected boolean cancelled;
+    protected String trackingState;
+    protected String trackingStateHint;
+    protected Container container;
+    protected DrillbitEndpoint endpoint;
+    protected long startTime;
+    protected int memoryMb;
+    protected int vcores;
+    protected double disks;
+    protected String containerId;
+    protected String nmLink;
+    protected long endTime;
+    protected String disposition;
+    protected int tryCount;
+
+    private Map<TaskState,String> stateHints = makeStateHints( );
+    private Map<TrackingState,String> trackingStateHints = makeTrackingStateHints( );
+
+    public TaskModel(Task task, boolean live) {
+      id = task.taskId;
+      groupName = task.scheduler.getName();
+      taskState = task.getState();
+      taskStateHint = stateHints.get(taskState);
+      state = taskState.getLabel();
+      cancelled = task.isCancelled();
+      isLive = live && taskState == TaskState.RUNNING;
+      TrackingState tState = task.getTrackingState();
+      trackingState = tState.getDisplayName();
+      trackingStateHint = trackingStateHints.get(tState);
+      container = task.container;
+      startTime = task.launchTime;
+      if (task.container != null) {
+        containerId = task.container.getId().toString();
+        Resource resource = task.container.getResource();
+        memoryMb = resource.getMemory();
+        vcores = resource.getVirtualCores();
+        disks = task.getContainerSpec().disks;
+
+        // Emulate the NM link. Used for debugging, gets us to
+        // the page on the NM UI for this container so we can see
+        // logs, etc.
+
+        nmLink = "http://" + task.container.getNodeHttpAddress();
+      } else {
+        memoryMb = task.scheduler.getResource().memoryMb;
+        vcores = task.scheduler.getResource().vCores;
+      }
+      endpoint = (DrillbitEndpoint) task.properties
+          .get(ZKRegistry.ENDPOINT_PROPERTY);
+      if (!live) {
+        endTime = task.completionTime;
+        tryCount = task.tryCount;
+
+        // Determine disposition from most general to most
+        // specific sources of information.
+
+        disposition = state;
+        if (task.disposition != null) {
+          disposition = task.disposition.toString();
+        }
+        if (task.completionStatus != null) {
+          disposition = reformatDiagnostics( task.completionStatus.getDiagnostics() );
+        }
+        if (task.error != null) {
+          disposition = task.error.getMessage();
+        }
+      }
+    }
+
+    private enum FormatState { PRE_STACK, IN_STACK, POST_STACK };
+
+    /**
+     * YARN diagnostics are verbose: they contain a stack trace of the YARN node
+     * manager thread (not Drill), and contain blank lines, the container ID,
+     * etc. Remove unnecessary cruft to make the diagnostics simpler and smaller
+     * in the Web UI.
+     *
+     * @param orig YARN diagnostics
+     * @return cleaned-up version.
+     */
+
+    public static String reformatDiagnostics( String orig ) {
+      try {
+        StringBuilder buf = new StringBuilder( );
+        BufferedReader reader = new BufferedReader( new StringReader( orig ) );
+        String line;
+        FormatState state = FormatState.PRE_STACK;
+        while ( (line = reader.readLine()) != null ) {
+          switch( state ) {
+          case PRE_STACK:
+            if ( line.startsWith( "Container id:") ) {
+              continue;
+            }
+            if ( line.startsWith( "Stack trace:" ) ) {
+              state = FormatState.IN_STACK;
+              continue;
+            }
+            break;
+          case IN_STACK:
+            if ( line.trim().isEmpty() ) {
+              state = FormatState.POST_STACK;
+            }
+            continue;
+          case POST_STACK:
+          default:
+            break;
+          }
+          if ( line.trim().isEmpty() ) {
+            continue;
+          }
+          buf.append( line );
+          buf.append( "<br/>\n" );
+        }
+        buf.append( "See log file for details." );
+        return buf.toString();
+      } catch (IOException e) {
+        // Will never occur. But, if the impossible happens, just return
+        // the original diagnostics.
+
+        return orig.replace("\n", "<br>\n");
+      }
+    }
+
+    private Map<TaskState, String> makeStateHints() {
+      Map<TaskState, String> hints = new HashMap<>();
+      hints.put(TaskState.START, "Queued to send a container request to YARN.");
+      hints.put(TaskState.REQUESTING, "Container request sent to YARN.");
+      hints.put(TaskState.LAUNCHING,
+          "YARN provided a container, send launch request.");
+      hints.put(TaskState.WAIT_START_ACK,
+          "Drillbit launched, waiting for ZooKeeper registration.");
+      hints.put(TaskState.RUNNING, "Drillbit is running normally.");
+      hints.put(TaskState.ENDING,
+          "Graceful shutdown request sent to the Drillbit.");
+      hints.put(TaskState.KILLING,
+          "Sent the YARN Node Manager a request to forcefully kill the Drillbit.");
+      hints.put(TaskState.WAIT_END_ACK,
+          "Drillbit has shut down; waiting for ZooKeeper to confirm.");
+      // The UI will never display the END state.
+      hints.put(TaskState.END, "The Drillbit has shut down.");
+      return hints;
+    }
+
+    private Map<TrackingState, String> makeTrackingStateHints() {
+      Map<TrackingState, String> hints = new HashMap<>();
+      // UNTRACKED state is not used by Drillbits.
+      hints.put(TrackingState.UNTRACKED, "Task is not tracked in ZooKeeper.");
+      hints.put(TrackingState.NEW,
+          "Drillbit has not yet registered with ZooKeeper.");
+      hints.put(TrackingState.START_ACK,
+          "Drillbit has registered normally with ZooKeeper.");
+      hints.put(TrackingState.END_ACK,
+          "Drillbit is no longer registered with ZooKeeper.");
+      return hints;
+    }
+
+    public String getTaskId() {
+      return Integer.toString(id);
+    }
+
+    public String getGroupName( ) { return groupName; }
+
+    public boolean isLive( ) {
+      return isLive;
+    }
+
+    public String getHost( ) {
+      if ( container == null ) {
+        return ""; }
+      return container.getNodeId().getHost();
+    }
+
+    public String getLink( ) {
+      if ( endpoint == null ) {
+        return ""; }
+      String port = DrillOnYarnConfig.config( ).getString( DrillOnYarnConfig.DRILLBIT_HTTP_PORT );
+      String protocol = "http:";
+      if ( DrillOnYarnConfig.config().getBoolean( DrillOnYarnConfig.DRILLBIT_USE_HTTPS ) ) {
+        protocol = "https:";
+      }
+      return protocol + "//" + endpoint.getAddress() + ":" + port + "/";
+    }
+
+    public String getState( ) { return state.toString(); }
+    public String getStateHint( ) { return taskStateHint; }
+    public boolean isCancelled( ) { return cancelled; }
+
+    public boolean isCancellable( ) {
+      return ! cancelled  &&  taskState.isCancellable( );
+    }
+
+    public String getTrackingState( ) { return trackingState; }
+    public String getTrackingStateHint( ) { return trackingStateHint; }
+
+    public String getStartTime( ) {
+      if ( startTime == 0 ) {
+        return ""; }
+      return DoYUtil.toIsoTime( startTime );
+    }
+
+    public int getMemory( ) { return memoryMb; }
+    public int getVcores( ) { return vcores; }
+    public String getDisks( ) {
+      return String.format( "%.2f", disks );
+    }
+    public boolean hasContainer( ) { return containerId != null; }
+    public String getContainerId( ) { return displayString( containerId ); }
+    public String getNmLink( ) { return displayString( nmLink ); }
+    public String getDisposition( ) { return displayString( disposition ); }
+    public int getTryCount( ) { return tryCount; }
+    public String displayString( String value ) { return (value == null) ? "" : value; }
+
+    public String getEndTime( ) {
+      if ( endTime == 0 ) {
+        return ""; }
+      return DoYUtil.toIsoTime( endTime );
+    }
+  }
+
+  public static class UnmanagedDrillbitModel
+  {
+    protected String host;
+    protected String ports;
+
+    public UnmanagedDrillbitModel( String endpoint ) {
+      String parts[] = endpoint.split( ":" );
+      if ( parts.length < 4 ) {
+        // Should never occur, but better save than sorry.
+
+        host = endpoint;
+        ports = "";
+      }
+      else {
+        host = parts[0];
+        List<String> thePorts = new ArrayList<>( );
+        thePorts.add( parts[1] );
+        thePorts.add( parts[2] );
+        thePorts.add( parts[3] );
+        ports = DoYUtil.join( ", ", thePorts );
+      }
+    }
+
+    public String getHost( ) { return host; }
+    public String getPorts( ) { return ports; }
+  }
+
+  protected boolean supportsDisks;
+  protected List<TaskModel> results = new ArrayList<>( );
+
+  public List<TaskModel> getTasks( ) { return results; }
+  public boolean hasTasks( ) { return ! results.isEmpty(); }
+  public boolean supportsDiskResource( ) { return supportsDisks; }
+
+  public static class TasksModel extends AbstractTasksModel implements TaskVisitor
+  {
+    protected List<UnmanagedDrillbitModel> unmanaged;
+    protected List<String> blacklist;
+
+    @Override
+    public void visit(Task task) {
+      results.add( new TaskModel( task, true ) );
+    }
+
+    /**
+     * Sort tasks by Task ID.
+     */
+
+    public void sortTasks() {
+      Collections.sort( results, new Comparator<TaskModel>( ) {
+        @Override
+        public int compare(TaskModel t1, TaskModel t2) {
+          return Integer.compare( t1.id, t2.id );
+        }
+      });
+    }
+
+    /**
+     * List any anomalies: either stray Drillbits (those in ZK but not launched by DoY),
+     * or blacklisted nodes.
+     * <p>
+     * To avoid race conditions, do not use the controller visitor to invoke this method,
+     * we want to leave the controller unlocked and instead lock only the ZK registry.
+     *
+     * @param controller
+     */
+
+    public void listAnomalies(ClusterController controller) {
+        listUnmanaged(controller);
+        synchronized( controller ) {
+          blacklist = ((ClusterControllerImpl) controller).getNodeInventory().getBlacklist();
+        }
+        Collections.sort( blacklist );
+    }
+
+    private void listUnmanaged(ClusterController controller) {
+      ZKRegistry zkRegistry = (ZKRegistry) controller.getProperty( ZKRegistry.CONTROLLER_PROPERTY );
+      if ( zkRegistry == null ) {
+        return;
+      }
+      List<String> endpoints = zkRegistry.listUnmanagedDrillits( );
+      if ( endpoints.isEmpty() ) {
+        return; }
+      unmanaged = new ArrayList<>( );
+      for ( String endpoint : endpoints ) {
+        unmanaged.add( new UnmanagedDrillbitModel( endpoint ) );
+      }
+    }
+
+    public List<UnmanagedDrillbitModel>getUnnamaged( ) { return unmanaged; }
+    public boolean hasUnmanagedDrillbits( ) { return unmanaged != null; }
+    public int getUnmanagedDrillbitCount( ) {
+      return (unmanaged == null) ? 0 : unmanaged.size( );
+    }
+    public boolean hasBlacklist( ) { return ! blacklist.isEmpty(); }
+    public int getBlacklistCount( ) { return blacklist.size( ); }
+    public List<String> getBlacklist( ) { return blacklist; }
+  }
+
+  public static class HistoryModel extends AbstractTasksModel implements ControllerVisitor
+  {
+    @Override
+    public void visit(ClusterController controller) {
+      ClusterControllerImpl impl = (ClusterControllerImpl) controller;
+      for ( Task task : impl.getHistory( ) ) {
+        results.add( new TaskModel( task, false ) );
+      }
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AmRestApi.java
----------------------------------------------------------------------
diff --git a/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AmRestApi.java b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AmRestApi.java
new file mode 100644
index 0000000..21ddc4b
--- /dev/null
+++ b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AmRestApi.java
@@ -0,0 +1,296 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.yarn.appMaster.http;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import javax.annotation.security.PermitAll;
+import javax.ws.rs.DefaultValue;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.MediaType;
+
+import org.apache.drill.yarn.appMaster.Dispatcher;
+import org.apache.drill.yarn.appMaster.http.AbstractTasksModel.TaskModel;
+import org.apache.drill.yarn.appMaster.http.ControllerModel.ClusterGroupModel;
+import org.apache.drill.yarn.core.DoYUtil;
+import org.apache.drill.yarn.core.DrillOnYarnConfig;
+import org.apache.drill.yarn.core.NameValuePair;
+import org.apache.drill.yarn.zk.ZKClusterCoordinatorDriver;
+
+public class AmRestApi extends PageTree
+{
+  @Path("/config")
+  @PermitAll
+  public static class ConfigResource
+  {
+    @GET
+    @Produces(MediaType.APPLICATION_JSON)
+    public Map<String,Object> getConfig( ) {
+      Map<String, Object> map = new HashMap<>();
+      for (NameValuePair pair : DrillOnYarnConfig.instance().getPairs()) {
+        map.put(pair.getName(), pair.getValue());
+      }
+      return map;
+    }
+  }
+
+  /**
+   * Returns cluster status as a tree of JSON objects. Done as explicitly-defined
+   * maps to specify the key names (which must not change to avoid breaking
+   * compatibility) and to handle type conversions.
+   */
+
+  @Path("/status")
+  @PermitAll
+  public static class StatusResource
+  {
+    @GET
+    @Produces(MediaType.APPLICATION_JSON)
+    public Map<String,Object> getStatus( ) {
+      ControllerModel model = new ControllerModel( );
+      dispatcher.getController().visit( model );
+
+      Map<String,Object> root = new HashMap<>( );
+      root.put( "state", model.state.toString() );
+
+      Map<String, Object> summary = new HashMap<>();
+      summary.put("drillMemoryMb", model.totalDrillMemory);
+      summary.put("drillVcores", model.totalDrillVcores);
+      summary.put("yarnMemoryMb", model.yarnMemory);
+      summary.put("yarnVcores", model.yarnVcores);
+      summary.put("liveBitCount", model.liveCount);
+      summary.put("totalBitCount", model.taskCount);
+      summary.put("targetBitCount", model.targetCount);
+      summary.put("unmanagedCount", model.getUnmanagedCount());
+      summary.put("blackListCount", model.getBlacklistCount());
+      summary.put("freeNodeCount", model.getFreeNodeCount());
+      root.put("summary", summary);
+
+      List<Map<String, Object>> pools = new ArrayList<>();
+      for (ClusterGroupModel pool : model.groups) {
+        Map<String, Object> poolObj = new HashMap<>();
+        poolObj.put("name", pool.name);
+        poolObj.put("type", pool.type);
+        poolObj.put("liveBitCount", pool.liveCount);
+        poolObj.put("targetBitCount", pool.targetCount);
+        poolObj.put("totalBitCount", pool.taskCount);
+        poolObj.put("totalMemoryMb", pool.memory);
+        poolObj.put("totalVcores", pool.vcores);
+        pools.add(poolObj);
+      }
+      root.put("pools", pools);
+
+      AbstractTasksModel.TasksModel tasksModel = new AbstractTasksModel.TasksModel();
+      dispatcher.getController().visitTasks(tasksModel);
+      List<Map<String, Object>> bits = new ArrayList<>();
+      for (TaskModel task : tasksModel.results) {
+        Map<String, Object> bitObj = new HashMap<>();
+        bitObj.put("containerId", task.container.getId().toString());
+        bitObj.put("host", task.getHost());
+        bitObj.put("id", task.id);
+        bitObj.put("live", task.isLive());
+        bitObj.put("memoryMb", task.memoryMb);
+        bitObj.put("vcores", task.vcores);
+        bitObj.put("pool", task.groupName);
+        bitObj.put("state", task.state);
+        bitObj.put("trackingState", task.trackingState);
+        bitObj.put("endpoint",
+            ZKClusterCoordinatorDriver.asString(task.endpoint));
+        bitObj.put("link", task.getLink());
+        bitObj.put("startTime", task.getStartTime());
+        bits.add(bitObj);
+      }
+      root.put("drillbits", bits);
+
+      return root;
+    }
+  }
+
+  /**
+   * Stop the cluster. Uses a key to validate the request. The value of the key is
+   * set in the Drill-on-YARN configuration file. The purpose is simply to prevent
+   * accidental cluster shutdown when experimenting with the REST API; this is
+   * not meant to be a security mechanism.
+   *
+   * @param key
+   * @return
+   */
+
+  @Path("/stop")
+  @PermitAll
+  public static class StopResource
+  {
+    @DefaultValue( "" )
+    @QueryParam( "key" )
+    String key;
+
+    @POST
+    @Produces(MediaType.APPLICATION_JSON)
+    public Map<String,String> postStop( )
+    {
+      Map<String, String> error = checkKey(key);
+      if (error != null) {
+        return error;
+      }
+
+      dispatcher.getController().shutDown();
+      return successResponse("Shutting down");
+    }
+  }
+
+  @Path("/resize/{quantity}")
+  @PermitAll
+  public static class ResizeResource
+  {
+    @PathParam(value = "quantity")
+    String quantity;
+    @DefaultValue( "" )
+    @QueryParam( "key" )
+    String key;
+
+    @POST
+    @Produces(MediaType.APPLICATION_JSON)
+    public Map<String,String> postResize( )
+    {
+      ResizeRequest request = new ResizeRequest(key, quantity);
+      if (request.error != null) {
+        return request.error;
+      }
+
+      int curSize = dispatcher.getController().getTargetCount();
+      dispatcher.getController().resizeTo(request.n);
+      return successResponse("Resizing from " + curSize + " to " + request.n);
+    }
+  }
+
+  protected static class ResizeRequest
+  {
+    Map<String,String> error;
+    int n;
+
+    public ResizeRequest( String key, String quantity ) {
+      error = checkKey(key);
+      if (error != null) {
+        return;
+      }
+      try {
+        n = Integer.parseInt(quantity);
+      } catch (NumberFormatException e) {
+        error = errorResponse("Invalid argument: " + quantity);
+      }
+      if (n < 0) {
+        error = errorResponse("Invalid argument: " + quantity);
+      }
+    }
+  }
+
+  @Path("/grow/{quantity}")
+  @PermitAll
+  public static class GrowResource
+  {
+    @PathParam(value = "quantity")
+    @DefaultValue( "1" )
+    String quantity;
+    @DefaultValue( "" )
+    @QueryParam( "key" )
+    String key;
+
+    @POST
+    @Produces(MediaType.APPLICATION_JSON)
+    public Map<String,String> postResize( )
+    {
+      ResizeRequest request = new ResizeRequest(key, quantity);
+      if (request.error != null) {
+        return request.error;
+      }
+
+      int curSize = dispatcher.getController().getTargetCount();
+      int newSize = curSize + request.n;
+      dispatcher.getController().resizeTo(newSize);
+      return successResponse("Growing by " + request.n + " to " + newSize);
+    }
+  }
+
+  @Path("/shrink/{quantity}")
+  @PermitAll
+  public static class ShrinkResource
+  {
+    @PathParam(value = "quantity")
+    @DefaultValue( "1" )
+    String quantity;
+    @DefaultValue( "" )
+    @QueryParam( "key" )
+    String key;
+
+    @POST
+    @Produces(MediaType.APPLICATION_JSON)
+    public Map<String,String> postResize( )
+    {
+      ResizeRequest request = new ResizeRequest(key, quantity);
+      if (request.error != null) {
+        return request.error;
+      }
+      int curSize = dispatcher.getController().getTargetCount();
+      int newSize = Math.max(curSize - request.n, 0);
+      dispatcher.getController().resizeTo(newSize);
+      return successResponse("Shrinking by " + request.n + " to " + newSize);
+    }
+  }
+
+  private static Map<String, String> checkKey(String key) {
+    String masterKey = DrillOnYarnConfig.config()
+        .getString(DrillOnYarnConfig.HTTP_REST_KEY);
+    if (!DoYUtil.isBlank(masterKey) && !masterKey.equals(key)) {
+      return errorResponse("Invalid Key");
+    }
+    return null;
+  }
+
+  private static Map<String, String> errorResponse(String msg) {
+    Map<String, String> resp = new HashMap<>();
+    resp.put("status", "error");
+    resp.put("message", msg);
+    return resp;
+  }
+
+  private static Map<String, String> successResponse(String msg) {
+    Map<String, String> resp = new HashMap<>();
+    resp.put("status", "ok");
+    resp.put("message", msg);
+    return resp;
+  }
+
+  public AmRestApi(Dispatcher dispatcher) {
+    super(dispatcher);
+
+    register(ConfigResource.class);
+    register(StatusResource.class);
+    register(StopResource.class);
+    register(ResizeResource.class);
+    register(GrowResource.class);
+    register(ShrinkResource.class);
+  }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AuthDynamicFeature.java
----------------------------------------------------------------------
diff --git a/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AuthDynamicFeature.java b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AuthDynamicFeature.java
new file mode 100644
index 0000000..55cd59a
--- /dev/null
+++ b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AuthDynamicFeature.java
@@ -0,0 +1,114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.yarn.appMaster.http;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.drill.yarn.appMaster.http.WebUiPageTree.LogInLogOutPages;
+import org.glassfish.jersey.server.model.AnnotatedMethod;
+
+import javax.annotation.Priority;
+import javax.annotation.security.PermitAll;
+import javax.annotation.security.RolesAllowed;
+import javax.ws.rs.Priorities;
+import javax.ws.rs.container.ContainerRequestContext;
+import javax.ws.rs.container.ContainerRequestFilter;
+import javax.ws.rs.container.DynamicFeature;
+import javax.ws.rs.container.ResourceInfo;
+import javax.ws.rs.core.FeatureContext;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.SecurityContext;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URLEncoder;
+
+/**
+ * Implementation of {@link DynamicFeature}. As part of the setup it adds the
+ * auth check filter {@link AuthCheckFilter} for resources that need to have
+ * user authenticated. If authentication is not done, request is forwarded to
+ * login page.
+ * <p>
+ * Shameless copy of
+ * {@link org.apache.drill.exec.server.rest.auth.DynamicFeature}; the two
+ * implementations should be merged at some point. The difference is only the
+ * log in/log out constant references.
+ */
+
+public class AuthDynamicFeature implements DynamicFeature {
+  private static final Log LOG = LogFactory.getLog(AuthDynamicFeature.class);
+
+  @Override
+  public void configure(final ResourceInfo resourceInfo,
+      final FeatureContext configuration) {
+    AnnotatedMethod am = new AnnotatedMethod(resourceInfo.getResourceMethod());
+
+    // RolesAllowed on the method takes precedence over PermitAll
+    RolesAllowed ra = am.getAnnotation(RolesAllowed.class);
+    if (ra != null) {
+      configuration.register(AuthCheckFilter.INSTANCE);
+      return;
+    }
+
+    // PermitAll takes precedence over RolesAllowed on the class
+    if (am.isAnnotationPresent(PermitAll.class)) {
+      // Do nothing.
+      return;
+    }
+
+    // RolesAllowed on the class takes precedence over PermitAll
+    ra = resourceInfo.getResourceClass().getAnnotation(RolesAllowed.class);
+    if (ra != null) {
+      configuration.register(AuthCheckFilter.INSTANCE);
+    }
+  }
+
+  @Priority(Priorities.AUTHENTICATION) // authentication filter - should go
+                                       // first before all other filters.
+  private static class AuthCheckFilter implements ContainerRequestFilter {
+    private static AuthCheckFilter INSTANCE = new AuthCheckFilter();
+
+    @Override
+    public void filter(ContainerRequestContext requestContext)
+        throws IOException {
+      final SecurityContext sc = requestContext.getSecurityContext();
+      if (!isUserLoggedIn(sc)) {
+        try {
+          final String destResource = URLEncoder.encode(
+              requestContext.getUriInfo().getRequestUri().toString(), "UTF-8");
+          final URI loginURI = requestContext.getUriInfo().getBaseUriBuilder()
+              .path(LogInLogOutPages.LOGIN_RESOURCE)
+              .queryParam(LogInLogOutPages.REDIRECT_QUERY_PARM, destResource)
+              .build();
+          requestContext
+              .abortWith(Response.temporaryRedirect(loginURI).build());
+        } catch (final Exception ex) {
+          final String errMsg = String.format(
+              "Failed to forward the request to login page: %s",
+              ex.getMessage());
+          LOG.error(errMsg, ex);
+          requestContext
+              .abortWith(Response.serverError().entity(errMsg).build());
+        }
+      }
+    }
+  }
+
+  public static boolean isUserLoggedIn(final SecurityContext sc) {
+    return sc != null && sc.getUserPrincipal() != null;
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/ControllerModel.java
----------------------------------------------------------------------
diff --git a/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/ControllerModel.java b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/ControllerModel.java
new file mode 100644
index 0000000..8947df5
--- /dev/null
+++ b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/ControllerModel.java
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.yarn.appMaster.http;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import javax.xml.bind.annotation.XmlRootElement;
+
+import org.apache.drill.yarn.appMaster.AMYarnFacade.YarnAppHostReport;
+import org.apache.drill.yarn.appMaster.ClusterController;
+import org.apache.drill.yarn.appMaster.ClusterControllerImpl;
+import org.apache.drill.yarn.appMaster.ClusterControllerImpl.State;
+import org.apache.drill.yarn.appMaster.ControllerVisitor;
+import org.apache.drill.yarn.appMaster.Scheduler;
+import org.apache.drill.yarn.appMaster.SchedulerStateActions;
+import org.apache.drill.yarn.core.ContainerRequestSpec;
+import org.apache.drill.yarn.core.DrillOnYarnConfig;
+import org.apache.drill.yarn.zk.ZKRegistry;
+
+import com.typesafe.config.Config;
+
+@XmlRootElement
+public class ControllerModel implements ControllerVisitor {
+  public static class ClusterGroupModel {
+    protected String name;
+    protected String type;
+    protected int targetCount;
+    protected int taskCount;
+    protected int liveCount;
+    protected int memory;
+    protected int vcores;
+    protected double disks;
+
+    public String getName( ) { return name; }
+    public String getType( ) { return type; }
+    public int getTargetCount( ) { return targetCount; }
+    public int getTaskCount( ) { return taskCount; }
+    public int getLiveCount( ) { return liveCount; }
+    public int getMemory( ) { return memory; }
+    public int getVcores( ) { return vcores; }
+    public String getDisks( ) {
+      return String.format( "%.02f", disks );
+    }
+  }
+
+  protected String zkConnectStr;
+  protected String zkRoot;
+  protected String zkClusterId;
+  protected ClusterControllerImpl.State state;
+  protected String stateHint;
+  protected boolean supportsDisks;
+  protected int yarnMemory;
+  protected int yarnVcores;
+  protected int yarnNodeCount;
+  protected int taskCount;
+  protected int liveCount;
+  protected int unmanagedCount;
+  protected int targetCount;
+  protected int totalDrillMemory;
+  protected int totalDrillVcores;
+  protected double totalDrillDisks;
+  protected int blacklistCount;
+  protected int freeNodeCount;
+  protected YarnAppHostReport appRpt;
+  protected int refreshSecs;
+  protected List<ClusterGroupModel> groups = new ArrayList<>( );
+
+  public boolean supportsDiskResource( ) { return supportsDisks; }
+  public int getRefreshSecs( ) { return refreshSecs; }
+  public String getZkConnectionStr( ) { return zkConnectStr; }
+  public String getZkRoot( ) { return zkRoot; }
+  public String getZkClusterId( ) { return zkClusterId; }
+  public String getAppId( ) { return appRpt.appId; }
+  public String getRmHost( ) { return appRpt.rmHost; }
+  public String getRmLink( ) { return appRpt.rmUrl; }
+  public String getNmHost( ) { return appRpt.nmHost; }
+  public String getNmLink( ) { return appRpt.nmUrl; }
+  public String getRmAppLink( ) { return appRpt.rmAppUrl; }
+  public String getNmAppLink( ) { return appRpt.nmAppUrl; }
+  public String getState( ) { return state.toString( ); }
+  public String getStateHint( ) { return stateHint; }
+  public int getYarnMemory( ) { return yarnMemory; }
+  public int getYarnVcores( ) { return yarnVcores; }
+  public int getDrillTotalMemory( ) { return totalDrillMemory; }
+  public int getDrillTotalVcores( ) { return totalDrillVcores; }
+  public String getDrillTotalDisks( ) {
+    return String.format( "%.2f", totalDrillDisks );
+  }
+  public int getYarnNodeCount( ) { return yarnNodeCount; }
+  public int getTaskCount( ) { return taskCount; }
+  public int getLiveCount( ) { return liveCount; }
+  public int getUnmanagedCount( ) { return unmanagedCount; }
+  public int getTargetCount( ) { return targetCount; }
+  public List<ClusterGroupModel> getGroups( ) { return groups; }
+  public int getBlacklistCount( ) { return blacklistCount; }
+  public int getFreeNodeCount( ) { return freeNodeCount; }
+
+  private static Map<ClusterControllerImpl.State,String> stateHints = makeStateHints( );
+
+  @Override
+  public void visit(ClusterController controller) {
+    Config config = DrillOnYarnConfig.config();
+    refreshSecs = config.getInt( DrillOnYarnConfig.HTTP_REFRESH_SECS );
+    zkConnectStr = config.getString( DrillOnYarnConfig.ZK_CONNECT );
+    zkRoot = config.getString( DrillOnYarnConfig.ZK_ROOT );
+    zkClusterId = config.getString( DrillOnYarnConfig.CLUSTER_ID );
+
+    ClusterControllerImpl impl = (ClusterControllerImpl) controller;
+    appRpt = impl.getYarn().getAppHostReport();
+
+    state = impl.getState( );
+    stateHint = stateHints.get( state );
+
+    // Removed based on feedback. Users should check the
+    // YARN RM UI instead.
+
+//    if ( state == State.LIVE ) {
+//      RegisterApplicationMasterResponse resp = impl.getYarn( ).getRegistrationResponse();
+//      yarnVcores = resp.getMaximumResourceCapability().getVirtualCores();
+//      yarnMemory = resp.getMaximumResourceCapability().getMemory();
+//      yarnNodeCount = impl.getYarn( ).getNodeCount();
+//    }
+    capturePools( impl );
+    supportsDisks = impl.supportsDiskResource();
+
+    blacklistCount = impl.getNodeInventory( ).getBlacklist( ).size( );
+    freeNodeCount = impl.getFreeNodeCount();
+  }
+
+  private void capturePools(ClusterControllerImpl impl) {
+    for ( SchedulerStateActions pool : impl.getPools( ) ) {
+      ControllerModel.ClusterGroupModel poolModel = new ControllerModel.ClusterGroupModel( );
+      Scheduler sched = pool.getScheduler();
+      ContainerRequestSpec containerSpec = sched.getResource( );
+      poolModel.name = sched.getName();
+      poolModel.type = sched.getType( );
+      poolModel.targetCount = sched.getTarget();
+      poolModel.memory = containerSpec.memoryMb;
+      poolModel.vcores = containerSpec.vCores;
+      poolModel.disks = containerSpec.disks;
+      poolModel.taskCount = pool.getTaskCount();
+      poolModel.liveCount = pool.getLiveCount( );
+      targetCount += poolModel.targetCount;
+      taskCount += poolModel.taskCount;
+      liveCount += poolModel.liveCount;
+      totalDrillMemory += poolModel.liveCount * poolModel.memory;
+      totalDrillVcores += poolModel.liveCount * poolModel.vcores;
+      totalDrillDisks += poolModel.liveCount * poolModel.disks;
+      groups.add( poolModel );
+    }
+    if ( state != State.LIVE ) {
+      targetCount = 0;
+    }
+  }
+
+  /**
+   * Count the unmanaged drillbits. Do this as a separate call, not via the
+   * {@link #visit(ClusterController) visit} method, to avoid locking both
+   * the cluster controller and ZK registry.
+   *
+   * @param controller
+   */
+
+  public void countStrayDrillbits(ClusterController controller) {
+    ZKRegistry zkRegistry = (ZKRegistry) controller.getProperty( ZKRegistry.CONTROLLER_PROPERTY );
+    if ( zkRegistry != null ) {
+      unmanagedCount = zkRegistry.listUnmanagedDrillits().size();
+    }
+  }
+
+  /**
+   * Create a table of user-visible descriptions for each controller state.
+   *
+   * @return
+   */
+
+  private static Map<State, String> makeStateHints() {
+    Map<ClusterControllerImpl.State,String> hints = new HashMap<>( );
+    // UI likely will never display the FAILED state.
+    hints.put( ClusterControllerImpl.State.START, "AM is starting up." );
+    hints.put( ClusterControllerImpl.State.LIVE, "AM is operating normally." );
+    hints.put( ClusterControllerImpl.State.ENDING, "AM is shutting down." );
+    // UI will never display the ENDED state.
+    hints.put( ClusterControllerImpl.State.ENDED, "AM is about to exit." );
+    // UI will never display the FAILED state.
+    hints.put( ClusterControllerImpl.State.FAILED, "AM failed to start and is about to exit." );
+    return hints;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/PageTree.java
----------------------------------------------------------------------
diff --git a/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/PageTree.java b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/PageTree.java
new file mode 100644
index 0000000..e4d5dc1
--- /dev/null
+++ b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/PageTree.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.yarn.appMaster.http;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.ws.rs.core.SecurityContext;
+
+import org.apache.drill.exec.server.rest.auth.DrillUserPrincipal;
+import org.apache.drill.yarn.appMaster.Dispatcher;
+import org.apache.drill.yarn.core.DrillOnYarnConfig;
+import org.glassfish.jersey.server.ResourceConfig;
+
+import com.typesafe.config.Config;
+
+/**
+ * Base class for a tree of web pages (or REST resources) represented
+ * as POJOs. Since the AM web UI is simple, this is the most convenient,
+ * compact way to implement the UI.
+ */
+
+public class PageTree extends ResourceConfig {
+  // These items are a bit clumsy. We need them, but we can't make them
+  // instance variables without a bunch of messiness in the page classes.
+  // So, we let them be static. No harm in setting them multiple times.
+
+  static Dispatcher dispatcher;
+  static Config config;
+
+  public PageTree(Dispatcher dispatcher) {
+    PageTree.dispatcher = dispatcher;
+    config = DrillOnYarnConfig.config();
+  }
+
+  /**
+   * Creates a FreeMarker model that contains two top-level items:
+   * the model itself (as in the default implementation) and the
+   * cluster name (used as a title on each UI page.)
+   *
+   * @param base
+   * @return
+   */
+
+  public static Map<String, Object> toModel(SecurityContext sc, Object base) {
+    Map<String, Object> model = new HashMap<>();
+    model.put("model", base);
+    return toMapModel(sc, model);
+  }
+
+  public static Map<String, Object> toMapModel(SecurityContext sc,
+      Map<String, Object> model) {
+    model.put("clusterName", config.getString(DrillOnYarnConfig.APP_NAME));
+    boolean useAuth = AMSecurityManagerImpl.isEnabled();
+    final boolean isUserLoggedIn = (useAuth)
+        ? AuthDynamicFeature.isUserLoggedIn(sc) : false;
+    model.put("showLogin", useAuth && !isUserLoggedIn);
+    model.put("showLogout", isUserLoggedIn);
+    model.put("docsLink", config.getString(DrillOnYarnConfig.HTTP_DOCS_LINK));
+    String userName = isUserLoggedIn ? sc.getUserPrincipal().getName()
+        : DrillUserPrincipal.ANONYMOUS_USER;
+    model.put("loggedInUserName", userName);
+    return model;
+  }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/WebServer.java
----------------------------------------------------------------------
diff --git a/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/WebServer.java b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/WebServer.java
new file mode 100644
index 0000000..aeeafde
--- /dev/null
+++ b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/WebServer.java
@@ -0,0 +1,467 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.yarn.appMaster.http;
+
+import static org.apache.drill.exec.server.rest.auth.DrillUserPrincipal.ADMIN_ROLE;
+
+import java.math.BigInteger;
+import java.security.KeyPair;
+import java.security.KeyPairGenerator;
+import java.security.KeyStore;
+import java.security.Principal;
+import java.security.SecureRandom;
+import java.security.cert.X509Certificate;
+import java.util.Collections;
+import java.util.Date;
+import java.util.Set;
+
+import javax.servlet.http.HttpSession;
+import javax.servlet.http.HttpSessionEvent;
+import javax.servlet.http.HttpSessionListener;
+
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.drill.yarn.appMaster.Dispatcher;
+import org.apache.drill.yarn.core.DrillOnYarnConfig;
+import org.bouncycastle.asn1.x500.X500NameBuilder;
+import org.bouncycastle.asn1.x500.style.BCStyle;
+import org.bouncycastle.cert.X509v3CertificateBuilder;
+import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter;
+import org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder;
+import org.bouncycastle.operator.ContentSigner;
+import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder;
+import org.eclipse.jetty.http.HttpVersion;
+import org.eclipse.jetty.security.ConstraintMapping;
+import org.eclipse.jetty.security.ConstraintSecurityHandler;
+import org.eclipse.jetty.security.DefaultIdentityService;
+import org.eclipse.jetty.security.DefaultUserIdentity;
+import org.eclipse.jetty.security.IdentityService;
+import org.eclipse.jetty.security.LoginService;
+import org.eclipse.jetty.security.SecurityHandler;
+import org.eclipse.jetty.security.authentication.FormAuthenticator;
+import org.eclipse.jetty.security.authentication.SessionAuthentication;
+import org.eclipse.jetty.server.HttpConfiguration;
+import org.eclipse.jetty.server.HttpConnectionFactory;
+import org.eclipse.jetty.server.SecureRequestCustomizer;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.server.ServerConnector;
+import org.eclipse.jetty.server.SessionManager;
+import org.eclipse.jetty.server.SslConnectionFactory;
+import org.eclipse.jetty.server.UserIdentity;
+import org.eclipse.jetty.server.handler.ErrorHandler;
+import org.eclipse.jetty.server.session.HashSessionManager;
+import org.eclipse.jetty.server.session.SessionHandler;
+import org.eclipse.jetty.servlet.DefaultServlet;
+import org.eclipse.jetty.servlet.ServletContextHandler;
+import org.eclipse.jetty.servlet.ServletHolder;
+import org.eclipse.jetty.util.resource.Resource;
+import org.eclipse.jetty.util.ssl.SslContextFactory;
+import org.glassfish.jersey.servlet.ServletContainer;
+import org.joda.time.DateTime;
+
+import com.google.common.collect.ImmutableSet;
+import com.typesafe.config.Config;
+
+/**
+ * Wrapper around the Jetty web server.
+ * <p>
+ * Adapted from Drill's drill.exec.WebServer class. Would be good to create a
+ * common base class later, but the goal for the initial project is to avoid
+ * Drill code changes.
+ *
+ * @see <a href=
+ *      "http://www.eclipse.org/jetty/documentation/current/embedding-jetty.html">
+ *      Jetty Embedding documentation</a>
+ */
+
+public class WebServer implements AutoCloseable {
+  private static final Log LOG = LogFactory.getLog(WebServer.class);
+  private final Server jettyServer;
+  private Dispatcher dispatcher;
+
+  public WebServer(Dispatcher dispatcher) {
+    this.dispatcher = dispatcher;
+    Config config = DrillOnYarnConfig.config();
+    if (config.getBoolean(DrillOnYarnConfig.HTTP_ENABLED)) {
+      jettyServer = new Server();
+    } else {
+      jettyServer = null;
+    }
+  }
+
+  /**
+   * Start the web server including setup.
+   *
+   * @throws Exception
+   */
+  public void start() throws Exception {
+    if (jettyServer == null) {
+      return;
+    }
+
+    build();
+    jettyServer.start();
+  }
+
+  private void build() throws Exception {
+    Config config = DrillOnYarnConfig.config();
+    buildConnector(config);
+    buildServlets(config);
+  }
+
+  private void buildConnector(Config config) throws Exception {
+    final ServerConnector serverConnector;
+    if (config.getBoolean(DrillOnYarnConfig.HTTP_ENABLE_SSL)) {
+      serverConnector = createHttpsConnector(config);
+    } else {
+      serverConnector = createHttpConnector(config);
+    }
+    jettyServer.addConnector(serverConnector);
+  }
+
+  /**
+   * Build the web app with embedded servlets.
+   * <p>
+   * <b>ServletContextHandler</b>: is a Jetty-provided handler that add the
+   * extra bits needed to set up the context that servlets expect. Think of it
+   * as an adapter between the (simple) Jetty handler and the (more complex)
+   * servlet API.
+   *
+   */
+  private void buildServlets(Config config) {
+
+    final ServletContextHandler servletContextHandler = new ServletContextHandler(
+        null, "/");
+    servletContextHandler.setErrorHandler(createErrorHandler());
+    jettyServer.setHandler(servletContextHandler);
+
+    // Servlet holder for the pages of the Drill AM web app. The web app is a
+    // javax.ws application driven from annotations. The servlet holder "does
+    // the right thing" to drive the application, which is rooted at "/".
+    // The servlet container comes from Jersey, and manages the servlet
+    // lifecycle.
+
+    final ServletHolder servletHolder = new ServletHolder(
+        new ServletContainer(new WebUiPageTree(dispatcher)));
+    servletHolder.setInitOrder(1);
+    servletContextHandler.addServlet(servletHolder, "/*");
+
+    final ServletHolder restHolder = new ServletHolder(
+        new ServletContainer(new AmRestApi(dispatcher)));
+    restHolder.setInitOrder(2);
+    servletContextHandler.addServlet(restHolder, "/rest/*");
+
+    // Static resources (CSS, images, etc.)
+
+    setupStaticResources(servletContextHandler);
+
+    // Security, if requested.
+
+    if (AMSecurityManagerImpl.isEnabled()) {
+      servletContextHandler.setSecurityHandler(createSecurityHandler(config));
+      servletContextHandler.setSessionHandler(createSessionHandler(config,
+          servletContextHandler.getSecurityHandler()));
+    }
+  }
+
+  private ErrorHandler createErrorHandler() {
+    // Error handler to show detailed errors.
+    // Should probably be turned off in production.
+    final ErrorHandler errorHandler = new ErrorHandler();
+    errorHandler.setShowStacks(true);
+    errorHandler.setShowMessageInTitle(true);
+    return errorHandler;
+  }
+
+  private void setupStaticResources(
+      ServletContextHandler servletContextHandler) {
+
+    // Access to static resources (JS pages, images, etc.)
+    // The static resources themselves come from Drill exec sub-project
+    // and the Drill-on-YARN project.
+    //
+    // We handle static content this way because we want to do it
+    // in the context of a servlet app, so we need the Jetty "default servlet"
+    // that handles static content. That servlet is designed to take its
+    // properties
+    // from the web.xml, file; but can also take them programmatically as done
+    // here. (The Jetty manual suggests a simpler handler, but that is a
+    // non-Servlet
+    // version.)
+
+    final ServletHolder staticHolder = new ServletHolder("static",
+        DefaultServlet.class);
+    staticHolder.setInitParameter("resourceBase",
+        Resource.newClassPathResource("/rest/static").toString());
+    staticHolder.setInitParameter("dirAllowed", "false");
+    staticHolder.setInitParameter("pathInfoOnly", "true");
+    servletContextHandler.addServlet(staticHolder, "/static/*");
+
+    final ServletHolder amStaticHolder = new ServletHolder("am-static",
+        DefaultServlet.class);
+    amStaticHolder.setInitParameter("resourceBase",
+        Resource.newClassPathResource("/drill-am/static").toString());
+    amStaticHolder.setInitParameter("dirAllowed", "false");
+    amStaticHolder.setInitParameter("pathInfoOnly", "true");
+    servletContextHandler.addServlet(amStaticHolder, "/drill-am/static/*");
+  }
+
+  public static class AMUserPrincipal implements Principal {
+    public final String userName;
+
+    public AMUserPrincipal(String userName) {
+      this.userName = userName;
+    }
+
+    @Override
+    public String getName() {
+      return userName;
+    }
+  }
+
+  public static class AmLoginService implements LoginService {
+    private AMSecurityManager securityMgr;
+    protected IdentityService identityService = new DefaultIdentityService();
+
+    public AmLoginService(AMSecurityManager securityMgr) {
+      this.securityMgr = securityMgr;
+    }
+
+    @Override
+    public String getName() {
+      return "drill-am";
+    }
+
+    @Override
+    public UserIdentity login(String username, Object credentials) {
+      if (!securityMgr.login(username, (String) credentials)) {
+        return null;
+      }
+      return new DefaultUserIdentity(null, new AMUserPrincipal(username),
+          new String[] { ADMIN_ROLE });
+    }
+
+    @Override
+    public boolean validate(UserIdentity user) {
+      return true;
+    }
+
+    @Override
+    public IdentityService getIdentityService() {
+      return identityService;
+    }
+
+    @Override
+    public void setIdentityService(IdentityService service) {
+      this.identityService = service;
+    }
+
+    @Override
+    public void logout(UserIdentity user) {
+    }
+
+    // @Override
+    // protected UserIdentity loadUser(String username) {
+    // // TODO Auto-generated method stub
+    // return null;
+    // }
+    //
+    // @Override
+    // protected void loadUsers() throws IOException {
+    // putUser( "fred", new Password( "wilma" ), new String[] { ADMIN_ROLE } );
+    // }
+
+  }
+
+  /**
+   * @return
+   * @return
+   * @see http://www.eclipse.org/jetty/documentation/current/embedded-examples.html
+   */
+
+  private ConstraintSecurityHandler createSecurityHandler(Config config) {
+    ConstraintSecurityHandler security = new ConstraintSecurityHandler();
+
+    Set<String> knownRoles = ImmutableSet.of(ADMIN_ROLE);
+    security.setConstraintMappings(Collections.<ConstraintMapping> emptyList(),
+        knownRoles);
+
+    security.setAuthenticator(new FormAuthenticator("/login", "/login", true));
+    security
+        .setLoginService(new AmLoginService(AMSecurityManagerImpl.instance()));
+
+    return security;
+  }
+
+  /**
+   * @return A {@link SessionHandler} which contains a
+   *         {@link HashSessionManager}
+   */
+  private SessionHandler createSessionHandler(Config config,
+      final SecurityHandler securityHandler) {
+    SessionManager sessionManager = new HashSessionManager();
+    sessionManager.setMaxInactiveInterval(
+        config.getInt(DrillOnYarnConfig.HTTP_SESSION_MAX_IDLE_SECS));
+    sessionManager.addEventListener(new HttpSessionListener() {
+      @Override
+      public void sessionCreated(HttpSessionEvent se) {
+        // No-op
+      }
+
+      @Override
+      public void sessionDestroyed(HttpSessionEvent se) {
+        final HttpSession session = se.getSession();
+        if (session == null) {
+          return;
+        }
+
+        final Object authCreds = session
+            .getAttribute(SessionAuthentication.__J_AUTHENTICATED);
+        if (authCreds != null) {
+          final SessionAuthentication sessionAuth = (SessionAuthentication) authCreds;
+          securityHandler.logout(sessionAuth);
+          session.removeAttribute(SessionAuthentication.__J_AUTHENTICATED);
+        }
+      }
+    });
+
+    return new SessionHandler(sessionManager);
+  }
+
+  /**
+   * Create HTTP connector.
+   *
+   * @return Initialized {@link ServerConnector} instance for HTTP connections.
+   * @throws Exception
+   */
+  private ServerConnector createHttpConnector(Config config) throws Exception {
+    LOG.info("Setting up HTTP connector for web server");
+    final HttpConfiguration httpConfig = new HttpConfiguration();
+    final ServerConnector httpConnector = new ServerConnector(jettyServer,
+        new HttpConnectionFactory(httpConfig));
+    httpConnector.setPort(config.getInt(DrillOnYarnConfig.HTTP_PORT));
+
+    return httpConnector;
+  }
+
+  /**
+   * Create an HTTPS connector for given jetty server instance. If the admin has
+   * specified keystore/truststore settings they will be used else a self-signed
+   * certificate is generated and used.
+   * <p>
+   * This is a shameless copy of
+   * {@link org.apache.drill.exec.server.rest.Webserver#createHttpsConnector( )}.
+   * The two should be merged at some point. The primary issue is that the Drill
+   * version is tightly coupled to Drillbit configuration.
+   *
+   * @return Initialized {@link ServerConnector} for HTTPS connections.
+   * @throws Exception
+   */
+
+  private ServerConnector createHttpsConnector(Config config) throws Exception {
+    LOG.info("Setting up HTTPS connector for web server");
+
+    final SslContextFactory sslContextFactory = new SslContextFactory();
+
+    // if (config.hasPath(ExecConstants.HTTP_KEYSTORE_PATH) &&
+    // !Strings.isNullOrEmpty(config.getString(ExecConstants.HTTP_KEYSTORE_PATH)))
+    // {
+    // LOG.info("Using configured SSL settings for web server");
+    // sslContextFactory.setKeyStorePath(config.getString(ExecConstants.HTTP_KEYSTORE_PATH));
+    // sslContextFactory.setKeyStorePassword(config.getString(ExecConstants.HTTP_KEYSTORE_PASSWORD));
+    //
+    // // TrustStore and TrustStore password are optional
+    // if (config.hasPath(ExecConstants.HTTP_TRUSTSTORE_PATH)) {
+    // sslContextFactory.setTrustStorePath(config.getString(ExecConstants.HTTP_TRUSTSTORE_PATH));
+    // if (config.hasPath(ExecConstants.HTTP_TRUSTSTORE_PASSWORD)) {
+    // sslContextFactory.setTrustStorePassword(config.getString(ExecConstants.HTTP_TRUSTSTORE_PASSWORD));
+    // }
+    // }
+    // } else {
+    LOG.info("Using generated self-signed SSL settings for web server");
+    final SecureRandom random = new SecureRandom();
+
+    // Generate a private-public key pair
+    final KeyPairGenerator keyPairGenerator = KeyPairGenerator
+        .getInstance("RSA");
+    keyPairGenerator.initialize(1024, random);
+    final KeyPair keyPair = keyPairGenerator.generateKeyPair();
+
+    final DateTime now = DateTime.now();
+
+    // Create builder for certificate attributes
+    final X500NameBuilder nameBuilder = new X500NameBuilder(BCStyle.INSTANCE)
+        .addRDN(BCStyle.OU, "Apache Drill (auth-generated)")
+        .addRDN(BCStyle.O, "Apache Software Foundation (auto-generated)")
+        .addRDN(BCStyle.CN, "Drill AM");
+
+    final Date notBefore = now.minusMinutes(1).toDate();
+    final Date notAfter = now.plusYears(5).toDate();
+    final BigInteger serialNumber = new BigInteger(128, random);
+
+    // Create a certificate valid for 5years from now.
+    final X509v3CertificateBuilder certificateBuilder = new JcaX509v3CertificateBuilder(
+        nameBuilder.build(), // attributes
+        serialNumber, notBefore, notAfter, nameBuilder.build(),
+        keyPair.getPublic());
+
+    // Sign the certificate using the private key
+    final ContentSigner contentSigner = new JcaContentSignerBuilder(
+        "SHA256WithRSAEncryption").build(keyPair.getPrivate());
+    final X509Certificate certificate = new JcaX509CertificateConverter()
+        .getCertificate(certificateBuilder.build(contentSigner));
+
+    // Check the validity
+    certificate.checkValidity(now.toDate());
+
+    // Make sure the certificate is self-signed.
+    certificate.verify(certificate.getPublicKey());
+
+    // Generate a random password for keystore protection
+    final String keyStorePasswd = RandomStringUtils.random(20);
+    final KeyStore keyStore = KeyStore.getInstance("JKS");
+    keyStore.load(null, null);
+    keyStore.setKeyEntry("DrillAutoGeneratedCert", keyPair.getPrivate(),
+        keyStorePasswd.toCharArray(),
+        new java.security.cert.Certificate[] { certificate });
+
+    sslContextFactory.setKeyStore(keyStore);
+    sslContextFactory.setKeyStorePassword(keyStorePasswd);
+    // }
+
+    final HttpConfiguration httpsConfig = new HttpConfiguration();
+    httpsConfig.addCustomizer(new SecureRequestCustomizer());
+
+    // SSL Connector
+    final ServerConnector sslConnector = new ServerConnector(jettyServer,
+        new SslConnectionFactory(sslContextFactory,
+            HttpVersion.HTTP_1_1.asString()),
+        new HttpConnectionFactory(httpsConfig));
+    sslConnector.setPort(config.getInt(DrillOnYarnConfig.HTTP_PORT));
+
+    return sslConnector;
+  }
+
+  @Override
+  public void close() throws Exception {
+    if (jettyServer != null) {
+      jettyServer.stop();
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/WebUiPageTree.java
----------------------------------------------------------------------
diff --git a/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/WebUiPageTree.java b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/WebUiPageTree.java
new file mode 100644
index 0000000..fc44e45
--- /dev/null
+++ b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/WebUiPageTree.java
@@ -0,0 +1,527 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.yarn.appMaster.http;
+
+import static org.apache.drill.exec.server.rest.auth.DrillUserPrincipal.ADMIN_ROLE;
+
+import java.net.URI;
+import java.net.URLDecoder;
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.annotation.security.PermitAll;
+import javax.annotation.security.RolesAllowed;
+import javax.inject.Inject;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import javax.servlet.http.HttpSession;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.FormParam;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.SecurityContext;
+import javax.ws.rs.core.UriBuilder;
+import javax.ws.rs.core.UriInfo;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.drill.yarn.appMaster.Dispatcher;
+import org.apache.drill.yarn.core.DoYUtil;
+import org.apache.drill.yarn.core.DrillOnYarnConfig;
+import org.eclipse.jetty.security.authentication.FormAuthenticator;
+import org.glassfish.jersey.server.filter.RolesAllowedDynamicFeature;
+import org.glassfish.jersey.server.mvc.Viewable;
+import org.glassfish.jersey.server.mvc.freemarker.FreemarkerMvcFeature;
+
+/**
+ * The Drill AM web UI. The format is highly compact. We use javax.ws.rs to mark
+ * up a Pojo with page path, permissions and HTTP methods. The ADMIN_ROLE is
+ * reused from Drill's web UI.
+ * <p>
+ * In general, all pages require admin role, except for two: the login page and
+ * the redirect page which the YARN web UI follows to start the AM UI.
+ */
+
+public class WebUiPageTree extends PageTree {
+
+  /**
+   * Main DoY page that displays cluster status, and the status of
+   * the resource groups. Available only to the admin user when
+   * DoY is secured.
+   */
+
+  @Path("/")
+  @RolesAllowed(ADMIN_ROLE)
+  public static class RootPage {
+    @Inject
+    SecurityContext sc;
+
+    @GET
+    @Produces(MediaType.TEXT_HTML)
+    public Viewable getRoot() {
+      ControllerModel model = new ControllerModel();
+      dispatcher.getController().visit(model);
+      model.countStrayDrillbits(dispatcher.getController());
+      return new Viewable("/drill-am/index.ftl", toModel(sc, model));
+    }
+  }
+
+  /**
+   * Pages, adapted from Drill, that display the login and logout pages.
+   * Login uses the security mechanism, again borrowed from Drill, to
+   * validate the user against either the simple user/password
+   * configured in DoY, or the user who launched DoY using the
+   * Drill security mechanism.
+   */
+
+  @Path("/")
+  @PermitAll
+  public static class LogInLogOutPages {
+    @Inject
+    SecurityContext sc;
+
+    public static final String REDIRECT_QUERY_PARM = "redirect";
+    public static final String LOGIN_RESOURCE = "login";
+
+    @GET
+    @Path("/login")
+    @Produces(MediaType.TEXT_HTML)
+    public Viewable getLoginPage(@Context HttpServletRequest request,
+        @Context HttpServletResponse response, @Context SecurityContext sc,
+        @Context UriInfo uriInfo,
+        @QueryParam(REDIRECT_QUERY_PARM) String redirect) throws Exception {
+
+      if (!StringUtils.isEmpty(redirect)) {
+        // If the URL has redirect in it, set the redirect URI in session, so
+        // that after the login is successful, request
+        // is forwarded to the redirect page.
+        final HttpSession session = request.getSession(true);
+        final URI destURI = UriBuilder
+            .fromUri(URLDecoder.decode(redirect, "UTF-8")).build();
+        session.setAttribute(FormAuthenticator.__J_URI, destURI.toString());
+      }
+
+      return new Viewable("/drill-am/login.ftl", toModel(sc, (Object) null));
+    }
+
+    // Request type is POST because POST request which contains the login
+    // credentials are invalid and the request is
+    // dispatched here directly.
+    @POST
+    @Path("/login")
+    @Produces(MediaType.TEXT_HTML)
+    public Viewable getLoginPageAfterValidationError() {
+      return new Viewable("/drill-am/login.ftl",
+          toModel(sc, "Invalid user name or password."));
+    }
+
+    @GET
+    @Path("/logout")
+    public Viewable logout(@Context HttpServletRequest req,
+        @Context HttpServletResponse resp) throws Exception {
+      final HttpSession session = req.getSession();
+      if (session != null) {
+        session.invalidate();
+      }
+
+      req.getRequestDispatcher("/login").forward(req, resp);
+      return null;
+    }
+  }
+
+  /**
+   * DoY provides a link to YARN to display the AM UI. YARN wants to display the
+   * linked page in a frame, which does not play well with the DoY UI. To avoid
+   * this, we give YARN a link to this redirect page which does nothing other
+   * than to redirect the browser to the (full) DoY main UI.
+   */
+
+  @Path("/redirect")
+  @PermitAll
+  public static class RedirectPage {
+    @GET
+    @Produces(MediaType.TEXT_HTML)
+    public Viewable getRoot() {
+      Map<String, String> map = new HashMap<>();
+      String baseUrl = DoYUtil.unwrapAmUrl(dispatcher.getTrackingUrl());
+      map.put("amLink", baseUrl);
+      map.put("clusterName", config.getString(DrillOnYarnConfig.APP_NAME));
+      return new Viewable("/drill-am/redirect.ftl", map);
+    }
+  }
+
+  /**
+   * Display the configuration page which displays the contents of
+   * DoY and selected Drill config as name/value pairs. Visible only
+   * to the admin when DoY is secure.
+   */
+
+  @Path("/config")
+  @RolesAllowed(ADMIN_ROLE)
+  public static class ConfigPage {
+    @Inject
+    private SecurityContext sc;
+
+    @GET
+    @Produces(MediaType.TEXT_HTML)
+    public Viewable getRoot() {
+      return new Viewable("/drill-am/config.ftl",
+          toModel(sc, DrillOnYarnConfig.instance().getPairs()));
+    }
+  }
+
+  /**
+   * Displays the list of Drillbits showing details for each Drillbit.
+   * (DoY uses the generic term "task", but, at present, the only
+   * task that DoY runs is a Drillbit.
+   */
+
+  @Path("/drillbits")
+  @RolesAllowed(ADMIN_ROLE)
+  public static class DrillbitsPage {
+    @Inject
+    private SecurityContext sc;
+
+    @GET
+    @Produces(MediaType.TEXT_HTML)
+    public Viewable getRoot() {
+      AbstractTasksModel.TasksModel model = new AbstractTasksModel.TasksModel();
+      dispatcher.getController().visitTasks(model);
+      model.listAnomalies(dispatcher.getController());
+      model.sortTasks();
+
+      // Done this funky way because FreeMarker only understands lists if they
+      // are members of a hash (grumble, grumble...)
+
+      Map<String, Object> map = new HashMap<>();
+      map.put("model", model);
+      map.put("tasks", model.getTasks());
+      if (model.hasUnmanagedDrillbits()) {
+        map.put("strays", model.getUnnamaged());
+      }
+      if (model.hasBlacklist()) {
+        map.put("blacklist", model.getBlacklist());
+      }
+      map.put("showDisks", dispatcher.getController().supportsDiskResource());
+      map.put("refreshSecs", DrillOnYarnConfig.config()
+          .getInt(DrillOnYarnConfig.HTTP_REFRESH_SECS));
+      return new Viewable("/drill-am/tasks.ftl", toMapModel(sc, map));
+    }
+  }
+
+  /**
+   * Displays a warning page to ask the user if they want to cancel
+   * a Drillbit. This is a bit old-school; we display this as a
+   * separate page. A good future enhancement is to do this as
+   * a pop-up in Javascript. The GET request display the confirmation
+   * page, the PUT request confirms cancellation and does the deed.
+   * The task to be cancelled appears as a query parameter:
+   * <pre>.../cancel?id=&lt;task id></pre>
+   */
+
+  @Path("/cancel/")
+  @RolesAllowed(ADMIN_ROLE)
+  public static class CancelDrillbitPage {
+    @Inject
+    private SecurityContext sc;
+
+    @QueryParam("id")
+    private int id;
+
+    @GET
+    @Produces(MediaType.TEXT_HTML)
+    public Viewable getPage() {
+      ConfirmShrink confirm;
+      if (dispatcher.getController().isTaskLive(id)) {
+        confirm = new ConfirmShrink(ConfirmShrink.Mode.KILL);
+      } else {
+        confirm = new ConfirmShrink(ConfirmShrink.Mode.CANCEL);
+      }
+      confirm.id = id;
+      return new Viewable("/drill-am/shrink-warning.ftl", toModel(sc, confirm));
+    }
+
+    @POST
+    @Produces(MediaType.TEXT_HTML)
+    public Viewable postPage() {
+      Acknowledge ack;
+      if (dispatcher.getController().cancelTask(id)) {
+        ack = new Acknowledge(Acknowledge.Mode.CANCELLED);
+      } else {
+        ack = new Acknowledge(Acknowledge.Mode.INVALID_TASK);
+      }
+      ack.value = id;
+      return new Viewable("/drill-am/confirm.ftl", toModel(sc, ack));
+    }
+  }
+
+  /**
+   * Displays a history of completed tasks which indicates failed or cancelled
+   * Drillbits. Helps the admin to understand what has been happening on the
+   * cluster if Drillbits have died.
+   */
+
+  @Path("/history")
+  @RolesAllowed(ADMIN_ROLE)
+  public static class HistoryPage {
+    @Inject
+    SecurityContext sc;
+
+    @GET
+    @Produces(MediaType.TEXT_HTML)
+    public Viewable getRoot() {
+      AbstractTasksModel.HistoryModel model = new AbstractTasksModel.HistoryModel();
+      dispatcher.getController().visit(model);
+      Map<String, Object> map = new HashMap<>();
+      map.put("model", model.results);
+      map.put("refreshSecs", DrillOnYarnConfig.config()
+          .getInt(DrillOnYarnConfig.HTTP_REFRESH_SECS));
+      return new Viewable("/drill-am/history.ftl", toMapModel(sc, map));
+    }
+  }
+
+  /**
+   * Page that lets the admin change the cluster size or shut down the cluster.
+   */
+
+  @Path("/manage")
+  @RolesAllowed(ADMIN_ROLE)
+  public static class ManagePage {
+    @Inject
+    SecurityContext sc;
+
+    @GET
+    @Produces(MediaType.TEXT_HTML)
+    public Viewable getRoot() {
+      ControllerModel model = new ControllerModel();
+      dispatcher.getController().visit(model);
+      return new Viewable("/drill-am/manage.ftl", toModel(sc, model));
+    }
+  }
+
+  /**
+   * Passes information to the acknowledgement page.
+   */
+
+  public static class Acknowledge {
+    public enum Mode {
+      STOPPED, INVALID_RESIZE, INVALID_ACTION, NULL_RESIZE, RESIZED, CANCELLED, INVALID_TASK
+    };
+
+    Mode mode;
+    Object value;
+
+    public Acknowledge(Mode mode) {
+      this.mode = mode;
+    }
+
+    public String getType() {
+      return mode.toString();
+    }
+
+    public Object getValue() {
+      return value;
+    }
+  }
+
+  /**
+   * Passes information to the confirmation page.
+   */
+
+  public static class ConfirmShrink {
+    public enum Mode {
+      SHRINK, STOP, CANCEL, KILL
+    };
+
+    Mode mode;
+    int value;
+    int id;
+
+    public ConfirmShrink(Mode mode) {
+      this.mode = mode;
+    }
+
+    public boolean isStop() {
+      return mode == Mode.STOP;
+    }
+
+    public boolean isCancel() {
+      return mode == Mode.CANCEL;
+    }
+
+    public boolean isKill() {
+      return mode == Mode.KILL;
+    }
+
+    public boolean isShrink() {
+      return mode == Mode.SHRINK;
+    }
+
+    public int getCount() {
+      return value;
+    }
+
+    public int getId() {
+      return id;
+    }
+  }
+
+  /**
+   * Confirm that the user wants to resize the cluster. Displays a warning if
+   * the user wants to shrink the cluster, since, at present, doing so will
+   * kill any in-flight queries. The GET request display the warning,
+   * the POST request confirms the action. The action itself is provided
+   * as query parameters:
+   * <pre>.../resize?type=&lt;type>&n=&lt;quantity></pre>
+   * Where the type is one of "resize", "grow", "shrink" or
+   * "force-shrink" and n is the associated quantity.
+   * <p>
+   * Note that the manage page only provides the "resize" option; the
+   * grow and shrink options were removed from the Web UI and are only
+   * visible through the REST API.
+   */
+
+  @Path("/resize")
+  @RolesAllowed(ADMIN_ROLE)
+  public static class ResizePage {
+    @Inject
+    SecurityContext sc;
+
+    @FormParam("n")
+    int n;
+    @FormParam("type")
+    String type;
+
+    @POST
+    @Consumes(MediaType.APPLICATION_FORM_URLENCODED)
+    @Produces(MediaType.TEXT_HTML)
+    public Viewable resize() {
+      int curSize = dispatcher.getController().getTargetCount();
+      if (n <= 0) {
+        Acknowledge confirm = new Acknowledge(Acknowledge.Mode.INVALID_RESIZE);
+        confirm.value = n;
+        return new Viewable("/drill-am/confirm.ftl", toModel(sc, confirm));
+      }
+      if (type == null) {
+        type = "null";
+      }
+      int newSize;
+      boolean confirmed = false;
+      if (type.equalsIgnoreCase("resize")) {
+        newSize = n;
+      } else if (type.equalsIgnoreCase("grow")) {
+        newSize = curSize + n;
+      } else if (type.equalsIgnoreCase("shrink")) {
+        newSize = curSize - n;
+      } else if (type.equalsIgnoreCase("force-shrink")) {
+        newSize = curSize - n;
+        confirmed = true;
+      } else {
+        Acknowledge confirm = new Acknowledge(Acknowledge.Mode.INVALID_ACTION);
+        confirm.value = type;
+        return new Viewable("/drill-am/confirm.ftl", toModel(sc, confirm));
+      }
+
+      if (curSize == newSize) {
+        Acknowledge confirm = new Acknowledge(Acknowledge.Mode.NULL_RESIZE);
+        confirm.value = newSize;
+        return new Viewable("/drill-am/confirm.ftl", toModel(sc, confirm));
+      } else if (confirmed || curSize < newSize) {
+        Acknowledge confirm = new Acknowledge(Acknowledge.Mode.RESIZED);
+        confirm.value = dispatcher.getController().resizeTo(newSize);
+        return new Viewable("/drill-am/confirm.ftl", toModel(sc, confirm));
+      } else {
+        ConfirmShrink confirm = new ConfirmShrink(ConfirmShrink.Mode.SHRINK);
+        confirm.value = curSize - newSize;
+        return new Viewable("/drill-am/shrink-warning.ftl",
+            toModel(sc, confirm));
+      }
+    }
+  }
+
+  /**
+   * Confirmation page when the admin asks to stop the cluster.
+   * The GET request displays the confirmation, the POST does
+   * the deed. As for other confirmation pages, this is an old-style,
+   * quick & dirty solution. A more modern solution would be to use JavaScript
+   * to pop up a confirmation dialog.
+   */
+
+  @Path("/stop/")
+  @RolesAllowed(ADMIN_ROLE)
+  public static class StopPage {
+    @Inject
+    SecurityContext sc;
+
+    @GET
+    @Produces(MediaType.TEXT_HTML)
+    public Viewable requestStop() {
+      ConfirmShrink confirm = new ConfirmShrink(ConfirmShrink.Mode.STOP);
+      return new Viewable("/drill-am/shrink-warning.ftl", toModel(sc, confirm));
+    }
+
+    @POST
+    @Produces(MediaType.TEXT_HTML)
+    public Viewable doStop() {
+      dispatcher.getController().shutDown();
+      Acknowledge confirm = new Acknowledge(Acknowledge.Mode.STOPPED);
+      return new Viewable("/drill-am/confirm.ftl", toModel(sc, confirm));
+    }
+  }
+
+  /**
+   * Build the pages for the Web UI using Freemarker to implement the
+   * MVC mechanism. This class builds on a rather complex mechanism; understand
+   * that to understand what the lines of code below are doing.
+   *
+   * @param dispatcher the DoY AM dispatcher that receives requests for
+   * information about, or requests to change the state of, the Drill clutser
+   */
+
+  public WebUiPageTree(Dispatcher dispatcher) {
+    super(dispatcher);
+
+    // Markup engine
+    register(FreemarkerMvcFeature.class);
+
+    // Web UI Pages
+    register(RootPage.class);
+    register(RedirectPage.class);
+    register(ConfigPage.class);
+    register(DrillbitsPage.class);
+    register(CancelDrillbitPage.class);
+    register(HistoryPage.class);
+    register(ManagePage.class);
+    register(ResizePage.class);
+    register(StopPage.class);
+
+    // Authorization
+    // See: https://jersey.java.net/documentation/latest/security.html
+
+    if (AMSecurityManagerImpl.isEnabled()) {
+      register(LogInLogOutPages.class);
+      register(AuthDynamicFeature.class);
+      register(RolesAllowedDynamicFeature.class);
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/package-info.java
----------------------------------------------------------------------
diff --git a/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/package-info.java b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/package-info.java
new file mode 100644
index 0000000..13f1bd8
--- /dev/null
+++ b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/package-info.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ *
+ */
+package org.apache.drill.yarn.appMaster.http;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/package-info.java
----------------------------------------------------------------------
diff --git a/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/package-info.java b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/package-info.java
new file mode 100644
index 0000000..0ff835d
--- /dev/null
+++ b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/package-info.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Implements the Drill Application Master for YARN.
+ * <p>
+ * Note that AM implementation classes use org.apache.commons.logging
+ * to be consistent with the logging used within YARN itself. However,
+ * the AM uses Drill's class path which uses logback logging. To enable
+ * logging, modify
+ * <code>$DRILL_HOME/conf/logback.xml</code> and add a section something
+ * like this:
+ * <pre><code>
+ *   &lt;logger name="org.apache.drill.yarn" additivity="false">
+ *    &lt;level value="trace" />
+ *    &lt;appender-ref ref="STDOUT" />
+ *   &lt;/logger>
+ * </code></pre>
+ */
+
+package org.apache.drill.yarn.appMaster;
\ No newline at end of file


Mime
View raw message