http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AMSecurityManagerImpl.java
----------------------------------------------------------------------
diff --git a/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AMSecurityManagerImpl.java b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AMSecurityManagerImpl.java
new file mode 100644
index 0000000..d31690e
--- /dev/null
+++ b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AMSecurityManagerImpl.java
@@ -0,0 +1,221 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.yarn.appMaster.http;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.scanner.persistence.ScanResult;
+import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.exception.DrillbitStartupException;
+import org.apache.drill.exec.rpc.user.security.UserAuthenticationException;
+import org.apache.drill.exec.rpc.user.security.UserAuthenticator;
+import org.apache.drill.exec.rpc.user.security.UserAuthenticatorFactory;
+import org.apache.drill.exec.util.ImpersonationUtil;
+import org.apache.drill.yarn.appMaster.AMWrapperException;
+import org.apache.drill.yarn.core.DoYUtil;
+import org.apache.drill.yarn.core.DrillOnYarnConfig;
+
+import com.typesafe.config.Config;
+
+/**
+ * Implements the three supported AM security models: Drill,
+ * hard-coded user and password, and open access.
+ */
+
+public class AMSecurityManagerImpl implements AMSecurityManager {
+ private static final Log LOG = LogFactory.getLog(AMSecurityManagerImpl.class);
+
+ /**
+ * Thin layer around the Drill authentication system to adapt from
+ * Drill-on-YARN's environment to that expected by the Drill classes.
+ */
+ private static class DrillSecurityManager implements AMSecurityManager {
+ private UserAuthenticator authenticator;
+
+ @Override
+ public void init() {
+ try {
+ DrillOnYarnConfig doyConfig = DrillOnYarnConfig.instance();
+ DrillConfig config = doyConfig.getDrillConfig();
+ ScanResult classpathScan = doyConfig.getClassPathScan();
+ if (config.getBoolean(ExecConstants.USER_AUTHENTICATION_ENABLED)) {
+ authenticator = UserAuthenticatorFactory.createAuthenticator(config,
+ classpathScan);
+ } else {
+ authenticator = null;
+ }
+ } catch (DrillbitStartupException e) {
+ LOG.info("Authentication initialization failed", e);
+ throw new AMWrapperException("Security init failed", e);
+ }
+ }
+
+ @Override
+ public boolean login(String user, String password) {
+ if (authenticator == null) {
+ return true;
+ }
+ try {
+ authenticator.authenticate(user, password);
+ } catch (UserAuthenticationException e) {
+ LOG.info("Authentication failed for user " + user, e);
+ return false;
+ }
+ return ImpersonationUtil.getProcessUserName().equals(user);
+ }
+
+ @Override
+ public void close() {
+ try {
+ if (authenticator != null) {
+ authenticator.close();
+ }
+ } catch (IOException e) {
+ LOG.info("Ignoring error on authenticator close", e);
+ }
+ }
+
+ @Override
+ public boolean requiresLogin() {
+ return authenticator != null;
+ }
+ }
+
+ /**
+ * Simple security manager: user name and password reside in the DoY config
+ * file.
+ */
+
+ private static class SimpleSecurityManager implements AMSecurityManager {
+
+ private String userName;
+ private String password;
+
+ @Override
+ public void init() {
+ Config config = DrillOnYarnConfig.config();
+ userName = config.getString(DrillOnYarnConfig.HTTP_USER_NAME);
+ password = config.getString(DrillOnYarnConfig.HTTP_PASSWORD);
+ if (DoYUtil.isBlank(userName)) {
+ LOG.warn("Simple HTTP authentication is enabled, but "
+ + DrillOnYarnConfig.HTTP_USER_NAME + " is blank.");
+ }
+ if (DoYUtil.isBlank(userName)) {
+ LOG.warn("Simple HTTP authentication is enabled, but "
+ + DrillOnYarnConfig.HTTP_PASSWORD + " is blank.");
+ }
+ }
+
+ @Override
+ public boolean requiresLogin() {
+ return !DoYUtil.isBlank(userName);
+ }
+
+ @Override
+ public boolean login(String user, String pwd) {
+ if (!requiresLogin()) {
+ return true;
+ }
+ boolean ok = userName.equals(user) && password.equals(pwd);
+ if (!ok) {
+ LOG.info(
+ "Failed login attempt with simple authorization for user " + user);
+ }
+ return ok;
+ }
+
+ @Override
+ public void close() {
+ // Nothing to do
+ }
+
+ }
+
+ private static AMSecurityManagerImpl instance;
+
+ private AMSecurityManager managerImpl;
+
+ private AMSecurityManagerImpl() {
+ }
+
+ public static void setup() {
+ instance = new AMSecurityManagerImpl();
+ instance.init();
+ }
+
+ /**
+ * Look at the DoY config file to decide which security system (if any) to
+ * use.
+ */
+
+ @Override
+ public void init() {
+ Config config = DrillOnYarnConfig.config();
+ String authType = config.getString(DrillOnYarnConfig.HTTP_AUTH_TYPE);
+ if (DrillOnYarnConfig.AUTH_TYPE_DRILL.equals(authType)) {
+ // Drill authentication. Requires both DoY to select Drill
+ // auth, and for Drill's auth to be enabled.
+ if(config.getBoolean(ExecConstants.USER_AUTHENTICATION_ENABLED)) {
+ managerImpl = new DrillSecurityManager();
+ managerImpl.init();
+ }
+ } else if (DrillOnYarnConfig.AUTH_TYPE_SIMPLE.equals(authType)) {
+ managerImpl = new SimpleSecurityManager();
+ managerImpl.init();
+ } else if (DoYUtil.isBlank(authType)
+ || DrillOnYarnConfig.AUTH_TYPE_NONE.equals(authType)) {
+ ;
+ } else {
+ LOG.error("Unrecognized authorization type for "
+ + DrillOnYarnConfig.HTTP_AUTH_TYPE + ": " + authType
+ + " - assuming no auth.");
+ }
+ }
+
+ @Override
+ public boolean login(String user, String password) {
+ if (managerImpl == null) {
+ return true;
+ }
+ return managerImpl.login(user, password);
+ }
+
+ @Override
+ public void close() {
+ if (managerImpl != null) {
+ managerImpl.close();
+ managerImpl = null;
+ }
+ }
+
+ @Override
+ public boolean requiresLogin() {
+ return managerImpl != null;
+ }
+
+ public static AMSecurityManager instance() {
+ return instance;
+ }
+
+ public static boolean isEnabled() {
+ return instance != null && instance.managerImpl != null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AbstractTasksModel.java
----------------------------------------------------------------------
diff --git a/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AbstractTasksModel.java b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AbstractTasksModel.java
new file mode 100644
index 0000000..bc4e8d6
--- /dev/null
+++ b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AbstractTasksModel.java
@@ -0,0 +1,380 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.yarn.appMaster.http;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
+import org.apache.drill.yarn.appMaster.ClusterController;
+import org.apache.drill.yarn.appMaster.ClusterControllerImpl;
+import org.apache.drill.yarn.appMaster.ControllerVisitor;
+import org.apache.drill.yarn.appMaster.Task;
+import org.apache.drill.yarn.appMaster.Task.TrackingState;
+import org.apache.drill.yarn.appMaster.TaskState;
+import org.apache.drill.yarn.appMaster.TaskVisitor;
+import org.apache.drill.yarn.core.DoYUtil;
+import org.apache.drill.yarn.core.DrillOnYarnConfig;
+import org.apache.drill.yarn.zk.ZKRegistry;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.Resource;
+
+public abstract class AbstractTasksModel {
+ public static class TaskModel {
+ public int id;
+ protected String groupName;
+ protected boolean isLive;
+ protected TaskState taskState;
+ protected String taskStateHint;
+ protected String state;
+ protected boolean cancelled;
+ protected String trackingState;
+ protected String trackingStateHint;
+ protected Container container;
+ protected DrillbitEndpoint endpoint;
+ protected long startTime;
+ protected int memoryMb;
+ protected int vcores;
+ protected double disks;
+ protected String containerId;
+ protected String nmLink;
+ protected long endTime;
+ protected String disposition;
+ protected int tryCount;
+
+ private Map<TaskState,String> stateHints = makeStateHints( );
+ private Map<TrackingState,String> trackingStateHints = makeTrackingStateHints( );
+
+ public TaskModel(Task task, boolean live) {
+ id = task.taskId;
+ groupName = task.scheduler.getName();
+ taskState = task.getState();
+ taskStateHint = stateHints.get(taskState);
+ state = taskState.getLabel();
+ cancelled = task.isCancelled();
+ isLive = live && taskState == TaskState.RUNNING;
+ TrackingState tState = task.getTrackingState();
+ trackingState = tState.getDisplayName();
+ trackingStateHint = trackingStateHints.get(tState);
+ container = task.container;
+ startTime = task.launchTime;
+ if (task.container != null) {
+ containerId = task.container.getId().toString();
+ Resource resource = task.container.getResource();
+ memoryMb = resource.getMemory();
+ vcores = resource.getVirtualCores();
+ disks = task.getContainerSpec().disks;
+
+ // Emulate the NM link. Used for debugging, gets us to
+ // the page on the NM UI for this container so we can see
+ // logs, etc.
+
+ nmLink = "http://" + task.container.getNodeHttpAddress();
+ } else {
+ memoryMb = task.scheduler.getResource().memoryMb;
+ vcores = task.scheduler.getResource().vCores;
+ }
+ endpoint = (DrillbitEndpoint) task.properties
+ .get(ZKRegistry.ENDPOINT_PROPERTY);
+ if (!live) {
+ endTime = task.completionTime;
+ tryCount = task.tryCount;
+
+ // Determine disposition from most general to most
+ // specific sources of information.
+
+ disposition = state;
+ if (task.disposition != null) {
+ disposition = task.disposition.toString();
+ }
+ if (task.completionStatus != null) {
+ disposition = reformatDiagnostics( task.completionStatus.getDiagnostics() );
+ }
+ if (task.error != null) {
+ disposition = task.error.getMessage();
+ }
+ }
+ }
+
+ private enum FormatState { PRE_STACK, IN_STACK, POST_STACK };
+
+ /**
+ * YARN diagnostics are verbose: they contain a stack trace of the YARN node
+ * manager thread (not Drill), and contain blank lines, the container ID,
+ * etc. Remove unnecessary cruft to make the diagnostics simpler and smaller
+ * in the Web UI.
+ *
+ * @param orig YARN diagnostics
+ * @return cleaned-up version.
+ */
+
+ public static String reformatDiagnostics( String orig ) {
+ try {
+ StringBuilder buf = new StringBuilder( );
+ BufferedReader reader = new BufferedReader( new StringReader( orig ) );
+ String line;
+ FormatState state = FormatState.PRE_STACK;
+ while ( (line = reader.readLine()) != null ) {
+ switch( state ) {
+ case PRE_STACK:
+ if ( line.startsWith( "Container id:") ) {
+ continue;
+ }
+ if ( line.startsWith( "Stack trace:" ) ) {
+ state = FormatState.IN_STACK;
+ continue;
+ }
+ break;
+ case IN_STACK:
+ if ( line.trim().isEmpty() ) {
+ state = FormatState.POST_STACK;
+ }
+ continue;
+ case POST_STACK:
+ default:
+ break;
+ }
+ if ( line.trim().isEmpty() ) {
+ continue;
+ }
+ buf.append( line );
+ buf.append( "<br/>\n" );
+ }
+ buf.append( "See log file for details." );
+ return buf.toString();
+ } catch (IOException e) {
+ // Will never occur. But, if the impossible happens, just return
+ // the original diagnostics.
+
+ return orig.replace("\n", "<br>\n");
+ }
+ }
+
+ private Map<TaskState, String> makeStateHints() {
+ Map<TaskState, String> hints = new HashMap<>();
+ hints.put(TaskState.START, "Queued to send a container request to YARN.");
+ hints.put(TaskState.REQUESTING, "Container request sent to YARN.");
+ hints.put(TaskState.LAUNCHING,
+ "YARN provided a container, send launch request.");
+ hints.put(TaskState.WAIT_START_ACK,
+ "Drillbit launched, waiting for ZooKeeper registration.");
+ hints.put(TaskState.RUNNING, "Drillbit is running normally.");
+ hints.put(TaskState.ENDING,
+ "Graceful shutdown request sent to the Drillbit.");
+ hints.put(TaskState.KILLING,
+ "Sent the YARN Node Manager a request to forcefully kill the Drillbit.");
+ hints.put(TaskState.WAIT_END_ACK,
+ "Drillbit has shut down; waiting for ZooKeeper to confirm.");
+ // The UI will never display the END state.
+ hints.put(TaskState.END, "The Drillbit has shut down.");
+ return hints;
+ }
+
+ private Map<TrackingState, String> makeTrackingStateHints() {
+ Map<TrackingState, String> hints = new HashMap<>();
+ // UNTRACKED state is not used by Drillbits.
+ hints.put(TrackingState.UNTRACKED, "Task is not tracked in ZooKeeper.");
+ hints.put(TrackingState.NEW,
+ "Drillbit has not yet registered with ZooKeeper.");
+ hints.put(TrackingState.START_ACK,
+ "Drillbit has registered normally with ZooKeeper.");
+ hints.put(TrackingState.END_ACK,
+ "Drillbit is no longer registered with ZooKeeper.");
+ return hints;
+ }
+
+ public String getTaskId() {
+ return Integer.toString(id);
+ }
+
+ public String getGroupName( ) { return groupName; }
+
+ public boolean isLive( ) {
+ return isLive;
+ }
+
+ public String getHost( ) {
+ if ( container == null ) {
+ return ""; }
+ return container.getNodeId().getHost();
+ }
+
+ public String getLink( ) {
+ if ( endpoint == null ) {
+ return ""; }
+ String port = DrillOnYarnConfig.config( ).getString( DrillOnYarnConfig.DRILLBIT_HTTP_PORT );
+ String protocol = "http:";
+ if ( DrillOnYarnConfig.config().getBoolean( DrillOnYarnConfig.DRILLBIT_USE_HTTPS ) ) {
+ protocol = "https:";
+ }
+ return protocol + "//" + endpoint.getAddress() + ":" + port + "/";
+ }
+
+ public String getState( ) { return state.toString(); }
+ public String getStateHint( ) { return taskStateHint; }
+ public boolean isCancelled( ) { return cancelled; }
+
+ public boolean isCancellable( ) {
+ return ! cancelled && taskState.isCancellable( );
+ }
+
+ public String getTrackingState( ) { return trackingState; }
+ public String getTrackingStateHint( ) { return trackingStateHint; }
+
+ public String getStartTime( ) {
+ if ( startTime == 0 ) {
+ return ""; }
+ return DoYUtil.toIsoTime( startTime );
+ }
+
+ public int getMemory( ) { return memoryMb; }
+ public int getVcores( ) { return vcores; }
+ public String getDisks( ) {
+ return String.format( "%.2f", disks );
+ }
+ public boolean hasContainer( ) { return containerId != null; }
+ public String getContainerId( ) { return displayString( containerId ); }
+ public String getNmLink( ) { return displayString( nmLink ); }
+ public String getDisposition( ) { return displayString( disposition ); }
+ public int getTryCount( ) { return tryCount; }
+ public String displayString( String value ) { return (value == null) ? "" : value; }
+
+ public String getEndTime( ) {
+ if ( endTime == 0 ) {
+ return ""; }
+ return DoYUtil.toIsoTime( endTime );
+ }
+ }
+
+ public static class UnmanagedDrillbitModel
+ {
+ protected String host;
+ protected String ports;
+
+ public UnmanagedDrillbitModel( String endpoint ) {
+ String parts[] = endpoint.split( ":" );
+ if ( parts.length < 4 ) {
+ // Should never occur, but better save than sorry.
+
+ host = endpoint;
+ ports = "";
+ }
+ else {
+ host = parts[0];
+ List<String> thePorts = new ArrayList<>( );
+ thePorts.add( parts[1] );
+ thePorts.add( parts[2] );
+ thePorts.add( parts[3] );
+ ports = DoYUtil.join( ", ", thePorts );
+ }
+ }
+
+ public String getHost( ) { return host; }
+ public String getPorts( ) { return ports; }
+ }
+
+ protected boolean supportsDisks;
+ protected List<TaskModel> results = new ArrayList<>( );
+
+ public List<TaskModel> getTasks( ) { return results; }
+ public boolean hasTasks( ) { return ! results.isEmpty(); }
+ public boolean supportsDiskResource( ) { return supportsDisks; }
+
+ public static class TasksModel extends AbstractTasksModel implements TaskVisitor
+ {
+ protected List<UnmanagedDrillbitModel> unmanaged;
+ protected List<String> blacklist;
+
+ @Override
+ public void visit(Task task) {
+ results.add( new TaskModel( task, true ) );
+ }
+
+ /**
+ * Sort tasks by Task ID.
+ */
+
+ public void sortTasks() {
+ Collections.sort( results, new Comparator<TaskModel>( ) {
+ @Override
+ public int compare(TaskModel t1, TaskModel t2) {
+ return Integer.compare( t1.id, t2.id );
+ }
+ });
+ }
+
+ /**
+ * List any anomalies: either stray Drillbits (those in ZK but not launched by DoY),
+ * or blacklisted nodes.
+ * <p>
+ * To avoid race conditions, do not use the controller visitor to invoke this method,
+ * we want to leave the controller unlocked and instead lock only the ZK registry.
+ *
+ * @param controller
+ */
+
+ public void listAnomalies(ClusterController controller) {
+ listUnmanaged(controller);
+ synchronized( controller ) {
+ blacklist = ((ClusterControllerImpl) controller).getNodeInventory().getBlacklist();
+ }
+ Collections.sort( blacklist );
+ }
+
+ private void listUnmanaged(ClusterController controller) {
+ ZKRegistry zkRegistry = (ZKRegistry) controller.getProperty( ZKRegistry.CONTROLLER_PROPERTY );
+ if ( zkRegistry == null ) {
+ return;
+ }
+ List<String> endpoints = zkRegistry.listUnmanagedDrillits( );
+ if ( endpoints.isEmpty() ) {
+ return; }
+ unmanaged = new ArrayList<>( );
+ for ( String endpoint : endpoints ) {
+ unmanaged.add( new UnmanagedDrillbitModel( endpoint ) );
+ }
+ }
+
+ public List<UnmanagedDrillbitModel>getUnnamaged( ) { return unmanaged; }
+ public boolean hasUnmanagedDrillbits( ) { return unmanaged != null; }
+ public int getUnmanagedDrillbitCount( ) {
+ return (unmanaged == null) ? 0 : unmanaged.size( );
+ }
+ public boolean hasBlacklist( ) { return ! blacklist.isEmpty(); }
+ public int getBlacklistCount( ) { return blacklist.size( ); }
+ public List<String> getBlacklist( ) { return blacklist; }
+ }
+
+ public static class HistoryModel extends AbstractTasksModel implements ControllerVisitor
+ {
+ @Override
+ public void visit(ClusterController controller) {
+ ClusterControllerImpl impl = (ClusterControllerImpl) controller;
+ for ( Task task : impl.getHistory( ) ) {
+ results.add( new TaskModel( task, false ) );
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AmRestApi.java
----------------------------------------------------------------------
diff --git a/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AmRestApi.java b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AmRestApi.java
new file mode 100644
index 0000000..21ddc4b
--- /dev/null
+++ b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AmRestApi.java
@@ -0,0 +1,296 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.yarn.appMaster.http;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import javax.annotation.security.PermitAll;
+import javax.ws.rs.DefaultValue;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.MediaType;
+
+import org.apache.drill.yarn.appMaster.Dispatcher;
+import org.apache.drill.yarn.appMaster.http.AbstractTasksModel.TaskModel;
+import org.apache.drill.yarn.appMaster.http.ControllerModel.ClusterGroupModel;
+import org.apache.drill.yarn.core.DoYUtil;
+import org.apache.drill.yarn.core.DrillOnYarnConfig;
+import org.apache.drill.yarn.core.NameValuePair;
+import org.apache.drill.yarn.zk.ZKClusterCoordinatorDriver;
+
+public class AmRestApi extends PageTree
+{
+ @Path("/config")
+ @PermitAll
+ public static class ConfigResource
+ {
+ @GET
+ @Produces(MediaType.APPLICATION_JSON)
+ public Map<String,Object> getConfig( ) {
+ Map<String, Object> map = new HashMap<>();
+ for (NameValuePair pair : DrillOnYarnConfig.instance().getPairs()) {
+ map.put(pair.getName(), pair.getValue());
+ }
+ return map;
+ }
+ }
+
+ /**
+ * Returns cluster status as a tree of JSON objects. Done as explicitly-defined
+ * maps to specify the key names (which must not change to avoid breaking
+ * compatibility) and to handle type conversions.
+ */
+
+ @Path("/status")
+ @PermitAll
+ public static class StatusResource
+ {
+ @GET
+ @Produces(MediaType.APPLICATION_JSON)
+ public Map<String,Object> getStatus( ) {
+ ControllerModel model = new ControllerModel( );
+ dispatcher.getController().visit( model );
+
+ Map<String,Object> root = new HashMap<>( );
+ root.put( "state", model.state.toString() );
+
+ Map<String, Object> summary = new HashMap<>();
+ summary.put("drillMemoryMb", model.totalDrillMemory);
+ summary.put("drillVcores", model.totalDrillVcores);
+ summary.put("yarnMemoryMb", model.yarnMemory);
+ summary.put("yarnVcores", model.yarnVcores);
+ summary.put("liveBitCount", model.liveCount);
+ summary.put("totalBitCount", model.taskCount);
+ summary.put("targetBitCount", model.targetCount);
+ summary.put("unmanagedCount", model.getUnmanagedCount());
+ summary.put("blackListCount", model.getBlacklistCount());
+ summary.put("freeNodeCount", model.getFreeNodeCount());
+ root.put("summary", summary);
+
+ List<Map<String, Object>> pools = new ArrayList<>();
+ for (ClusterGroupModel pool : model.groups) {
+ Map<String, Object> poolObj = new HashMap<>();
+ poolObj.put("name", pool.name);
+ poolObj.put("type", pool.type);
+ poolObj.put("liveBitCount", pool.liveCount);
+ poolObj.put("targetBitCount", pool.targetCount);
+ poolObj.put("totalBitCount", pool.taskCount);
+ poolObj.put("totalMemoryMb", pool.memory);
+ poolObj.put("totalVcores", pool.vcores);
+ pools.add(poolObj);
+ }
+ root.put("pools", pools);
+
+ AbstractTasksModel.TasksModel tasksModel = new AbstractTasksModel.TasksModel();
+ dispatcher.getController().visitTasks(tasksModel);
+ List<Map<String, Object>> bits = new ArrayList<>();
+ for (TaskModel task : tasksModel.results) {
+ Map<String, Object> bitObj = new HashMap<>();
+ bitObj.put("containerId", task.container.getId().toString());
+ bitObj.put("host", task.getHost());
+ bitObj.put("id", task.id);
+ bitObj.put("live", task.isLive());
+ bitObj.put("memoryMb", task.memoryMb);
+ bitObj.put("vcores", task.vcores);
+ bitObj.put("pool", task.groupName);
+ bitObj.put("state", task.state);
+ bitObj.put("trackingState", task.trackingState);
+ bitObj.put("endpoint",
+ ZKClusterCoordinatorDriver.asString(task.endpoint));
+ bitObj.put("link", task.getLink());
+ bitObj.put("startTime", task.getStartTime());
+ bits.add(bitObj);
+ }
+ root.put("drillbits", bits);
+
+ return root;
+ }
+ }
+
+ /**
+ * Stop the cluster. Uses a key to validate the request. The value of the key is
+ * set in the Drill-on-YARN configuration file. The purpose is simply to prevent
+ * accidental cluster shutdown when experimenting with the REST API; this is
+ * not meant to be a security mechanism.
+ *
+ * @param key
+ * @return
+ */
+
+ @Path("/stop")
+ @PermitAll
+ public static class StopResource
+ {
+ @DefaultValue( "" )
+ @QueryParam( "key" )
+ String key;
+
+ @POST
+ @Produces(MediaType.APPLICATION_JSON)
+ public Map<String,String> postStop( )
+ {
+ Map<String, String> error = checkKey(key);
+ if (error != null) {
+ return error;
+ }
+
+ dispatcher.getController().shutDown();
+ return successResponse("Shutting down");
+ }
+ }
+
+ @Path("/resize/{quantity}")
+ @PermitAll
+ public static class ResizeResource
+ {
+ @PathParam(value = "quantity")
+ String quantity;
+ @DefaultValue( "" )
+ @QueryParam( "key" )
+ String key;
+
+ @POST
+ @Produces(MediaType.APPLICATION_JSON)
+ public Map<String,String> postResize( )
+ {
+ ResizeRequest request = new ResizeRequest(key, quantity);
+ if (request.error != null) {
+ return request.error;
+ }
+
+ int curSize = dispatcher.getController().getTargetCount();
+ dispatcher.getController().resizeTo(request.n);
+ return successResponse("Resizing from " + curSize + " to " + request.n);
+ }
+ }
+
+ protected static class ResizeRequest
+ {
+ Map<String,String> error;
+ int n;
+
+ public ResizeRequest( String key, String quantity ) {
+ error = checkKey(key);
+ if (error != null) {
+ return;
+ }
+ try {
+ n = Integer.parseInt(quantity);
+ } catch (NumberFormatException e) {
+ error = errorResponse("Invalid argument: " + quantity);
+ }
+ if (n < 0) {
+ error = errorResponse("Invalid argument: " + quantity);
+ }
+ }
+ }
+
+ @Path("/grow/{quantity}")
+ @PermitAll
+ public static class GrowResource
+ {
+ @PathParam(value = "quantity")
+ @DefaultValue( "1" )
+ String quantity;
+ @DefaultValue( "" )
+ @QueryParam( "key" )
+ String key;
+
+ @POST
+ @Produces(MediaType.APPLICATION_JSON)
+ public Map<String,String> postResize( )
+ {
+ ResizeRequest request = new ResizeRequest(key, quantity);
+ if (request.error != null) {
+ return request.error;
+ }
+
+ int curSize = dispatcher.getController().getTargetCount();
+ int newSize = curSize + request.n;
+ dispatcher.getController().resizeTo(newSize);
+ return successResponse("Growing by " + request.n + " to " + newSize);
+ }
+ }
+
+ @Path("/shrink/{quantity}")
+ @PermitAll
+ public static class ShrinkResource
+ {
+ @PathParam(value = "quantity")
+ @DefaultValue( "1" )
+ String quantity;
+ @DefaultValue( "" )
+ @QueryParam( "key" )
+ String key;
+
+ @POST
+ @Produces(MediaType.APPLICATION_JSON)
+ public Map<String,String> postResize( )
+ {
+ ResizeRequest request = new ResizeRequest(key, quantity);
+ if (request.error != null) {
+ return request.error;
+ }
+ int curSize = dispatcher.getController().getTargetCount();
+ int newSize = Math.max(curSize - request.n, 0);
+ dispatcher.getController().resizeTo(newSize);
+ return successResponse("Shrinking by " + request.n + " to " + newSize);
+ }
+ }
+
+ private static Map<String, String> checkKey(String key) {
+ String masterKey = DrillOnYarnConfig.config()
+ .getString(DrillOnYarnConfig.HTTP_REST_KEY);
+ if (!DoYUtil.isBlank(masterKey) && !masterKey.equals(key)) {
+ return errorResponse("Invalid Key");
+ }
+ return null;
+ }
+
+ private static Map<String, String> errorResponse(String msg) {
+ Map<String, String> resp = new HashMap<>();
+ resp.put("status", "error");
+ resp.put("message", msg);
+ return resp;
+ }
+
+ private static Map<String, String> successResponse(String msg) {
+ Map<String, String> resp = new HashMap<>();
+ resp.put("status", "ok");
+ resp.put("message", msg);
+ return resp;
+ }
+
+ public AmRestApi(Dispatcher dispatcher) {
+ super(dispatcher);
+
+ register(ConfigResource.class);
+ register(StatusResource.class);
+ register(StopResource.class);
+ register(ResizeResource.class);
+ register(GrowResource.class);
+ register(ShrinkResource.class);
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AuthDynamicFeature.java
----------------------------------------------------------------------
diff --git a/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AuthDynamicFeature.java b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AuthDynamicFeature.java
new file mode 100644
index 0000000..55cd59a
--- /dev/null
+++ b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AuthDynamicFeature.java
@@ -0,0 +1,114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.yarn.appMaster.http;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.drill.yarn.appMaster.http.WebUiPageTree.LogInLogOutPages;
+import org.glassfish.jersey.server.model.AnnotatedMethod;
+
+import javax.annotation.Priority;
+import javax.annotation.security.PermitAll;
+import javax.annotation.security.RolesAllowed;
+import javax.ws.rs.Priorities;
+import javax.ws.rs.container.ContainerRequestContext;
+import javax.ws.rs.container.ContainerRequestFilter;
+import javax.ws.rs.container.DynamicFeature;
+import javax.ws.rs.container.ResourceInfo;
+import javax.ws.rs.core.FeatureContext;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.SecurityContext;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URLEncoder;
+
+/**
+ * Implementation of {@link DynamicFeature}. As part of the setup it adds the
+ * auth check filter {@link AuthCheckFilter} for resources that need to have
+ * user authenticated. If authentication is not done, request is forwarded to
+ * login page.
+ * <p>
+ * Shameless copy of
+ * {@link org.apache.drill.exec.server.rest.auth.DynamicFeature}; the two
+ * implementations should be merged at some point. The difference is only the
+ * log in/log out constant references.
+ */
+
+public class AuthDynamicFeature implements DynamicFeature {
+ private static final Log LOG = LogFactory.getLog(AuthDynamicFeature.class);
+
+ @Override
+ public void configure(final ResourceInfo resourceInfo,
+ final FeatureContext configuration) {
+ AnnotatedMethod am = new AnnotatedMethod(resourceInfo.getResourceMethod());
+
+ // RolesAllowed on the method takes precedence over PermitAll
+ RolesAllowed ra = am.getAnnotation(RolesAllowed.class);
+ if (ra != null) {
+ configuration.register(AuthCheckFilter.INSTANCE);
+ return;
+ }
+
+ // PermitAll takes precedence over RolesAllowed on the class
+ if (am.isAnnotationPresent(PermitAll.class)) {
+ // Do nothing.
+ return;
+ }
+
+ // RolesAllowed on the class takes precedence over PermitAll
+ ra = resourceInfo.getResourceClass().getAnnotation(RolesAllowed.class);
+ if (ra != null) {
+ configuration.register(AuthCheckFilter.INSTANCE);
+ }
+ }
+
+ @Priority(Priorities.AUTHENTICATION) // authentication filter - should go
+ // first before all other filters.
+ private static class AuthCheckFilter implements ContainerRequestFilter {
+ private static AuthCheckFilter INSTANCE = new AuthCheckFilter();
+
+ @Override
+ public void filter(ContainerRequestContext requestContext)
+ throws IOException {
+ final SecurityContext sc = requestContext.getSecurityContext();
+ if (!isUserLoggedIn(sc)) {
+ try {
+ final String destResource = URLEncoder.encode(
+ requestContext.getUriInfo().getRequestUri().toString(), "UTF-8");
+ final URI loginURI = requestContext.getUriInfo().getBaseUriBuilder()
+ .path(LogInLogOutPages.LOGIN_RESOURCE)
+ .queryParam(LogInLogOutPages.REDIRECT_QUERY_PARM, destResource)
+ .build();
+ requestContext
+ .abortWith(Response.temporaryRedirect(loginURI).build());
+ } catch (final Exception ex) {
+ final String errMsg = String.format(
+ "Failed to forward the request to login page: %s",
+ ex.getMessage());
+ LOG.error(errMsg, ex);
+ requestContext
+ .abortWith(Response.serverError().entity(errMsg).build());
+ }
+ }
+ }
+ }
+
+ public static boolean isUserLoggedIn(final SecurityContext sc) {
+ return sc != null && sc.getUserPrincipal() != null;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/ControllerModel.java
----------------------------------------------------------------------
diff --git a/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/ControllerModel.java b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/ControllerModel.java
new file mode 100644
index 0000000..8947df5
--- /dev/null
+++ b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/ControllerModel.java
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.yarn.appMaster.http;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import javax.xml.bind.annotation.XmlRootElement;
+
+import org.apache.drill.yarn.appMaster.AMYarnFacade.YarnAppHostReport;
+import org.apache.drill.yarn.appMaster.ClusterController;
+import org.apache.drill.yarn.appMaster.ClusterControllerImpl;
+import org.apache.drill.yarn.appMaster.ClusterControllerImpl.State;
+import org.apache.drill.yarn.appMaster.ControllerVisitor;
+import org.apache.drill.yarn.appMaster.Scheduler;
+import org.apache.drill.yarn.appMaster.SchedulerStateActions;
+import org.apache.drill.yarn.core.ContainerRequestSpec;
+import org.apache.drill.yarn.core.DrillOnYarnConfig;
+import org.apache.drill.yarn.zk.ZKRegistry;
+
+import com.typesafe.config.Config;
+
+@XmlRootElement
+public class ControllerModel implements ControllerVisitor {
+ public static class ClusterGroupModel {
+ protected String name;
+ protected String type;
+ protected int targetCount;
+ protected int taskCount;
+ protected int liveCount;
+ protected int memory;
+ protected int vcores;
+ protected double disks;
+
+ public String getName( ) { return name; }
+ public String getType( ) { return type; }
+ public int getTargetCount( ) { return targetCount; }
+ public int getTaskCount( ) { return taskCount; }
+ public int getLiveCount( ) { return liveCount; }
+ public int getMemory( ) { return memory; }
+ public int getVcores( ) { return vcores; }
+ public String getDisks( ) {
+ return String.format( "%.02f", disks );
+ }
+ }
+
+ protected String zkConnectStr;
+ protected String zkRoot;
+ protected String zkClusterId;
+ protected ClusterControllerImpl.State state;
+ protected String stateHint;
+ protected boolean supportsDisks;
+ protected int yarnMemory;
+ protected int yarnVcores;
+ protected int yarnNodeCount;
+ protected int taskCount;
+ protected int liveCount;
+ protected int unmanagedCount;
+ protected int targetCount;
+ protected int totalDrillMemory;
+ protected int totalDrillVcores;
+ protected double totalDrillDisks;
+ protected int blacklistCount;
+ protected int freeNodeCount;
+ protected YarnAppHostReport appRpt;
+ protected int refreshSecs;
+ protected List<ClusterGroupModel> groups = new ArrayList<>( );
+
+ public boolean supportsDiskResource( ) { return supportsDisks; }
+ public int getRefreshSecs( ) { return refreshSecs; }
+ public String getZkConnectionStr( ) { return zkConnectStr; }
+ public String getZkRoot( ) { return zkRoot; }
+ public String getZkClusterId( ) { return zkClusterId; }
+ public String getAppId( ) { return appRpt.appId; }
+ public String getRmHost( ) { return appRpt.rmHost; }
+ public String getRmLink( ) { return appRpt.rmUrl; }
+ public String getNmHost( ) { return appRpt.nmHost; }
+ public String getNmLink( ) { return appRpt.nmUrl; }
+ public String getRmAppLink( ) { return appRpt.rmAppUrl; }
+ public String getNmAppLink( ) { return appRpt.nmAppUrl; }
+ public String getState( ) { return state.toString( ); }
+ public String getStateHint( ) { return stateHint; }
+ public int getYarnMemory( ) { return yarnMemory; }
+ public int getYarnVcores( ) { return yarnVcores; }
+ public int getDrillTotalMemory( ) { return totalDrillMemory; }
+ public int getDrillTotalVcores( ) { return totalDrillVcores; }
+ public String getDrillTotalDisks( ) {
+ return String.format( "%.2f", totalDrillDisks );
+ }
+ public int getYarnNodeCount( ) { return yarnNodeCount; }
+ public int getTaskCount( ) { return taskCount; }
+ public int getLiveCount( ) { return liveCount; }
+ public int getUnmanagedCount( ) { return unmanagedCount; }
+ public int getTargetCount( ) { return targetCount; }
+ public List<ClusterGroupModel> getGroups( ) { return groups; }
+ public int getBlacklistCount( ) { return blacklistCount; }
+ public int getFreeNodeCount( ) { return freeNodeCount; }
+
+ private static Map<ClusterControllerImpl.State,String> stateHints = makeStateHints( );
+
+ @Override
+ public void visit(ClusterController controller) {
+ Config config = DrillOnYarnConfig.config();
+ refreshSecs = config.getInt( DrillOnYarnConfig.HTTP_REFRESH_SECS );
+ zkConnectStr = config.getString( DrillOnYarnConfig.ZK_CONNECT );
+ zkRoot = config.getString( DrillOnYarnConfig.ZK_ROOT );
+ zkClusterId = config.getString( DrillOnYarnConfig.CLUSTER_ID );
+
+ ClusterControllerImpl impl = (ClusterControllerImpl) controller;
+ appRpt = impl.getYarn().getAppHostReport();
+
+ state = impl.getState( );
+ stateHint = stateHints.get( state );
+
+ // Removed based on feedback. Users should check the
+ // YARN RM UI instead.
+
+// if ( state == State.LIVE ) {
+// RegisterApplicationMasterResponse resp = impl.getYarn( ).getRegistrationResponse();
+// yarnVcores = resp.getMaximumResourceCapability().getVirtualCores();
+// yarnMemory = resp.getMaximumResourceCapability().getMemory();
+// yarnNodeCount = impl.getYarn( ).getNodeCount();
+// }
+ capturePools( impl );
+ supportsDisks = impl.supportsDiskResource();
+
+ blacklistCount = impl.getNodeInventory( ).getBlacklist( ).size( );
+ freeNodeCount = impl.getFreeNodeCount();
+ }
+
+ private void capturePools(ClusterControllerImpl impl) {
+ for ( SchedulerStateActions pool : impl.getPools( ) ) {
+ ControllerModel.ClusterGroupModel poolModel = new ControllerModel.ClusterGroupModel( );
+ Scheduler sched = pool.getScheduler();
+ ContainerRequestSpec containerSpec = sched.getResource( );
+ poolModel.name = sched.getName();
+ poolModel.type = sched.getType( );
+ poolModel.targetCount = sched.getTarget();
+ poolModel.memory = containerSpec.memoryMb;
+ poolModel.vcores = containerSpec.vCores;
+ poolModel.disks = containerSpec.disks;
+ poolModel.taskCount = pool.getTaskCount();
+ poolModel.liveCount = pool.getLiveCount( );
+ targetCount += poolModel.targetCount;
+ taskCount += poolModel.taskCount;
+ liveCount += poolModel.liveCount;
+ totalDrillMemory += poolModel.liveCount * poolModel.memory;
+ totalDrillVcores += poolModel.liveCount * poolModel.vcores;
+ totalDrillDisks += poolModel.liveCount * poolModel.disks;
+ groups.add( poolModel );
+ }
+ if ( state != State.LIVE ) {
+ targetCount = 0;
+ }
+ }
+
+ /**
+ * Count the unmanaged drillbits. Do this as a separate call, not via the
+ * {@link #visit(ClusterController) visit} method, to avoid locking both
+ * the cluster controller and ZK registry.
+ *
+ * @param controller
+ */
+
+ public void countStrayDrillbits(ClusterController controller) {
+ ZKRegistry zkRegistry = (ZKRegistry) controller.getProperty( ZKRegistry.CONTROLLER_PROPERTY );
+ if ( zkRegistry != null ) {
+ unmanagedCount = zkRegistry.listUnmanagedDrillits().size();
+ }
+ }
+
+ /**
+ * Create a table of user-visible descriptions for each controller state.
+ *
+ * @return
+ */
+
+ private static Map<State, String> makeStateHints() {
+ Map<ClusterControllerImpl.State,String> hints = new HashMap<>( );
+ // UI likely will never display the FAILED state.
+ hints.put( ClusterControllerImpl.State.START, "AM is starting up." );
+ hints.put( ClusterControllerImpl.State.LIVE, "AM is operating normally." );
+ hints.put( ClusterControllerImpl.State.ENDING, "AM is shutting down." );
+ // UI will never display the ENDED state.
+ hints.put( ClusterControllerImpl.State.ENDED, "AM is about to exit." );
+ // UI will never display the FAILED state.
+ hints.put( ClusterControllerImpl.State.FAILED, "AM failed to start and is about to exit." );
+ return hints;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/PageTree.java
----------------------------------------------------------------------
diff --git a/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/PageTree.java b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/PageTree.java
new file mode 100644
index 0000000..e4d5dc1
--- /dev/null
+++ b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/PageTree.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.yarn.appMaster.http;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.ws.rs.core.SecurityContext;
+
+import org.apache.drill.exec.server.rest.auth.DrillUserPrincipal;
+import org.apache.drill.yarn.appMaster.Dispatcher;
+import org.apache.drill.yarn.core.DrillOnYarnConfig;
+import org.glassfish.jersey.server.ResourceConfig;
+
+import com.typesafe.config.Config;
+
+/**
+ * Base class for a tree of web pages (or REST resources) represented
+ * as POJOs. Since the AM web UI is simple, this is the most convenient,
+ * compact way to implement the UI.
+ */
+
+public class PageTree extends ResourceConfig {
+ // These items are a bit clumsy. We need them, but we can't make them
+ // instance variables without a bunch of messiness in the page classes.
+ // So, we let them be static. No harm in setting them multiple times.
+
+ static Dispatcher dispatcher;
+ static Config config;
+
+ public PageTree(Dispatcher dispatcher) {
+ PageTree.dispatcher = dispatcher;
+ config = DrillOnYarnConfig.config();
+ }
+
+ /**
+ * Creates a FreeMarker model that contains two top-level items:
+ * the model itself (as in the default implementation) and the
+ * cluster name (used as a title on each UI page.)
+ *
+ * @param base
+ * @return
+ */
+
+ public static Map<String, Object> toModel(SecurityContext sc, Object base) {
+ Map<String, Object> model = new HashMap<>();
+ model.put("model", base);
+ return toMapModel(sc, model);
+ }
+
+ public static Map<String, Object> toMapModel(SecurityContext sc,
+ Map<String, Object> model) {
+ model.put("clusterName", config.getString(DrillOnYarnConfig.APP_NAME));
+ boolean useAuth = AMSecurityManagerImpl.isEnabled();
+ final boolean isUserLoggedIn = (useAuth)
+ ? AuthDynamicFeature.isUserLoggedIn(sc) : false;
+ model.put("showLogin", useAuth && !isUserLoggedIn);
+ model.put("showLogout", isUserLoggedIn);
+ model.put("docsLink", config.getString(DrillOnYarnConfig.HTTP_DOCS_LINK));
+ String userName = isUserLoggedIn ? sc.getUserPrincipal().getName()
+ : DrillUserPrincipal.ANONYMOUS_USER;
+ model.put("loggedInUserName", userName);
+ return model;
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/WebServer.java
----------------------------------------------------------------------
diff --git a/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/WebServer.java b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/WebServer.java
new file mode 100644
index 0000000..aeeafde
--- /dev/null
+++ b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/WebServer.java
@@ -0,0 +1,467 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.yarn.appMaster.http;
+
+import static org.apache.drill.exec.server.rest.auth.DrillUserPrincipal.ADMIN_ROLE;
+
+import java.math.BigInteger;
+import java.security.KeyPair;
+import java.security.KeyPairGenerator;
+import java.security.KeyStore;
+import java.security.Principal;
+import java.security.SecureRandom;
+import java.security.cert.X509Certificate;
+import java.util.Collections;
+import java.util.Date;
+import java.util.Set;
+
+import javax.servlet.http.HttpSession;
+import javax.servlet.http.HttpSessionEvent;
+import javax.servlet.http.HttpSessionListener;
+
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.drill.yarn.appMaster.Dispatcher;
+import org.apache.drill.yarn.core.DrillOnYarnConfig;
+import org.bouncycastle.asn1.x500.X500NameBuilder;
+import org.bouncycastle.asn1.x500.style.BCStyle;
+import org.bouncycastle.cert.X509v3CertificateBuilder;
+import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter;
+import org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder;
+import org.bouncycastle.operator.ContentSigner;
+import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder;
+import org.eclipse.jetty.http.HttpVersion;
+import org.eclipse.jetty.security.ConstraintMapping;
+import org.eclipse.jetty.security.ConstraintSecurityHandler;
+import org.eclipse.jetty.security.DefaultIdentityService;
+import org.eclipse.jetty.security.DefaultUserIdentity;
+import org.eclipse.jetty.security.IdentityService;
+import org.eclipse.jetty.security.LoginService;
+import org.eclipse.jetty.security.SecurityHandler;
+import org.eclipse.jetty.security.authentication.FormAuthenticator;
+import org.eclipse.jetty.security.authentication.SessionAuthentication;
+import org.eclipse.jetty.server.HttpConfiguration;
+import org.eclipse.jetty.server.HttpConnectionFactory;
+import org.eclipse.jetty.server.SecureRequestCustomizer;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.server.ServerConnector;
+import org.eclipse.jetty.server.SessionManager;
+import org.eclipse.jetty.server.SslConnectionFactory;
+import org.eclipse.jetty.server.UserIdentity;
+import org.eclipse.jetty.server.handler.ErrorHandler;
+import org.eclipse.jetty.server.session.HashSessionManager;
+import org.eclipse.jetty.server.session.SessionHandler;
+import org.eclipse.jetty.servlet.DefaultServlet;
+import org.eclipse.jetty.servlet.ServletContextHandler;
+import org.eclipse.jetty.servlet.ServletHolder;
+import org.eclipse.jetty.util.resource.Resource;
+import org.eclipse.jetty.util.ssl.SslContextFactory;
+import org.glassfish.jersey.servlet.ServletContainer;
+import org.joda.time.DateTime;
+
+import com.google.common.collect.ImmutableSet;
+import com.typesafe.config.Config;
+
+/**
+ * Wrapper around the Jetty web server.
+ * <p>
+ * Adapted from Drill's drill.exec.WebServer class. Would be good to create a
+ * common base class later, but the goal for the initial project is to avoid
+ * Drill code changes.
+ *
+ * @see <a href=
+ * "http://www.eclipse.org/jetty/documentation/current/embedding-jetty.html">
+ * Jetty Embedding documentation</a>
+ */
+
+public class WebServer implements AutoCloseable {
+ private static final Log LOG = LogFactory.getLog(WebServer.class);
+ private final Server jettyServer;
+ private Dispatcher dispatcher;
+
+ public WebServer(Dispatcher dispatcher) {
+ this.dispatcher = dispatcher;
+ Config config = DrillOnYarnConfig.config();
+ if (config.getBoolean(DrillOnYarnConfig.HTTP_ENABLED)) {
+ jettyServer = new Server();
+ } else {
+ jettyServer = null;
+ }
+ }
+
+ /**
+ * Start the web server including setup.
+ *
+ * @throws Exception
+ */
+ public void start() throws Exception {
+ if (jettyServer == null) {
+ return;
+ }
+
+ build();
+ jettyServer.start();
+ }
+
+ private void build() throws Exception {
+ Config config = DrillOnYarnConfig.config();
+ buildConnector(config);
+ buildServlets(config);
+ }
+
+ private void buildConnector(Config config) throws Exception {
+ final ServerConnector serverConnector;
+ if (config.getBoolean(DrillOnYarnConfig.HTTP_ENABLE_SSL)) {
+ serverConnector = createHttpsConnector(config);
+ } else {
+ serverConnector = createHttpConnector(config);
+ }
+ jettyServer.addConnector(serverConnector);
+ }
+
+ /**
+ * Build the web app with embedded servlets.
+ * <p>
+ * <b>ServletContextHandler</b>: is a Jetty-provided handler that add the
+ * extra bits needed to set up the context that servlets expect. Think of it
+ * as an adapter between the (simple) Jetty handler and the (more complex)
+ * servlet API.
+ *
+ */
+ private void buildServlets(Config config) {
+
+ final ServletContextHandler servletContextHandler = new ServletContextHandler(
+ null, "/");
+ servletContextHandler.setErrorHandler(createErrorHandler());
+ jettyServer.setHandler(servletContextHandler);
+
+ // Servlet holder for the pages of the Drill AM web app. The web app is a
+ // javax.ws application driven from annotations. The servlet holder "does
+ // the right thing" to drive the application, which is rooted at "/".
+ // The servlet container comes from Jersey, and manages the servlet
+ // lifecycle.
+
+ final ServletHolder servletHolder = new ServletHolder(
+ new ServletContainer(new WebUiPageTree(dispatcher)));
+ servletHolder.setInitOrder(1);
+ servletContextHandler.addServlet(servletHolder, "/*");
+
+ final ServletHolder restHolder = new ServletHolder(
+ new ServletContainer(new AmRestApi(dispatcher)));
+ restHolder.setInitOrder(2);
+ servletContextHandler.addServlet(restHolder, "/rest/*");
+
+ // Static resources (CSS, images, etc.)
+
+ setupStaticResources(servletContextHandler);
+
+ // Security, if requested.
+
+ if (AMSecurityManagerImpl.isEnabled()) {
+ servletContextHandler.setSecurityHandler(createSecurityHandler(config));
+ servletContextHandler.setSessionHandler(createSessionHandler(config,
+ servletContextHandler.getSecurityHandler()));
+ }
+ }
+
+ private ErrorHandler createErrorHandler() {
+ // Error handler to show detailed errors.
+ // Should probably be turned off in production.
+ final ErrorHandler errorHandler = new ErrorHandler();
+ errorHandler.setShowStacks(true);
+ errorHandler.setShowMessageInTitle(true);
+ return errorHandler;
+ }
+
+ private void setupStaticResources(
+ ServletContextHandler servletContextHandler) {
+
+ // Access to static resources (JS pages, images, etc.)
+ // The static resources themselves come from Drill exec sub-project
+ // and the Drill-on-YARN project.
+ //
+ // We handle static content this way because we want to do it
+ // in the context of a servlet app, so we need the Jetty "default servlet"
+ // that handles static content. That servlet is designed to take its
+ // properties
+ // from the web.xml, file; but can also take them programmatically as done
+ // here. (The Jetty manual suggests a simpler handler, but that is a
+ // non-Servlet
+ // version.)
+
+ final ServletHolder staticHolder = new ServletHolder("static",
+ DefaultServlet.class);
+ staticHolder.setInitParameter("resourceBase",
+ Resource.newClassPathResource("/rest/static").toString());
+ staticHolder.setInitParameter("dirAllowed", "false");
+ staticHolder.setInitParameter("pathInfoOnly", "true");
+ servletContextHandler.addServlet(staticHolder, "/static/*");
+
+ final ServletHolder amStaticHolder = new ServletHolder("am-static",
+ DefaultServlet.class);
+ amStaticHolder.setInitParameter("resourceBase",
+ Resource.newClassPathResource("/drill-am/static").toString());
+ amStaticHolder.setInitParameter("dirAllowed", "false");
+ amStaticHolder.setInitParameter("pathInfoOnly", "true");
+ servletContextHandler.addServlet(amStaticHolder, "/drill-am/static/*");
+ }
+
+ public static class AMUserPrincipal implements Principal {
+ public final String userName;
+
+ public AMUserPrincipal(String userName) {
+ this.userName = userName;
+ }
+
+ @Override
+ public String getName() {
+ return userName;
+ }
+ }
+
+ public static class AmLoginService implements LoginService {
+ private AMSecurityManager securityMgr;
+ protected IdentityService identityService = new DefaultIdentityService();
+
+ public AmLoginService(AMSecurityManager securityMgr) {
+ this.securityMgr = securityMgr;
+ }
+
+ @Override
+ public String getName() {
+ return "drill-am";
+ }
+
+ @Override
+ public UserIdentity login(String username, Object credentials) {
+ if (!securityMgr.login(username, (String) credentials)) {
+ return null;
+ }
+ return new DefaultUserIdentity(null, new AMUserPrincipal(username),
+ new String[] { ADMIN_ROLE });
+ }
+
+ @Override
+ public boolean validate(UserIdentity user) {
+ return true;
+ }
+
+ @Override
+ public IdentityService getIdentityService() {
+ return identityService;
+ }
+
+ @Override
+ public void setIdentityService(IdentityService service) {
+ this.identityService = service;
+ }
+
+ @Override
+ public void logout(UserIdentity user) {
+ }
+
+ // @Override
+ // protected UserIdentity loadUser(String username) {
+ // // TODO Auto-generated method stub
+ // return null;
+ // }
+ //
+ // @Override
+ // protected void loadUsers() throws IOException {
+ // putUser( "fred", new Password( "wilma" ), new String[] { ADMIN_ROLE } );
+ // }
+
+ }
+
+ /**
+ * @return
+ * @return
+ * @see http://www.eclipse.org/jetty/documentation/current/embedded-examples.html
+ */
+
+ private ConstraintSecurityHandler createSecurityHandler(Config config) {
+ ConstraintSecurityHandler security = new ConstraintSecurityHandler();
+
+ Set<String> knownRoles = ImmutableSet.of(ADMIN_ROLE);
+ security.setConstraintMappings(Collections.<ConstraintMapping> emptyList(),
+ knownRoles);
+
+ security.setAuthenticator(new FormAuthenticator("/login", "/login", true));
+ security
+ .setLoginService(new AmLoginService(AMSecurityManagerImpl.instance()));
+
+ return security;
+ }
+
+ /**
+ * @return A {@link SessionHandler} which contains a
+ * {@link HashSessionManager}
+ */
+ private SessionHandler createSessionHandler(Config config,
+ final SecurityHandler securityHandler) {
+ SessionManager sessionManager = new HashSessionManager();
+ sessionManager.setMaxInactiveInterval(
+ config.getInt(DrillOnYarnConfig.HTTP_SESSION_MAX_IDLE_SECS));
+ sessionManager.addEventListener(new HttpSessionListener() {
+ @Override
+ public void sessionCreated(HttpSessionEvent se) {
+ // No-op
+ }
+
+ @Override
+ public void sessionDestroyed(HttpSessionEvent se) {
+ final HttpSession session = se.getSession();
+ if (session == null) {
+ return;
+ }
+
+ final Object authCreds = session
+ .getAttribute(SessionAuthentication.__J_AUTHENTICATED);
+ if (authCreds != null) {
+ final SessionAuthentication sessionAuth = (SessionAuthentication) authCreds;
+ securityHandler.logout(sessionAuth);
+ session.removeAttribute(SessionAuthentication.__J_AUTHENTICATED);
+ }
+ }
+ });
+
+ return new SessionHandler(sessionManager);
+ }
+
+ /**
+ * Create HTTP connector.
+ *
+ * @return Initialized {@link ServerConnector} instance for HTTP connections.
+ * @throws Exception
+ */
+ private ServerConnector createHttpConnector(Config config) throws Exception {
+ LOG.info("Setting up HTTP connector for web server");
+ final HttpConfiguration httpConfig = new HttpConfiguration();
+ final ServerConnector httpConnector = new ServerConnector(jettyServer,
+ new HttpConnectionFactory(httpConfig));
+ httpConnector.setPort(config.getInt(DrillOnYarnConfig.HTTP_PORT));
+
+ return httpConnector;
+ }
+
+ /**
+ * Create an HTTPS connector for given jetty server instance. If the admin has
+ * specified keystore/truststore settings they will be used else a self-signed
+ * certificate is generated and used.
+ * <p>
+ * This is a shameless copy of
+ * {@link org.apache.drill.exec.server.rest.Webserver#createHttpsConnector( )}.
+ * The two should be merged at some point. The primary issue is that the Drill
+ * version is tightly coupled to Drillbit configuration.
+ *
+ * @return Initialized {@link ServerConnector} for HTTPS connections.
+ * @throws Exception
+ */
+
+ private ServerConnector createHttpsConnector(Config config) throws Exception {
+ LOG.info("Setting up HTTPS connector for web server");
+
+ final SslContextFactory sslContextFactory = new SslContextFactory();
+
+ // if (config.hasPath(ExecConstants.HTTP_KEYSTORE_PATH) &&
+ // !Strings.isNullOrEmpty(config.getString(ExecConstants.HTTP_KEYSTORE_PATH)))
+ // {
+ // LOG.info("Using configured SSL settings for web server");
+ // sslContextFactory.setKeyStorePath(config.getString(ExecConstants.HTTP_KEYSTORE_PATH));
+ // sslContextFactory.setKeyStorePassword(config.getString(ExecConstants.HTTP_KEYSTORE_PASSWORD));
+ //
+ // // TrustStore and TrustStore password are optional
+ // if (config.hasPath(ExecConstants.HTTP_TRUSTSTORE_PATH)) {
+ // sslContextFactory.setTrustStorePath(config.getString(ExecConstants.HTTP_TRUSTSTORE_PATH));
+ // if (config.hasPath(ExecConstants.HTTP_TRUSTSTORE_PASSWORD)) {
+ // sslContextFactory.setTrustStorePassword(config.getString(ExecConstants.HTTP_TRUSTSTORE_PASSWORD));
+ // }
+ // }
+ // } else {
+ LOG.info("Using generated self-signed SSL settings for web server");
+ final SecureRandom random = new SecureRandom();
+
+ // Generate a private-public key pair
+ final KeyPairGenerator keyPairGenerator = KeyPairGenerator
+ .getInstance("RSA");
+ keyPairGenerator.initialize(1024, random);
+ final KeyPair keyPair = keyPairGenerator.generateKeyPair();
+
+ final DateTime now = DateTime.now();
+
+ // Create builder for certificate attributes
+ final X500NameBuilder nameBuilder = new X500NameBuilder(BCStyle.INSTANCE)
+ .addRDN(BCStyle.OU, "Apache Drill (auth-generated)")
+ .addRDN(BCStyle.O, "Apache Software Foundation (auto-generated)")
+ .addRDN(BCStyle.CN, "Drill AM");
+
+ final Date notBefore = now.minusMinutes(1).toDate();
+ final Date notAfter = now.plusYears(5).toDate();
+ final BigInteger serialNumber = new BigInteger(128, random);
+
+ // Create a certificate valid for 5years from now.
+ final X509v3CertificateBuilder certificateBuilder = new JcaX509v3CertificateBuilder(
+ nameBuilder.build(), // attributes
+ serialNumber, notBefore, notAfter, nameBuilder.build(),
+ keyPair.getPublic());
+
+ // Sign the certificate using the private key
+ final ContentSigner contentSigner = new JcaContentSignerBuilder(
+ "SHA256WithRSAEncryption").build(keyPair.getPrivate());
+ final X509Certificate certificate = new JcaX509CertificateConverter()
+ .getCertificate(certificateBuilder.build(contentSigner));
+
+ // Check the validity
+ certificate.checkValidity(now.toDate());
+
+ // Make sure the certificate is self-signed.
+ certificate.verify(certificate.getPublicKey());
+
+ // Generate a random password for keystore protection
+ final String keyStorePasswd = RandomStringUtils.random(20);
+ final KeyStore keyStore = KeyStore.getInstance("JKS");
+ keyStore.load(null, null);
+ keyStore.setKeyEntry("DrillAutoGeneratedCert", keyPair.getPrivate(),
+ keyStorePasswd.toCharArray(),
+ new java.security.cert.Certificate[] { certificate });
+
+ sslContextFactory.setKeyStore(keyStore);
+ sslContextFactory.setKeyStorePassword(keyStorePasswd);
+ // }
+
+ final HttpConfiguration httpsConfig = new HttpConfiguration();
+ httpsConfig.addCustomizer(new SecureRequestCustomizer());
+
+ // SSL Connector
+ final ServerConnector sslConnector = new ServerConnector(jettyServer,
+ new SslConnectionFactory(sslContextFactory,
+ HttpVersion.HTTP_1_1.asString()),
+ new HttpConnectionFactory(httpsConfig));
+ sslConnector.setPort(config.getInt(DrillOnYarnConfig.HTTP_PORT));
+
+ return sslConnector;
+ }
+
+ @Override
+ public void close() throws Exception {
+ if (jettyServer != null) {
+ jettyServer.stop();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/WebUiPageTree.java
----------------------------------------------------------------------
diff --git a/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/WebUiPageTree.java b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/WebUiPageTree.java
new file mode 100644
index 0000000..fc44e45
--- /dev/null
+++ b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/WebUiPageTree.java
@@ -0,0 +1,527 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.yarn.appMaster.http;
+
+import static org.apache.drill.exec.server.rest.auth.DrillUserPrincipal.ADMIN_ROLE;
+
+import java.net.URI;
+import java.net.URLDecoder;
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.annotation.security.PermitAll;
+import javax.annotation.security.RolesAllowed;
+import javax.inject.Inject;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import javax.servlet.http.HttpSession;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.FormParam;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.SecurityContext;
+import javax.ws.rs.core.UriBuilder;
+import javax.ws.rs.core.UriInfo;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.drill.yarn.appMaster.Dispatcher;
+import org.apache.drill.yarn.core.DoYUtil;
+import org.apache.drill.yarn.core.DrillOnYarnConfig;
+import org.eclipse.jetty.security.authentication.FormAuthenticator;
+import org.glassfish.jersey.server.filter.RolesAllowedDynamicFeature;
+import org.glassfish.jersey.server.mvc.Viewable;
+import org.glassfish.jersey.server.mvc.freemarker.FreemarkerMvcFeature;
+
+/**
+ * The Drill AM web UI. The format is highly compact. We use javax.ws.rs to mark
+ * up a Pojo with page path, permissions and HTTP methods. The ADMIN_ROLE is
+ * reused from Drill's web UI.
+ * <p>
+ * In general, all pages require admin role, except for two: the login page and
+ * the redirect page which the YARN web UI follows to start the AM UI.
+ */
+
+public class WebUiPageTree extends PageTree {
+
+ /**
+ * Main DoY page that displays cluster status, and the status of
+ * the resource groups. Available only to the admin user when
+ * DoY is secured.
+ */
+
+ @Path("/")
+ @RolesAllowed(ADMIN_ROLE)
+ public static class RootPage {
+ @Inject
+ SecurityContext sc;
+
+ @GET
+ @Produces(MediaType.TEXT_HTML)
+ public Viewable getRoot() {
+ ControllerModel model = new ControllerModel();
+ dispatcher.getController().visit(model);
+ model.countStrayDrillbits(dispatcher.getController());
+ return new Viewable("/drill-am/index.ftl", toModel(sc, model));
+ }
+ }
+
+ /**
+ * Pages, adapted from Drill, that display the login and logout pages.
+ * Login uses the security mechanism, again borrowed from Drill, to
+ * validate the user against either the simple user/password
+ * configured in DoY, or the user who launched DoY using the
+ * Drill security mechanism.
+ */
+
+ @Path("/")
+ @PermitAll
+ public static class LogInLogOutPages {
+ @Inject
+ SecurityContext sc;
+
+ public static final String REDIRECT_QUERY_PARM = "redirect";
+ public static final String LOGIN_RESOURCE = "login";
+
+ @GET
+ @Path("/login")
+ @Produces(MediaType.TEXT_HTML)
+ public Viewable getLoginPage(@Context HttpServletRequest request,
+ @Context HttpServletResponse response, @Context SecurityContext sc,
+ @Context UriInfo uriInfo,
+ @QueryParam(REDIRECT_QUERY_PARM) String redirect) throws Exception {
+
+ if (!StringUtils.isEmpty(redirect)) {
+ // If the URL has redirect in it, set the redirect URI in session, so
+ // that after the login is successful, request
+ // is forwarded to the redirect page.
+ final HttpSession session = request.getSession(true);
+ final URI destURI = UriBuilder
+ .fromUri(URLDecoder.decode(redirect, "UTF-8")).build();
+ session.setAttribute(FormAuthenticator.__J_URI, destURI.toString());
+ }
+
+ return new Viewable("/drill-am/login.ftl", toModel(sc, (Object) null));
+ }
+
+ // Request type is POST because POST request which contains the login
+ // credentials are invalid and the request is
+ // dispatched here directly.
+ @POST
+ @Path("/login")
+ @Produces(MediaType.TEXT_HTML)
+ public Viewable getLoginPageAfterValidationError() {
+ return new Viewable("/drill-am/login.ftl",
+ toModel(sc, "Invalid user name or password."));
+ }
+
+ @GET
+ @Path("/logout")
+ public Viewable logout(@Context HttpServletRequest req,
+ @Context HttpServletResponse resp) throws Exception {
+ final HttpSession session = req.getSession();
+ if (session != null) {
+ session.invalidate();
+ }
+
+ req.getRequestDispatcher("/login").forward(req, resp);
+ return null;
+ }
+ }
+
+ /**
+ * DoY provides a link to YARN to display the AM UI. YARN wants to display the
+ * linked page in a frame, which does not play well with the DoY UI. To avoid
+ * this, we give YARN a link to this redirect page which does nothing other
+ * than to redirect the browser to the (full) DoY main UI.
+ */
+
+ @Path("/redirect")
+ @PermitAll
+ public static class RedirectPage {
+ @GET
+ @Produces(MediaType.TEXT_HTML)
+ public Viewable getRoot() {
+ Map<String, String> map = new HashMap<>();
+ String baseUrl = DoYUtil.unwrapAmUrl(dispatcher.getTrackingUrl());
+ map.put("amLink", baseUrl);
+ map.put("clusterName", config.getString(DrillOnYarnConfig.APP_NAME));
+ return new Viewable("/drill-am/redirect.ftl", map);
+ }
+ }
+
+ /**
+ * Display the configuration page which displays the contents of
+ * DoY and selected Drill config as name/value pairs. Visible only
+ * to the admin when DoY is secure.
+ */
+
+ @Path("/config")
+ @RolesAllowed(ADMIN_ROLE)
+ public static class ConfigPage {
+ @Inject
+ private SecurityContext sc;
+
+ @GET
+ @Produces(MediaType.TEXT_HTML)
+ public Viewable getRoot() {
+ return new Viewable("/drill-am/config.ftl",
+ toModel(sc, DrillOnYarnConfig.instance().getPairs()));
+ }
+ }
+
+ /**
+ * Displays the list of Drillbits showing details for each Drillbit.
+ * (DoY uses the generic term "task", but, at present, the only
+ * task that DoY runs is a Drillbit.
+ */
+
+ @Path("/drillbits")
+ @RolesAllowed(ADMIN_ROLE)
+ public static class DrillbitsPage {
+ @Inject
+ private SecurityContext sc;
+
+ @GET
+ @Produces(MediaType.TEXT_HTML)
+ public Viewable getRoot() {
+ AbstractTasksModel.TasksModel model = new AbstractTasksModel.TasksModel();
+ dispatcher.getController().visitTasks(model);
+ model.listAnomalies(dispatcher.getController());
+ model.sortTasks();
+
+ // Done this funky way because FreeMarker only understands lists if they
+ // are members of a hash (grumble, grumble...)
+
+ Map<String, Object> map = new HashMap<>();
+ map.put("model", model);
+ map.put("tasks", model.getTasks());
+ if (model.hasUnmanagedDrillbits()) {
+ map.put("strays", model.getUnnamaged());
+ }
+ if (model.hasBlacklist()) {
+ map.put("blacklist", model.getBlacklist());
+ }
+ map.put("showDisks", dispatcher.getController().supportsDiskResource());
+ map.put("refreshSecs", DrillOnYarnConfig.config()
+ .getInt(DrillOnYarnConfig.HTTP_REFRESH_SECS));
+ return new Viewable("/drill-am/tasks.ftl", toMapModel(sc, map));
+ }
+ }
+
+ /**
+ * Displays a warning page to ask the user if they want to cancel
+ * a Drillbit. This is a bit old-school; we display this as a
+ * separate page. A good future enhancement is to do this as
+ * a pop-up in Javascript. The GET request display the confirmation
+ * page, the PUT request confirms cancellation and does the deed.
+ * The task to be cancelled appears as a query parameter:
+ * <pre>.../cancel?id=<task id></pre>
+ */
+
+ @Path("/cancel/")
+ @RolesAllowed(ADMIN_ROLE)
+ public static class CancelDrillbitPage {
+ @Inject
+ private SecurityContext sc;
+
+ @QueryParam("id")
+ private int id;
+
+ @GET
+ @Produces(MediaType.TEXT_HTML)
+ public Viewable getPage() {
+ ConfirmShrink confirm;
+ if (dispatcher.getController().isTaskLive(id)) {
+ confirm = new ConfirmShrink(ConfirmShrink.Mode.KILL);
+ } else {
+ confirm = new ConfirmShrink(ConfirmShrink.Mode.CANCEL);
+ }
+ confirm.id = id;
+ return new Viewable("/drill-am/shrink-warning.ftl", toModel(sc, confirm));
+ }
+
+ @POST
+ @Produces(MediaType.TEXT_HTML)
+ public Viewable postPage() {
+ Acknowledge ack;
+ if (dispatcher.getController().cancelTask(id)) {
+ ack = new Acknowledge(Acknowledge.Mode.CANCELLED);
+ } else {
+ ack = new Acknowledge(Acknowledge.Mode.INVALID_TASK);
+ }
+ ack.value = id;
+ return new Viewable("/drill-am/confirm.ftl", toModel(sc, ack));
+ }
+ }
+
+ /**
+ * Displays a history of completed tasks which indicates failed or cancelled
+ * Drillbits. Helps the admin to understand what has been happening on the
+ * cluster if Drillbits have died.
+ */
+
+ @Path("/history")
+ @RolesAllowed(ADMIN_ROLE)
+ public static class HistoryPage {
+ @Inject
+ SecurityContext sc;
+
+ @GET
+ @Produces(MediaType.TEXT_HTML)
+ public Viewable getRoot() {
+ AbstractTasksModel.HistoryModel model = new AbstractTasksModel.HistoryModel();
+ dispatcher.getController().visit(model);
+ Map<String, Object> map = new HashMap<>();
+ map.put("model", model.results);
+ map.put("refreshSecs", DrillOnYarnConfig.config()
+ .getInt(DrillOnYarnConfig.HTTP_REFRESH_SECS));
+ return new Viewable("/drill-am/history.ftl", toMapModel(sc, map));
+ }
+ }
+
+ /**
+ * Page that lets the admin change the cluster size or shut down the cluster.
+ */
+
+ @Path("/manage")
+ @RolesAllowed(ADMIN_ROLE)
+ public static class ManagePage {
+ @Inject
+ SecurityContext sc;
+
+ @GET
+ @Produces(MediaType.TEXT_HTML)
+ public Viewable getRoot() {
+ ControllerModel model = new ControllerModel();
+ dispatcher.getController().visit(model);
+ return new Viewable("/drill-am/manage.ftl", toModel(sc, model));
+ }
+ }
+
+ /**
+ * Passes information to the acknowledgement page.
+ */
+
+ public static class Acknowledge {
+ public enum Mode {
+ STOPPED, INVALID_RESIZE, INVALID_ACTION, NULL_RESIZE, RESIZED, CANCELLED, INVALID_TASK
+ };
+
+ Mode mode;
+ Object value;
+
+ public Acknowledge(Mode mode) {
+ this.mode = mode;
+ }
+
+ public String getType() {
+ return mode.toString();
+ }
+
+ public Object getValue() {
+ return value;
+ }
+ }
+
+ /**
+ * Passes information to the confirmation page.
+ */
+
+ public static class ConfirmShrink {
+ public enum Mode {
+ SHRINK, STOP, CANCEL, KILL
+ };
+
+ Mode mode;
+ int value;
+ int id;
+
+ public ConfirmShrink(Mode mode) {
+ this.mode = mode;
+ }
+
+ public boolean isStop() {
+ return mode == Mode.STOP;
+ }
+
+ public boolean isCancel() {
+ return mode == Mode.CANCEL;
+ }
+
+ public boolean isKill() {
+ return mode == Mode.KILL;
+ }
+
+ public boolean isShrink() {
+ return mode == Mode.SHRINK;
+ }
+
+ public int getCount() {
+ return value;
+ }
+
+ public int getId() {
+ return id;
+ }
+ }
+
+ /**
+ * Confirm that the user wants to resize the cluster. Displays a warning if
+ * the user wants to shrink the cluster, since, at present, doing so will
+ * kill any in-flight queries. The GET request display the warning,
+ * the POST request confirms the action. The action itself is provided
+ * as query parameters:
+ * <pre>.../resize?type=<type>&n=<quantity></pre>
+ * Where the type is one of "resize", "grow", "shrink" or
+ * "force-shrink" and n is the associated quantity.
+ * <p>
+ * Note that the manage page only provides the "resize" option; the
+ * grow and shrink options were removed from the Web UI and are only
+ * visible through the REST API.
+ */
+
+ @Path("/resize")
+ @RolesAllowed(ADMIN_ROLE)
+ public static class ResizePage {
+ @Inject
+ SecurityContext sc;
+
+ @FormParam("n")
+ int n;
+ @FormParam("type")
+ String type;
+
+ @POST
+ @Consumes(MediaType.APPLICATION_FORM_URLENCODED)
+ @Produces(MediaType.TEXT_HTML)
+ public Viewable resize() {
+ int curSize = dispatcher.getController().getTargetCount();
+ if (n <= 0) {
+ Acknowledge confirm = new Acknowledge(Acknowledge.Mode.INVALID_RESIZE);
+ confirm.value = n;
+ return new Viewable("/drill-am/confirm.ftl", toModel(sc, confirm));
+ }
+ if (type == null) {
+ type = "null";
+ }
+ int newSize;
+ boolean confirmed = false;
+ if (type.equalsIgnoreCase("resize")) {
+ newSize = n;
+ } else if (type.equalsIgnoreCase("grow")) {
+ newSize = curSize + n;
+ } else if (type.equalsIgnoreCase("shrink")) {
+ newSize = curSize - n;
+ } else if (type.equalsIgnoreCase("force-shrink")) {
+ newSize = curSize - n;
+ confirmed = true;
+ } else {
+ Acknowledge confirm = new Acknowledge(Acknowledge.Mode.INVALID_ACTION);
+ confirm.value = type;
+ return new Viewable("/drill-am/confirm.ftl", toModel(sc, confirm));
+ }
+
+ if (curSize == newSize) {
+ Acknowledge confirm = new Acknowledge(Acknowledge.Mode.NULL_RESIZE);
+ confirm.value = newSize;
+ return new Viewable("/drill-am/confirm.ftl", toModel(sc, confirm));
+ } else if (confirmed || curSize < newSize) {
+ Acknowledge confirm = new Acknowledge(Acknowledge.Mode.RESIZED);
+ confirm.value = dispatcher.getController().resizeTo(newSize);
+ return new Viewable("/drill-am/confirm.ftl", toModel(sc, confirm));
+ } else {
+ ConfirmShrink confirm = new ConfirmShrink(ConfirmShrink.Mode.SHRINK);
+ confirm.value = curSize - newSize;
+ return new Viewable("/drill-am/shrink-warning.ftl",
+ toModel(sc, confirm));
+ }
+ }
+ }
+
+ /**
+ * Confirmation page when the admin asks to stop the cluster.
+ * The GET request displays the confirmation, the POST does
+ * the deed. As for other confirmation pages, this is an old-style,
+ * quick & dirty solution. A more modern solution would be to use JavaScript
+ * to pop up a confirmation dialog.
+ */
+
+ @Path("/stop/")
+ @RolesAllowed(ADMIN_ROLE)
+ public static class StopPage {
+ @Inject
+ SecurityContext sc;
+
+ @GET
+ @Produces(MediaType.TEXT_HTML)
+ public Viewable requestStop() {
+ ConfirmShrink confirm = new ConfirmShrink(ConfirmShrink.Mode.STOP);
+ return new Viewable("/drill-am/shrink-warning.ftl", toModel(sc, confirm));
+ }
+
+ @POST
+ @Produces(MediaType.TEXT_HTML)
+ public Viewable doStop() {
+ dispatcher.getController().shutDown();
+ Acknowledge confirm = new Acknowledge(Acknowledge.Mode.STOPPED);
+ return new Viewable("/drill-am/confirm.ftl", toModel(sc, confirm));
+ }
+ }
+
+ /**
+ * Build the pages for the Web UI using Freemarker to implement the
+ * MVC mechanism. This class builds on a rather complex mechanism; understand
+ * that to understand what the lines of code below are doing.
+ *
+ * @param dispatcher the DoY AM dispatcher that receives requests for
+ * information about, or requests to change the state of, the Drill clutser
+ */
+
+ public WebUiPageTree(Dispatcher dispatcher) {
+ super(dispatcher);
+
+ // Markup engine
+ register(FreemarkerMvcFeature.class);
+
+ // Web UI Pages
+ register(RootPage.class);
+ register(RedirectPage.class);
+ register(ConfigPage.class);
+ register(DrillbitsPage.class);
+ register(CancelDrillbitPage.class);
+ register(HistoryPage.class);
+ register(ManagePage.class);
+ register(ResizePage.class);
+ register(StopPage.class);
+
+ // Authorization
+ // See: https://jersey.java.net/documentation/latest/security.html
+
+ if (AMSecurityManagerImpl.isEnabled()) {
+ register(LogInLogOutPages.class);
+ register(AuthDynamicFeature.class);
+ register(RolesAllowedDynamicFeature.class);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/package-info.java
----------------------------------------------------------------------
diff --git a/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/package-info.java b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/package-info.java
new file mode 100644
index 0000000..13f1bd8
--- /dev/null
+++ b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/package-info.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ *
+ */
+package org.apache.drill.yarn.appMaster.http;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/package-info.java
----------------------------------------------------------------------
diff --git a/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/package-info.java b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/package-info.java
new file mode 100644
index 0000000..0ff835d
--- /dev/null
+++ b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/package-info.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Implements the Drill Application Master for YARN.
+ * <p>
+ * Note that AM implementation classes use org.apache.commons.logging
+ * to be consistent with the logging used within YARN itself. However,
+ * the AM uses Drill's class path which uses logback logging. To enable
+ * logging, modify
+ * <code>$DRILL_HOME/conf/logback.xml</code> and add a section something
+ * like this:
+ * <pre><code>
+ * <logger name="org.apache.drill.yarn" additivity="false">
+ * <level value="trace" />
+ * <appender-ref ref="STDOUT" />
+ * </logger>
+ * </code></pre>
+ */
+
+package org.apache.drill.yarn.appMaster;
\ No newline at end of file
|