Author: shv Date: Sun Nov 6 02:31:49 2011 New Revision: 1198122 URL: http://svn.apache.org/viewvc?rev=1198122&view=rev Log: MAPREDUCE-1118. Enhance the JobTracker web-ui to ensure tabular columns are sortable, also added a /scheduler servlet to CapacityScheduler for enhanced UI for queue information. Contributed by Krishna Ramachandran and Milind Bhandarkar. Added: hadoop/common/branches/branch-0.22/mapreduce/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacitySchedulerServlet.java (with props) hadoop/common/branches/branch-0.22/mapreduce/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacitySchedulerServlet.java (with props) hadoop/common/branches/branch-0.22/mapreduce/src/webapps/static/sorttable.js Modified: hadoop/common/branches/branch-0.22/mapreduce/CHANGES.txt hadoop/common/branches/branch-0.22/mapreduce/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapred/JSPUtil.java Modified: hadoop/common/branches/branch-0.22/mapreduce/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/CHANGES.txt?rev=1198122&r1=1198121&r2=1198122&view=diff ============================================================================== --- hadoop/common/branches/branch-0.22/mapreduce/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.22/mapreduce/CHANGES.txt Sun Nov 6 02:31:49 2011 @@ -631,6 +631,11 @@ Release 0.22.0 - Unreleased MAPREDUCE-3139. SlivePartitioner generates negative partitions. (jghoman) + MAPREDUCE-1118. Enhance the JobTracker web-ui to ensure tabular columns + are sortable, also added a /scheduler servlet to CapacityScheduler for + enhanced UI for queue information. + (Krishna Ramachandran and Milind Bhandarkar via shv) + Release 0.21.1 - Unreleased NEW FEATURES Added: hadoop/common/branches/branch-0.22/mapreduce/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacitySchedulerServlet.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacitySchedulerServlet.java?rev=1198122&view=auto ============================================================================== --- hadoop/common/branches/branch-0.22/mapreduce/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacitySchedulerServlet.java (added) +++ hadoop/common/branches/branch-0.22/mapreduce/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacitySchedulerServlet.java Sun Nov 6 02:31:49 2011 @@ -0,0 +1,152 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapred; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintWriter; +import java.util.HashMap; +import java.util.Map; +import javax.servlet.ServletContext; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.hadoop.mapreduce.QueueInfo; +import org.apache.hadoop.util.StringUtils; + +/** + * Servlet for displaying fair scheduler information, installed at [job tracker + * URL]/scheduler when the {@link FairScheduler} is in use. + * + * The main features are viewing each job's task count and fair share, ability + * to change job priorities and pools from the UI, and ability to switch the + * scheduler to FIFO mode without restarting the JobTracker if this is required + * for any reason. + * + * + */ +public class CapacitySchedulerServlet extends HttpServlet { + private static final long serialVersionUID = 9104070533067306659L; + + private transient CapacityTaskScheduler scheduler; + private transient JobTracker jobTracker; + + @Override + public void init() throws ServletException { + super.init(); + ServletContext servletContext = this.getServletContext(); + this.scheduler = (CapacityTaskScheduler) servletContext + .getAttribute("scheduler"); + this.jobTracker = (JobTracker) scheduler.taskTrackerManager; + } + + @Override + protected void doPost(HttpServletRequest req, HttpServletResponse resp) + throws ServletException, IOException { + doGet(req, resp); // Same handler for both GET and POST + } + + @Override + public void doGet(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + // Print out the normal response + response.setContentType("text/html"); + + // Because the client may read arbitrarily slow, and we hold locks while + // the servlet output, we want to write to our own buffer which we know + // won't block. + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter out = new PrintWriter(baos); + String hostname = StringUtils.simpleHostname(jobTracker + .getJobTrackerMachine()); + out.print(""); + out.printf("%s Job Scheduler Admininstration\n", hostname); + out.print("\n"); + out.print(" \n"); + out.print("\n"); + out.printf("

%s " + + "Job Scheduler Administration

\n", hostname); + showQueues(out); + out.print("\n"); + out.close(); + + // Flush our buffer to the real servlet output + OutputStream servletOut = null; + try { + servletOut = response.getOutputStream(); + baos.writeTo(servletOut); + } finally { + if (servletOut != null) { + servletOut.close(); + } + } + } + + /** + * Print a view of pools to the given output writer. + */ + + private void showQueues(PrintWriter out) throws IOException { + synchronized (scheduler) { + out.print("

Queues

\n"); + out.print(" \n"); + out.print("" + + "" + "" + + "" + + "" + + "\n"); + JobQueuesManager queuesManager = scheduler.getJobQueuesManager(); + for (QueueSchedulingContext qsc : scheduler.getQueueInfoMap().values()) { + String name = qsc.getQueueName(); + out.print("\n"); + out.printf( + "\n", + name, name); + out.printf("\n", queuesManager.getJobQueue(name) + .getRunningJobs().size()); + out.printf("\n", qsc.getNumOfWaitingJobs()); + out.printf("\n", mapCapacity); + out.printf("\n", mapSlotsOccupied, + occupiedSlotsAsPercent); + out.printf("\n", qsc.getMapTSC().getNumRunningTasks()); + int reduceCapacity = qsc.getReduceCapacity(); + int reduceSlotsOccupied = qsc.getReduceTSC().getNumSlotsOccupied(); + float redOccupiedSlotsAsPercent = reduceCapacity != 0 ? ((float) reduceSlotsOccupied * 100 / reduceCapacity) + : 0; + out.printf("\n", reduceCapacity); + out.printf("\n", reduceSlotsOccupied, + redOccupiedSlotsAsPercent); + out.printf("\n", qsc.getReduceTSC().getNumRunningTasks()); + } + out.print("
QueueRunning JobsPending JobsCapacity PercentageMap Task CapacityMap Task Used CapacityRunning MapsReduce Task CapacityReduce Task Used CapacityRunning Reduces
%s%s%s\n", qsc.getCapacityPercent()); + + int mapCapacity = qsc.getMapCapacity(); + int mapSlotsOccupied = qsc.getMapTSC().getNumSlotsOccupied(); + float occupiedSlotsAsPercent = mapCapacity != 0 ? ((float) mapSlotsOccupied * 100 / mapCapacity) + : 0; + out.printf("%s%s (%.1f%% of Capacity)%s%s%s (%.1f%% of Capacity)%s
\n"); + } + } +} Propchange: hadoop/common/branches/branch-0.22/mapreduce/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacitySchedulerServlet.java ------------------------------------------------------------------------------ svn:mime-type = text/plain Modified: hadoop/common/branches/branch-0.22/mapreduce/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java?rev=1198122&r1=1198121&r2=1198122&view=diff ============================================================================== --- hadoop/common/branches/branch-0.22/mapreduce/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java (original) +++ hadoop/common/branches/branch-0.22/mapreduce/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java Sun Nov 6 02:31:49 2011 @@ -28,6 +28,7 @@ import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.http.HttpServer; import org.apache.hadoop.mapred.AbstractQueue.AbstractQueueComparator; import org.apache.hadoop.mapred.JobTracker.IllegalStateException; import org.apache.hadoop.mapreduce.TaskType; @@ -145,7 +146,7 @@ class CapacityTaskScheduler extends Task * There may be slight variations later, in which case we can make this * an abstract base class and have derived classes for Map and Reduce. */ - private static abstract class TaskSchedulingMgr { + static abstract class TaskSchedulingMgr { /** our TaskScheduler object */ protected CapacityTaskScheduler scheduler; @@ -758,6 +759,13 @@ class CapacityTaskScheduler extends Task public synchronized void start() throws IOException { if (started) return; super.start(); + if (taskTrackerManager instanceof JobTracker) { + JobTracker jobTracker = (JobTracker) taskTrackerManager; + HttpServer infoServer = jobTracker.infoServer; + infoServer.setAttribute("scheduler", this); + infoServer.addServlet("scheduler", "/scheduler", + CapacitySchedulerServlet.class); + } // Initialize MemoryMatcher MemoryMatcher.initializeMemoryRelatedConf(conf); @@ -1047,6 +1055,20 @@ class CapacityTaskScheduler extends Task return jobCollection; } + /** + * @return the queueInfoMap + */ + Map getQueueInfoMap() { + return queueInfoMap; + } + + /** + * @return the jobQueuesManager + */ + JobQueuesManager getJobQueuesManager() { + return jobQueuesManager; + } + synchronized JobInitializationPoller getInitializationPoller() { return initializationPoller; } @@ -1078,6 +1100,19 @@ class CapacityTaskScheduler extends Task } } + /** + * @return the mapScheduler + */ + TaskSchedulingMgr getMapScheduler() { + return mapScheduler; + } + + /** + * @return the reduceScheduler + */ + TaskSchedulingMgr getReduceScheduler() { + return reduceScheduler; + } /** * Use for testing purposes. Added: hadoop/common/branches/branch-0.22/mapreduce/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacitySchedulerServlet.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacitySchedulerServlet.java?rev=1198122&view=auto ============================================================================== --- hadoop/common/branches/branch-0.22/mapreduce/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacitySchedulerServlet.java (added) +++ hadoop/common/branches/branch-0.22/mapreduce/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacitySchedulerServlet.java Sun Nov 6 02:31:49 2011 @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapred; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Properties; +import java.net.URL; +import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; +import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig; + +public class TestCapacitySchedulerServlet extends + ClusterWithCapacityScheduler { + + /** + * Test case checks CapacitySchedulerServlet. Check if queues are + * initialized {@link CapacityTaskScheduler} + * + * @throws IOException + */ + public void testCapacitySchedulerServlet() throws IOException { + Properties schedulerProps = new Properties(); + String[] queues = new String[] { "Q1", "Q2" }; + for (String q : queues) { + schedulerProps.put(CapacitySchedulerConf + .toFullPropertyName(q, "capacity"), "50"); + schedulerProps.put(CapacitySchedulerConf.toFullPropertyName(q, + "minimum-user-limit-percent"), "100"); + } + Properties clusterProps = new Properties(); + clusterProps.put(TTConfig.TT_MAP_SLOTS, String.valueOf(2)); + clusterProps.put(TTConfig.TT_REDUCE_SLOTS, String.valueOf(2)); + clusterProps.put("mapred.queue.names", queues[0] + "," + queues[1]); + clusterProps.put(JTConfig.JT_PERSIST_JOBSTATUS, "false"); + startCluster(2, clusterProps, schedulerProps); + + JobTracker jt = getJobTracker(); + int port = jt.getInfoPort(); + String host = jt.getJobTrackerMachine(); + URL url = new URL("http://" + host + ":" + port + "/scheduler"); + String queueData = readOutput(url); + assertTrue(queueData.contains("Q1")); + assertTrue(queueData.contains("Q2")); + assertTrue(queueData.contains("50.0%")); + } + + private String readOutput(URL url) throws IOException { + StringBuilder out = new StringBuilder(); + InputStream in = url.openConnection().getInputStream(); + byte[] buffer = new byte[64 * 1024]; + int len = in.read(buffer); + while (len > 0) { + out.append(new String(buffer, 0, len)); + len = in.read(buffer); + } + return out.toString(); + } +} Propchange: hadoop/common/branches/branch-0.22/mapreduce/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacitySchedulerServlet.java ------------------------------------------------------------------------------ svn:mime-type = text/plain Modified: hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapred/JSPUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapred/JSPUtil.java?rev=1198122&r1=1198121&r2=1198122&view=diff ============================================================================== --- hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapred/JSPUtil.java (original) +++ hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapred/JSPUtil.java Sun Nov 6 02:31:49 2011 @@ -381,7 +381,7 @@ class JSPUtil { throws IOException { StringBuilder sb = new StringBuilder(); - sb.append("\n"); + sb.append("
\n"); Iterator iterator = tracker.retireJobs.getAll().descendingIterator(); Added: hadoop/common/branches/branch-0.22/mapreduce/src/webapps/static/sorttable.js URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/src/webapps/static/sorttable.js?rev=1198122&view=auto ============================================================================== --- hadoop/common/branches/branch-0.22/mapreduce/src/webapps/static/sorttable.js (added) +++ hadoop/common/branches/branch-0.22/mapreduce/src/webapps/static/sorttable.js Sun Nov 6 02:31:49 2011 @@ -0,0 +1,493 @@ +/* + SortTable + version 2 + 7th April 2007 + Stuart Langridge, http://www.kryogenix.org/code/browser/sorttable/ + + Instructions: + Download this file + Add to your HTML + Add class="sortable" to any table you'd like to make sortable + Click on the headers to sort + + Thanks to many, many people for contributions and suggestions. + Licenced as X11: http://www.kryogenix.org/code/browser/licence.html + This basically means: do what you want with it. +*/ + + +var stIsIE = /*@cc_on!@*/false; + +sorttable = { + init: function() { + // quit if this function has already been called + if (arguments.callee.done) return; + // flag this function so we don't do the same thing twice + arguments.callee.done = true; + // kill the timer + if (_timer) clearInterval(_timer); + + if (!document.createElement || !document.getElementsByTagName) return; + + sorttable.DATE_RE = /^(\d\d?)[\/\.-](\d\d?)[\/\.-]((\d\d)?\d\d)$/; + + forEach(document.getElementsByTagName('table'), function(table) { + if (table.className.search(/\bsortable\b/) != -1) { + sorttable.makeSortable(table); + } + }); + + }, + + makeSortable: function(table) { + if (table.getElementsByTagName('thead').length == 0) { + // table doesn't have a tHead. Since it should have, create one and + // put the first table row in it. + the = document.createElement('thead'); + the.appendChild(table.rows[0]); + table.insertBefore(the,table.firstChild); + } + // Safari doesn't support table.tHead, sigh + if (table.tHead == null) table.tHead = table.getElementsByTagName('thead')[0]; + + if (table.tHead.rows.length != 1) return; // can't cope with two header rows + + // Sorttable v1 put rows with a class of "sortbottom" at the bottom (as + // "total" rows, for example). This is B&R, since what you're supposed + // to do is put them in a tfoot. So, if there are sortbottom rows, + // for backwards compatibility, move them to tfoot (creating it if needed). + sortbottomrows = []; + for (var i=0; i5' : ' ▴'; + this.appendChild(sortrevind); + return; + } + if (this.className.search(/\bsorttable_sorted_reverse\b/) != -1) { + // if we're already sorted by this column in reverse, just + // re-reverse the table, which is quicker + sorttable.reverse(this.sorttable_tbody); + this.className = this.className.replace('sorttable_sorted_reverse', + 'sorttable_sorted'); + this.removeChild(document.getElementById('sorttable_sortrevind')); + sortfwdind = document.createElement('span'); + sortfwdind.id = "sorttable_sortfwdind"; + sortfwdind.innerHTML = stIsIE ? ' 6' : ' ▾'; + this.appendChild(sortfwdind); + return; + } + + // remove sorttable_sorted classes + theadrow = this.parentNode; + forEach(theadrow.childNodes, function(cell) { + if (cell.nodeType == 1) { // an element + cell.className = cell.className.replace('sorttable_sorted_reverse',''); + cell.className = cell.className.replace('sorttable_sorted',''); + } + }); + sortfwdind = document.getElementById('sorttable_sortfwdind'); + if (sortfwdind) { sortfwdind.parentNode.removeChild(sortfwdind); } + sortrevind = document.getElementById('sorttable_sortrevind'); + if (sortrevind) { sortrevind.parentNode.removeChild(sortrevind); } + + this.className += ' sorttable_sorted'; + sortfwdind = document.createElement('span'); + sortfwdind.id = "sorttable_sortfwdind"; + sortfwdind.innerHTML = stIsIE ? ' 6' : ' ▾'; + this.appendChild(sortfwdind); + + // build an array to sort. This is a Schwartzian transform thing, + // i.e., we "decorate" each row with the actual sort key, + // sort based on the sort keys, and then put the rows back in order + // which is a lot faster because you only do getInnerText once per row + row_array = []; + col = this.sorttable_columnindex; + rows = this.sorttable_tbody.rows; + for (var j=0; j 12) { + // definitely dd/mm + return sorttable.sort_ddmm; + } else if (second > 12) { + return sorttable.sort_mmdd; + } else { + // looks like a date, but we can't tell which, so assume + // that it's dd/mm (English imperialism!) and keep looking + sortfn = sorttable.sort_ddmm; + } + } + } + } + return sortfn; + }, + + getInnerText: function(node) { + // gets the text we want to use for sorting for a cell. + // strips leading and trailing whitespace. + // this is *not* a generic getInnerText function; it's special to sorttable. + // for example, you can override the cell text with a customkey attribute. + // it also gets .value for fields. + + hasInputs = (typeof node.getElementsByTagName == 'function') && + node.getElementsByTagName('input').length; + + if (node.getAttribute("sorttable_customkey") != null) { + return node.getAttribute("sorttable_customkey"); + } + else if (typeof node.textContent != 'undefined' && !hasInputs) { + return node.textContent.replace(/^\s+|\s+$/g, ''); + } + else if (typeof node.innerText != 'undefined' && !hasInputs) { + return node.innerText.replace(/^\s+|\s+$/g, ''); + } + else if (typeof node.text != 'undefined' && !hasInputs) { + return node.text.replace(/^\s+|\s+$/g, ''); + } + else { + switch (node.nodeType) { + case 3: + if (node.nodeName.toLowerCase() == 'input') { + return node.value.replace(/^\s+|\s+$/g, ''); + } + case 4: + return node.nodeValue.replace(/^\s+|\s+$/g, ''); + break; + case 1: + case 11: + var innerText = ''; + for (var i = 0; i < node.childNodes.length; i++) { + innerText += sorttable.getInnerText(node.childNodes[i]); + } + return innerText.replace(/^\s+|\s+$/g, ''); + break; + default: + return ''; + } + } + }, + + reverse: function(tbody) { + // reverse the rows in a tbody + newrows = []; + for (var i=0; i=0; i--) { + tbody.appendChild(newrows[i]); + } + delete newrows; + }, + + /* sort functions + each sort function takes two parameters, a and b + you are comparing a[0] and b[0] */ + sort_numeric: function(a,b) { + aa = parseFloat(a[0].replace(/[^0-9.-]/g,'')); + if (isNaN(aa)) aa = 0; + bb = parseFloat(b[0].replace(/[^0-9.-]/g,'')); + if (isNaN(bb)) bb = 0; + return aa-bb; + }, + sort_alpha: function(a,b) { + if (a[0]==b[0]) return 0; + if (a[0] 0 ) { + var q = list[i]; list[i] = list[i+1]; list[i+1] = q; + swap = true; + } + } // for + t--; + + if (!swap) break; + + for(var i = t; i > b; --i) { + if ( comp_func(list[i], list[i-1]) < 0 ) { + var q = list[i]; list[i] = list[i-1]; list[i-1] = q; + swap = true; + } + } // for + b++; + + } // while(swap) + } +} + +/* ****************************************************************** + Supporting functions: bundled here to avoid depending on a library + ****************************************************************** */ + +// Dean Edwards/Matthias Miller/John Resig + +/* for Mozilla/Opera9 */ +if (document.addEventListener) { + document.addEventListener("DOMContentLoaded", sorttable.init, false); +} + +/* for Internet Explorer */ +/*@cc_on @*/ +/*@if (@_win32) + document.write("