From mapreduce-commits-return-5415-apmail-hadoop-mapreduce-commits-archive=hadoop.apache.org@hadoop.apache.org Fri Feb 15 10:19:00 2013 Return-Path: X-Original-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id F2FC8EC1D for ; Fri, 15 Feb 2013 10:18:59 +0000 (UTC) Received: (qmail 9649 invoked by uid 500); 15 Feb 2013 10:18:59 -0000 Delivered-To: apmail-hadoop-mapreduce-commits-archive@hadoop.apache.org Received: (qmail 9330 invoked by uid 500); 15 Feb 2013 10:18:51 -0000 Mailing-List: contact mapreduce-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: mapreduce-dev@hadoop.apache.org Delivered-To: mailing list mapreduce-commits@hadoop.apache.org Received: (qmail 9263 invoked by uid 99); 15 Feb 2013 10:18:48 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 15 Feb 2013 10:18:48 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 15 Feb 2013 10:18:45 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 82C6C2388B75; Fri, 15 Feb 2013 10:18:08 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1446495 - in /hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project: ./ conf/ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/ hadoop-mapreduce-client/hadoop-mapreduc... Date: Fri, 15 Feb 2013 10:18:07 -0000 To: mapreduce-commits@hadoop.apache.org From: suresh@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20130215101808.82C6C2388B75@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: suresh Date: Fri Feb 15 10:18:03 2013 New Revision: 1446495 URL: http://svn.apache.org/r1446495 Log: Merged trunk to branch-trunk-win Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/ (props changed) hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/CHANGES.txt (contents, props changed) hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/conf/ (props changed) hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/recover/RecoveryService.java hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskImpl.java hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalClientProtocolProvider.java hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml (props changed) hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTaskPage.java hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTasksBlock.java hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTasksPage.java hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestLineRecordReader.java hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestChainMapReduce.java hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestClientProtocolProviderImpls.java Propchange: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-mapreduce-project:r1443752-1446489 Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/CHANGES.txt?rev=1446495&r1=1446494&r2=1446495&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/CHANGES.txt Fri Feb 15 10:18:03 2013 @@ -167,6 +167,13 @@ Release 2.0.4-beta - UNRELEASED MAPREDUCE-4671. AM does not tell the RM about container requests which are no longer needed. (Bikas Saha via sseth) + MAPREDUCE-4994. -jt generic command line option does not work. (sandyr via tucu) + + MAPREDUCE-5000. Fixes getCounters when speculating by fixing the selection + of the best attempt for a task. (Jason Lowe via sseth) + + MAPREDUCE-4994. Addendum fixing testcases failures. (sandyr via tucu) + Release 2.0.3-alpha - 2013-02-06 INCOMPATIBLE CHANGES @@ -694,6 +701,9 @@ Release 0.23.7 - UNRELEASED MAPREDUCE-4905. test org.apache.hadoop.mapred.pipes (Aleksey Gorshkov via bobby) + MAPREDUCE-4989. JSONify DataTables input data for Attempts page (Ravi + Prakash via jlowe) + OPTIMIZATIONS MAPREDUCE-4946. Fix a performance problem for large jobs by reducing the @@ -707,6 +717,9 @@ Release 0.23.7 - UNRELEASED MAPREDUCE-4458. Warn if java.library.path is used for AM or Task (Robert Parker via jeagles) + MAPREDUCE-4992. AM hangs in RecoveryService when recovering tasks with + speculative attempts (Robert Parker via jlowe) + Release 0.23.6 - UNRELEASED INCOMPATIBLE CHANGES Propchange: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/CHANGES.txt ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt:r1443752-1446489 Propchange: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/conf/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-mapreduce-project/conf:r1443752-1446489 Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java?rev=1446495&r1=1446494&r2=1446495&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java Fri Feb 15 10:18:03 2013 @@ -539,6 +539,10 @@ public abstract class TaskImpl implement //select the nextAttemptNumber with best progress // always called inside the Read Lock private TaskAttempt selectBestAttempt() { + if (successfulAttempt != null) { + return attempts.get(successfulAttempt); + } + float progress = 0f; TaskAttempt result = null; for (TaskAttempt at : attempts.values()) { Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/recover/RecoveryService.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/recover/RecoveryService.java?rev=1446495&r1=1446494&r2=1446495&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/recover/RecoveryService.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/recover/RecoveryService.java Fri Feb 15 10:18:03 2013 @@ -21,9 +21,12 @@ package org.apache.hadoop.mapreduce.v2.a import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; +import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -35,6 +38,7 @@ import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.OutputCommitter; import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.apache.hadoop.mapreduce.TaskAttemptID; import org.apache.hadoop.mapreduce.TaskType; import org.apache.hadoop.mapreduce.TypeConverter; import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser; @@ -108,7 +112,7 @@ public class RecoveryService extends Com private JobInfo jobInfo = null; private final Map completedTasks = new HashMap(); - + private final List pendingTaskScheduleEvents = new ArrayList(); @@ -193,6 +197,14 @@ public class RecoveryService extends Com .getAllTasks(); for (TaskInfo taskInfo : taskInfos.values()) { if (TaskState.SUCCEEDED.toString().equals(taskInfo.getTaskStatus())) { + Iterator> taskAttemptIterator = + taskInfo.getAllTaskAttempts().entrySet().iterator(); + while (taskAttemptIterator.hasNext()) { + Map.Entry currentEntry = taskAttemptIterator.next(); + if (!jobInfo.getAllCompletedTaskAttempts().containsKey(currentEntry.getKey())) { + taskAttemptIterator.remove(); + } + } completedTasks .put(TypeConverter.toYarn(taskInfo.getTaskId()), taskInfo); LOG.info("Read from history task " @@ -215,6 +227,7 @@ public class RecoveryService extends Com JobHistoryUtils.getConfiguredHistoryStagingDirPrefix(conf, jobId); Path histDirPath = FileContext.getFileContext(conf).makeQualified(new Path(jobhistoryDir)); + LOG.info("Trying file " + histDirPath.toString()); FileContext fc = FileContext.getFileContext(histDirPath.toUri(), conf); // read the previous history file historyFile = Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java?rev=1446495&r1=1446494&r2=1446495&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java Fri Feb 15 10:18:03 2013 @@ -27,18 +27,11 @@ import static org.apache.hadoop.yarn.web import java.util.Collection; +import org.apache.commons.lang.StringEscapeUtils; import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt; import org.apache.hadoop.mapreduce.v2.app.webapp.dao.TaskAttemptInfo; -import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.util.Times; import org.apache.hadoop.yarn.webapp.SubView; -import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; -import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE; -import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY; -import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TD; -import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TR; import org.apache.hadoop.yarn.webapp.view.HtmlBlock; import com.google.inject.Inject; @@ -60,7 +53,7 @@ public class TaskPage extends AppView { h2($(TITLE)); return; } - TBODY> tbody = html. + html. table("#attempts"). thead(). tr(). @@ -72,49 +65,46 @@ public class TaskPage extends AppView { th(".tsh", "Started"). th(".tsh", "Finished"). th(".tsh", "Elapsed"). - th(".note", "Note")._()._(). - tbody(); + th(".note", "Note")._()._(); + // Write all the data into a JavaScript array of arrays for JQuery + // DataTables to display + StringBuilder attemptsTableData = new StringBuilder("[\n"); + for (TaskAttempt attempt : getTaskAttempts()) { TaskAttemptInfo ta = new TaskAttemptInfo(attempt, true); - String taid = ta.getId(); String progress = percent(ta.getProgress() / 100); - ContainerId containerId = ta.getAssignedContainerId(); String nodeHttpAddr = ta.getNode(); - long startTime = ta.getStartTime(); - long finishTime = ta.getFinishTime(); - long elapsed = ta.getElapsedTime(); String diag = ta.getNote() == null ? "" : ta.getNote(); - TR>> row = tbody.tr(); - TD>>> nodeTd = row. - td(".id", taid). - td(".progress", progress). - td(".state", ta.getState()).td(); - if (nodeHttpAddr == null) { - nodeTd._("N/A"); - } else { - nodeTd. - a(".nodelink", url(HttpConfig.getSchemePrefix(), - nodeHttpAddr), nodeHttpAddr); - } - nodeTd._(); - if (containerId != null) { - String containerIdStr = ta.getAssignedContainerIdStr(); - row.td(). - a(".logslink", url(HttpConfig.getSchemePrefix(), - nodeHttpAddr, "node", "containerlogs", - containerIdStr, app.getJob().getUserName()), "logs")._(); - } else { - row.td()._("N/A")._(); - } - - row. - td(".ts", Times.format(startTime)). - td(".ts", Times.format(finishTime)). - td(".dt", StringUtils.formatTime(elapsed)). - td(".note", diag)._(); + attemptsTableData.append("[\"") + .append(ta.getId()).append("\",\"") + .append(progress).append("\",\"") + .append(ta.getState().toString()).append("\",\"") + + .append(nodeHttpAddr == null ? "N/A" : + "" + + nodeHttpAddr + "") + .append("\",\"") + + .append(ta.getAssignedContainerId() == null ? "N/A" : + "logs") + .append("\",\"") + + .append(ta.getStartTime()).append("\",\"") + .append(ta.getFinishTime()).append("\",\"") + .append(ta.getElapsedTime()).append("\",\"") + .append(StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml( + diag))).append("\"],\n"); + } + //Remove the last comma and close off the array of arrays + if(attemptsTableData.charAt(attemptsTableData.length() - 2) == ',') { + attemptsTableData.delete(attemptsTableData.length()-2, attemptsTableData.length()-1); } - tbody._()._(); + attemptsTableData.append("]"); + html.script().$type("text/javascript"). + _("var attemptsTableData=" + attemptsTableData)._(); } protected boolean isValidRequest() { @@ -140,9 +130,24 @@ public class TaskPage extends AppView { } private String attemptsTableInit() { - return tableInit(). - // Sort by id upon page load - append(", aaSorting: [[0, 'asc']]"). - append("}").toString(); + return tableInit() + .append(", 'aaData': attemptsTableData") + .append(", bDeferRender: true") + .append(", bProcessing: true") + .append("\n,aoColumnDefs:[\n") + + //logs column should not filterable (it includes container ID which may pollute searches) + .append("\n{'aTargets': [ 4 ]") + .append(", 'bSearchable': false }") + + .append("\n, {'sType':'numeric', 'aTargets': [ 5, 6") + .append(" ], 'mRender': renderHadoopDate }") + + .append("\n, {'sType':'numeric', 'aTargets': [ 7") + .append(" ], 'mRender': renderHadoopElapsedTime }]") + + // Sort by id upon page load + .append("\n, aaSorting: [[0, 'asc']]") + .append("}").toString(); } } Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java?rev=1446495&r1=1446494&r2=1446495&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java Fri Feb 15 10:18:03 2013 @@ -50,11 +50,15 @@ import org.apache.hadoop.mapreduce.v2.ap import org.apache.hadoop.mapreduce.v2.app.job.Job; import org.apache.hadoop.mapreduce.v2.app.job.Task; import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt; +import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptContainerLaunchedEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType; +import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEvent; +import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEventType; import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncher; import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncherEvent; import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.event.EventHandler; import org.junit.Test; @@ -734,12 +738,173 @@ public class TestRecovery { app.verifyCompleted(); validateOutput(); } - + + /** + * AM with 2 maps and 1 reduce. For 1st map, one attempt fails, one attempt + * completely disappears because of failed launch, one attempt gets killed and + * one attempt succeeds. AM crashes after the first tasks finishes and + * recovers completely and succeeds in the second generation. + * + * @throws Exception + */ + @Test + public void testSpeculative() throws Exception { + + int runCount = 0; + long am1StartTimeEst = System.currentTimeMillis(); + MRApp app = new MRAppWithHistory(2, 1, false, this.getClass().getName(), true, ++runCount); + Configuration conf = new Configuration(); + conf.setBoolean("mapred.mapper.new-api", true); + conf.setBoolean("mapred.reducer.new-api", true); + conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false); + conf.set(FileOutputFormat.OUTDIR, outputDir.toString()); + Job job = app.submit(conf); + app.waitForState(job, JobState.RUNNING); + long jobStartTime = job.getReport().getStartTime(); + //all maps would be running + Assert.assertEquals("No of tasks not correct", + 3, job.getTasks().size()); + + Iterator it = job.getTasks().values().iterator(); + Task mapTask1 = it.next(); + Task mapTask2 = it.next(); + Task reduceTask = it.next(); + + // all maps must be running + app.waitForState(mapTask1, TaskState.RUNNING); + app.waitForState(mapTask2, TaskState.RUNNING); + + // Launch a Speculative Task for the first Task + app.getContext().getEventHandler().handle( + new TaskEvent(mapTask1.getID(), TaskEventType.T_ADD_SPEC_ATTEMPT)); + int timeOut = 0; + while (mapTask1.getAttempts().size() != 2 && timeOut++ < 10) { + Thread.sleep(1000); + LOG.info("Waiting for next attempt to start"); + } + Iterator t1it = mapTask1.getAttempts().values().iterator(); + TaskAttempt task1Attempt1 = t1it.next(); + TaskAttempt task1Attempt2 = t1it.next(); + TaskAttempt task2Attempt = mapTask2.getAttempts().values().iterator().next(); + + ContainerId t1a2contId = task1Attempt2.getAssignedContainerID(); + + LOG.info(t1a2contId.toString()); + LOG.info(task1Attempt1.getID().toString()); + LOG.info(task1Attempt2.getID().toString()); + + // Launch container for speculative attempt + app.getContext().getEventHandler().handle( + new TaskAttemptContainerLaunchedEvent(task1Attempt2.getID(), runCount)); + + //before sending the TA_DONE, event make sure attempt has come to + //RUNNING state + app.waitForState(task1Attempt1, TaskAttemptState.RUNNING); + app.waitForState(task1Attempt2, TaskAttemptState.RUNNING); + app.waitForState(task2Attempt, TaskAttemptState.RUNNING); + + // reduces must be in NEW state + Assert.assertEquals("Reduce Task state not correct", + TaskState.RUNNING, reduceTask.getReport().getTaskState()); + + //send the done signal to the map 1 attempt 1 + app.getContext().getEventHandler().handle( + new TaskAttemptEvent( + task1Attempt1.getID(), + TaskAttemptEventType.TA_DONE)); + + app.waitForState(task1Attempt1, TaskAttemptState.SUCCEEDED); + + //wait for first map task to complete + app.waitForState(mapTask1, TaskState.SUCCEEDED); + long task1StartTime = mapTask1.getReport().getStartTime(); + long task1FinishTime = mapTask1.getReport().getFinishTime(); + + //stop the app + app.stop(); + + //rerun + //in rerun the 1st map will be recovered from previous run + long am2StartTimeEst = System.currentTimeMillis(); + app = new MRAppWithHistory(2, 1, false, this.getClass().getName(), false, ++runCount); + conf = new Configuration(); + conf.setBoolean(MRJobConfig.MR_AM_JOB_RECOVERY_ENABLE, true); + conf.setBoolean("mapred.mapper.new-api", true); + conf.setBoolean("mapred.reducer.new-api", true); + conf.set(FileOutputFormat.OUTDIR, outputDir.toString()); + conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false); + job = app.submit(conf); + app.waitForState(job, JobState.RUNNING); + //all maps would be running + Assert.assertEquals("No of tasks not correct", + 3, job.getTasks().size()); + it = job.getTasks().values().iterator(); + mapTask1 = it.next(); + mapTask2 = it.next(); + reduceTask = it.next(); + + // first map will be recovered, no need to send done + app.waitForState(mapTask1, TaskState.SUCCEEDED); + + app.waitForState(mapTask2, TaskState.RUNNING); + + task2Attempt = mapTask2.getAttempts().values().iterator().next(); + //before sending the TA_DONE, event make sure attempt has come to + //RUNNING state + app.waitForState(task2Attempt, TaskAttemptState.RUNNING); + + //send the done signal to the 2nd map task + app.getContext().getEventHandler().handle( + new TaskAttemptEvent( + mapTask2.getAttempts().values().iterator().next().getID(), + TaskAttemptEventType.TA_DONE)); + + //wait to get it completed + app.waitForState(mapTask2, TaskState.SUCCEEDED); + + //wait for reduce to be running before sending done + app.waitForState(reduceTask, TaskState.RUNNING); + + //send the done signal to the reduce + app.getContext().getEventHandler().handle( + new TaskAttemptEvent( + reduceTask.getAttempts().values().iterator().next().getID(), + TaskAttemptEventType.TA_DONE)); + + app.waitForState(job, JobState.SUCCEEDED); + app.verifyCompleted(); + Assert.assertEquals("Job Start time not correct", + jobStartTime, job.getReport().getStartTime()); + Assert.assertEquals("Task Start time not correct", + task1StartTime, mapTask1.getReport().getStartTime()); + Assert.assertEquals("Task Finish time not correct", + task1FinishTime, mapTask1.getReport().getFinishTime()); + Assert.assertEquals(2, job.getAMInfos().size()); + int attemptNum = 1; + // Verify AMInfo + for (AMInfo amInfo : job.getAMInfos()) { + Assert.assertEquals(attemptNum++, amInfo.getAppAttemptId() + .getAttemptId()); + Assert.assertEquals(amInfo.getAppAttemptId(), amInfo.getContainerId() + .getApplicationAttemptId()); + Assert.assertEquals(MRApp.NM_HOST, amInfo.getNodeManagerHost()); + Assert.assertEquals(MRApp.NM_PORT, amInfo.getNodeManagerPort()); + Assert.assertEquals(MRApp.NM_HTTP_PORT, amInfo.getNodeManagerHttpPort()); + } + long am1StartTimeReal = job.getAMInfos().get(0).getStartTime(); + long am2StartTimeReal = job.getAMInfos().get(1).getStartTime(); + Assert.assertTrue(am1StartTimeReal >= am1StartTimeEst + && am1StartTimeReal <= am2StartTimeEst); + Assert.assertTrue(am2StartTimeReal >= am2StartTimeEst + && am2StartTimeReal <= System.currentTimeMillis()); + + } + private void writeBadOutput(TaskAttempt attempt, Configuration conf) throws Exception { TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, TypeConverter.fromYarn(attempt.getID())); - + TextOutputFormat theOutputFormat = new TextOutputFormat(); RecordWriter theRecordWriter = theOutputFormat .getRecordWriter(tContext); Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskImpl.java?rev=1446495&r1=1446494&r2=1446495&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskImpl.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskImpl.java Fri Feb 15 10:18:03 2013 @@ -35,6 +35,9 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.Task; import org.apache.hadoop.mapred.TaskUmbilicalProtocol; +import org.apache.hadoop.mapreduce.Counter; +import org.apache.hadoop.mapreduce.Counters; +import org.apache.hadoop.mapreduce.TaskCounter; import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskInfo; import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier; import org.apache.hadoop.mapreduce.split.JobSplit.TaskSplitMetaInfo; @@ -52,7 +55,6 @@ import org.apache.hadoop.mapreduce.v2.ap import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEventType; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskTAttemptEvent; -import org.apache.hadoop.mapreduce.v2.app.job.impl.TaskAttemptImpl; import org.apache.hadoop.mapreduce.v2.app.metrics.MRAppMetrics; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.token.Token; @@ -143,6 +145,7 @@ public class TestTaskImpl { private float progress = 0; private TaskAttemptState state = TaskAttemptState.NEW; private TaskType taskType; + private Counters attemptCounters = TaskAttemptImpl.EMPTY_COUNTERS; public MockTaskAttemptImpl(TaskId taskId, int id, EventHandler eventHandler, TaskAttemptListener taskAttemptListener, Path jobFile, int partition, @@ -178,7 +181,15 @@ public class TestTaskImpl { public TaskAttemptState getState() { return state; } - + + @Override + public Counters getCounters() { + return attemptCounters; + } + + public void setCounters(Counters counters) { + attemptCounters = counters; + } } private class MockTask extends Task { @@ -687,4 +698,49 @@ public class TestTaskImpl { TaskEventType.T_ATTEMPT_KILLED)); assertEquals(TaskState.FAILED, mockTask.getState()); } + + @Test + public void testCountersWithSpeculation() { + mockTask = new MockTaskImpl(jobId, partition, dispatcher.getEventHandler(), + remoteJobConfFile, conf, taskAttemptListener, jobToken, + credentials, clock, + completedTasksFromPreviousRun, startCount, + metrics, appContext, TaskType.MAP) { + @Override + protected int getMaxAttempts() { + return 1; + } + }; + TaskId taskId = getNewTaskID(); + scheduleTaskAttempt(taskId); + launchTaskAttempt(getLastAttempt().getAttemptId()); + updateLastAttemptState(TaskAttemptState.RUNNING); + MockTaskAttemptImpl baseAttempt = getLastAttempt(); + + // add a speculative attempt + mockTask.handle(new TaskTAttemptEvent(getLastAttempt().getAttemptId(), + TaskEventType.T_ADD_SPEC_ATTEMPT)); + launchTaskAttempt(getLastAttempt().getAttemptId()); + updateLastAttemptState(TaskAttemptState.RUNNING); + MockTaskAttemptImpl specAttempt = getLastAttempt(); + assertEquals(2, taskAttempts.size()); + + Counters specAttemptCounters = new Counters(); + Counter cpuCounter = specAttemptCounters.findCounter( + TaskCounter.CPU_MILLISECONDS); + cpuCounter.setValue(1000); + specAttempt.setCounters(specAttemptCounters); + + // have the spec attempt succeed but second attempt at 1.0 progress as well + commitTaskAttempt(specAttempt.getAttemptId()); + specAttempt.setProgress(1.0f); + specAttempt.setState(TaskAttemptState.SUCCEEDED); + mockTask.handle(new TaskTAttemptEvent(specAttempt.getAttemptId(), + TaskEventType.T_ATTEMPT_SUCCEEDED)); + assertEquals(TaskState.SUCCEEDED, mockTask.getState()); + baseAttempt.setProgress(1.0f); + + Counters taskCounters = mockTask.getCounters(); + assertEquals("wrong counters for task", specAttemptCounters, taskCounters); + } } Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml?rev=1446495&r1=1446494&r2=1446495&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml Fri Feb 15 10:18:03 2013 @@ -54,63 +54,30 @@ - maven-antrun-plugin + org.apache.hadoop + hadoop-maven-plugins - create-protobuf-generated-sources-directory - initialize - - - - - - - run - - - - - - - org.codehaus.mojo - exec-maven-plugin - - - generate-sources - generate-sources - - protoc - - -I../../../hadoop-common-project/hadoop-common/src/main/proto/ - -I../../../hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/ - -Isrc/main/proto/ - --java_out=target/generated-sources/proto - src/main/proto/mr_protos.proto - src/main/proto/mr_service_protos.proto - src/main/proto/MRClientProtocol.proto - - - - exec - - - - - - - org.codehaus.mojo - build-helper-maven-plugin - - - add-source + compile-protoc generate-sources - add-source + protoc - - target/generated-sources/proto - + + ${basedir}/../../../hadoop-common-project/hadoop-common/src/main/proto + ${basedir}/../../../hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto + ${basedir}/src/main/proto + + + ${basedir}/src/main/proto + + mr_protos.proto + mr_service_protos.proto + MRClientProtocol.proto + + + ${project.build.directory}/generated-sources/java Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalClientProtocolProvider.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalClientProtocolProvider.java?rev=1446495&r1=1446494&r2=1446495&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalClientProtocolProvider.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalClientProtocolProvider.java Fri Feb 15 10:18:03 2013 @@ -26,7 +26,6 @@ import org.apache.hadoop.conf.Configurat import org.apache.hadoop.mapreduce.MRConfig; import org.apache.hadoop.mapreduce.protocol.ClientProtocol; import org.apache.hadoop.mapreduce.protocol.ClientProtocolProvider; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; @InterfaceAudience.Private public class LocalClientProtocolProvider extends ClientProtocolProvider { @@ -38,16 +37,11 @@ public class LocalClientProtocolProvider if (!MRConfig.LOCAL_FRAMEWORK_NAME.equals(framework)) { return null; } - String tracker = conf.get(JTConfig.JT_IPC_ADDRESS, "local"); - if ("local".equals(tracker)) { + if (conf.get("mapreduce.job.maps") == null) { conf.setInt("mapreduce.job.maps", 1); - return new LocalJobRunner(conf); - } else { - - throw new IOException("Invalid \"" + JTConfig.JT_IPC_ADDRESS - + "\" configuration value for LocalJobRunner : \"" - + tracker + "\""); } + + return new LocalJobRunner(conf); } @Override Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml?rev=1446495&r1=1446494&r2=1446495&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml Fri Feb 15 10:18:03 2013 @@ -68,24 +68,6 @@ - org.codehaus.mojo - build-helper-maven-plugin - - - add-source - generate-sources - - add-source - - - - target/generated-sources/avro - - - - - - org.apache.maven.plugins maven-antrun-plugin Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java?rev=1446495&r1=1446494&r2=1446495&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java Fri Feb 15 10:18:03 2013 @@ -246,6 +246,7 @@ public class JobHistoryParser implements attemptInfo.state = StringInterner.weakIntern(event.getState()); attemptInfo.counters = event.getCounters(); attemptInfo.hostname = StringInterner.weakIntern(event.getHostname()); + info.completedTaskAttemptsMap.put(event.getAttemptId(), attemptInfo); } private void handleReduceAttemptFinishedEvent @@ -262,6 +263,7 @@ public class JobHistoryParser implements attemptInfo.hostname = StringInterner.weakIntern(event.getHostname()); attemptInfo.port = event.getPort(); attemptInfo.rackname = StringInterner.weakIntern(event.getRackName()); + info.completedTaskAttemptsMap.put(event.getAttemptId(), attemptInfo); } private void handleMapAttemptFinishedEvent(MapAttemptFinishedEvent event) { @@ -276,6 +278,7 @@ public class JobHistoryParser implements attemptInfo.hostname = StringInterner.weakIntern(event.getHostname()); attemptInfo.port = event.getPort(); attemptInfo.rackname = StringInterner.weakIntern(event.getRackName()); + info.completedTaskAttemptsMap.put(event.getAttemptId(), attemptInfo); } private void handleTaskAttemptFailedEvent( @@ -306,6 +309,7 @@ public class JobHistoryParser implements taskInfo.successfulAttemptId = null; } } + info.completedTaskAttemptsMap.put(event.getTaskAttemptId(), attemptInfo); } private void handleTaskAttemptStartedEvent(TaskAttemptStartedEvent event) { @@ -443,6 +447,7 @@ public class JobHistoryParser implements Map jobACLs; Map tasksMap; + Map completedTaskAttemptsMap; List amInfos; AMInfo latestAmInfo; boolean uberized; @@ -456,6 +461,7 @@ public class JobHistoryParser implements finishedMaps = finishedReduces = 0; username = jobname = jobConfPath = jobQueueName = ""; tasksMap = new HashMap(); + completedTaskAttemptsMap = new HashMap(); jobACLs = new HashMap(); priority = JobPriority.NORMAL; } @@ -530,6 +536,8 @@ public class JobHistoryParser implements public Counters getReduceCounters() { return reduceCounters; } /** @return the map of all tasks in this job */ public Map getAllTasks() { return tasksMap; } + /** @return the map of all completed task attempts in this job */ + public Map getAllCompletedTaskAttempts() { return completedTaskAttemptsMap; } /** @return the priority of this job */ public String getPriority() { return priority.toString(); } public Map getJobACLs() { return jobACLs; } Propchange: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml:r1443752-1446489 Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTaskPage.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTaskPage.java?rev=1446495&r1=1446494&r2=1446495&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTaskPage.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTaskPage.java Fri Feb 15 10:18:03 2013 @@ -29,6 +29,7 @@ import static org.apache.hadoop.yarn.web import java.util.Collection; +import org.apache.commons.lang.StringEscapeUtils; import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.mapreduce.v2.api.records.TaskId; import org.apache.hadoop.mapreduce.v2.api.records.TaskType; @@ -110,13 +111,17 @@ public class HsTaskPage extends HsView { th(".note", "Note"); TBODY> tbody = headRow._()._().tbody(); - for (TaskAttempt ta : getTaskAttempts()) { + // Write all the data into a JavaScript array of arrays for JQuery + // DataTables to display + StringBuilder attemptsTableData = new StringBuilder("[\n"); + + for (TaskAttempt ta : getTaskAttempts()) { String taid = MRApps.toString(ta.getID()); String nodeHttpAddr = ta.getNodeHttpAddress(); String containerIdString = ta.getAssignedContainerID().toString(); String nodeIdString = ta.getAssignedContainerMgrAddress(); - String nodeRackName = ta.getNodeRackName(); + String nodeRackName = ta.getNodeRackName(); long attemptStartTime = ta.getLaunchTime(); long shuffleFinishTime = -1; @@ -138,58 +143,43 @@ public class HsTaskPage extends HsView { long attemptElapsed = Times.elapsed(attemptStartTime, attemptFinishTime, false); int sortId = ta.getID().getId() + (ta.getID().getTaskId().getId() * 10000); - - TR>> row = tbody.tr(); - TD>>> td = row.td(); - - td.br().$title(String.valueOf(sortId))._(). // sorting - _(taid)._().td(ta.getState().toString()).td().a(".nodelink", - HttpConfig.getSchemePrefix()+ nodeHttpAddr, - nodeRackName + "/" + nodeHttpAddr); - td._(); - row.td(). - a(".logslink", - url("logs", nodeIdString, containerIdString, taid, app.getJob() - .getUserName()), "logs")._(); - - row.td(). - br().$title(String.valueOf(attemptStartTime))._(). - _(Times.format(attemptStartTime))._(); + + attemptsTableData.append("[\"") + .append(sortId + " ").append(taid).append("\",\"") + .append(ta.getState().toString()).append("\",\"") + + .append("") + .append(nodeRackName + "/" + nodeHttpAddr + "\",\"") + + .append("logs\",\"") + + .append(attemptStartTime).append("\",\""); if(type == TaskType.REDUCE) { - row.td(). - br().$title(String.valueOf(shuffleFinishTime))._(). - _(Times.format(shuffleFinishTime))._(); - row.td(). - br().$title(String.valueOf(sortFinishTime))._(). - _(Times.format(sortFinishTime))._(); + attemptsTableData.append(shuffleFinishTime).append("\",\"") + .append(sortFinishTime).append("\",\""); } - row. - td(). - br().$title(String.valueOf(attemptFinishTime))._(). - _(Times.format(attemptFinishTime))._(); - + attemptsTableData.append(attemptFinishTime).append("\",\""); + if(type == TaskType.REDUCE) { - row.td(). - br().$title(String.valueOf(elapsedShuffleTime))._(). - _(formatTime(elapsedShuffleTime))._(); - row.td(). - br().$title(String.valueOf(elapsedSortTime))._(). - _(formatTime(elapsedSortTime))._(); - row.td(). - br().$title(String.valueOf(elapsedReduceTime))._(). - _(formatTime(elapsedReduceTime))._(); + attemptsTableData.append(elapsedShuffleTime).append("\",\"") + .append(elapsedSortTime).append("\",\"") + .append(elapsedReduceTime).append("\",\""); } - - row. - td(). - br().$title(String.valueOf(attemptElapsed))._(). - _(formatTime(attemptElapsed))._(). - td(".note", Joiner.on('\n').join(ta.getDiagnostics())); - row._(); + attemptsTableData.append(attemptElapsed).append("\",\"") + .append(StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml( + Joiner.on('\n').join(ta.getDiagnostics())))).append("\"],\n"); } - - + //Remove the last comma and close off the array of arrays + if(attemptsTableData.charAt(attemptsTableData.length() - 2) == ',') { + attemptsTableData.delete(attemptsTableData.length()-2, attemptsTableData.length()-1); + } + attemptsTableData.append("]"); + html.script().$type("text/javascript"). + _("var attemptsTableData=" + attemptsTableData)._(); + TR>> footRow = tbody._().tfoot().tr(); footRow. th().input("search_init").$type(InputType.text). @@ -237,10 +227,6 @@ public class HsTaskPage extends HsView { footRow._()._()._(); } - private String formatTime(long elapsed) { - return elapsed < 0 ? "N/A" : StringUtils.formatTime(elapsed); - } - /** * @return true if this is a valid request else false. */ @@ -292,24 +278,34 @@ public class HsTaskPage extends HsView { TaskId taskID = MRApps.toTaskID($(TASK_ID)); type = taskID.getTaskType(); } - StringBuilder b = tableInit(). - append(",aoColumnDefs:["); - - b.append("{'sType':'title-numeric', 'aTargets': [ 0"); - if(type == TaskType.REDUCE) { - b.append(", 7, 8, 9, 10"); - } else { //MAP - b.append(", 5"); - } - b.append(" ] }]"); - - // Sort by id upon page load - b.append(", aaSorting: [[0, 'asc']]"); - - b.append("}"); - return b.toString(); + StringBuilder b = tableInit() + .append(", 'aaData': attemptsTableData") + .append(", bDeferRender: true") + .append(", bProcessing: true") + .append("\n,aoColumnDefs:[\n") + + //logs column should not filterable (it includes container ID which may pollute searches) + .append("\n{'aTargets': [ 3 ]") + .append(", 'bSearchable': false }") + + .append("\n, {'sType':'numeric', 'aTargets': [ 0 ]") + .append(", 'mRender': parseHadoopAttemptID }") + + .append("\n, {'sType':'numeric', 'aTargets': [ 4, 5") + //Column numbers are different for maps and reduces + .append(type == TaskType.REDUCE ? ", 6, 7" : "") + .append(" ], 'mRender': renderHadoopDate }") + + .append("\n, {'sType':'numeric', 'aTargets': [") + .append(type == TaskType.REDUCE ? "8, 9, 10, 11" : "6") + .append(" ], 'mRender': renderHadoopElapsedTime }]") + + // Sort by id upon page load + .append("\n, aaSorting: [[0, 'asc']]") + .append("}"); + return b.toString(); } - + private String attemptsPostTableInit() { return "var asInitVals = new Array();\n" + "$('tfoot input').keyup( function () \n{"+ Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTasksBlock.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTasksBlock.java?rev=1446495&r1=1446494&r2=1446495&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTasksBlock.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTasksBlock.java Fri Feb 15 10:18:03 2013 @@ -140,6 +140,7 @@ public class HsTasksBlock extends HtmlBl attemptFinishTime = ta.getFinishTime(); attemptElapsed = ta.getElapsedTime(); } + tasksTableData.append("[\"") .append("") .append(tid).append("\",\"") @@ -205,9 +206,4 @@ public class HsTasksBlock extends HtmlBl footRow._()._()._(); } - - private String formatTime(long elapsed) { - return elapsed < 0 ? "N/A" : StringUtils.formatTime(elapsed); - } - } Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTasksPage.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTasksPage.java?rev=1446495&r1=1446494&r2=1446495&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTasksPage.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTasksPage.java Fri Feb 15 10:18:03 2013 @@ -67,33 +67,25 @@ public class HsTasksPage extends HsView type = MRApps.taskType(symbol); } StringBuilder b = tableInit(). - append(", 'aaData': tasksTableData"); - b.append(", bDeferRender: true"); - b.append(", bProcessing: true"); + append(", 'aaData': tasksTableData") + .append(", bDeferRender: true") + .append(", bProcessing: true") - b.append("\n, aoColumnDefs: [\n"); - b.append("{'sType':'numeric', 'aTargets': [ 0 ]"); - b.append(", 'mRender': parseHadoopID }"); + .append("\n, aoColumnDefs: [\n") + .append("{'sType':'numeric', 'aTargets': [ 0 ]") + .append(", 'mRender': parseHadoopID }") - b.append(", {'sType':'numeric', 'aTargets': [ 4"); - if(type == TaskType.REDUCE) { - b.append(", 9, 10, 11, 12"); - } else { //MAP - b.append(", 7"); - } - b.append(" ], 'mRender': renderHadoopElapsedTime }"); + .append(", {'sType':'numeric', 'aTargets': [ 4") + .append(type == TaskType.REDUCE ? ", 9, 10, 11, 12" : ", 7") + .append(" ], 'mRender': renderHadoopElapsedTime }") - b.append("\n, {'sType':'numeric', 'aTargets': [ 2, 3, 5"); - if(type == TaskType.REDUCE) { - b.append(", 6, 7, 8"); - } else { //MAP - b.append(", 6"); - } - b.append(" ], 'mRender': renderHadoopDate }]"); + .append("\n, {'sType':'numeric', 'aTargets': [ 2, 3, 5") + .append(type == TaskType.REDUCE ? ", 6, 7, 8" : ", 6") + .append(" ], 'mRender': renderHadoopDate }]") // Sort by id upon page load - b.append("\n, aaSorting: [[0, 'asc']]"); - b.append("}"); + .append("\n, aaSorting: [[0, 'asc']]") + .append("}"); return b.toString(); } Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestLineRecordReader.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestLineRecordReader.java?rev=1446495&r1=1446494&r2=1446495&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestLineRecordReader.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestLineRecordReader.java Fri Feb 15 10:18:03 2013 @@ -98,6 +98,7 @@ public class TestLineRecordReader extend InterruptedException, ClassNotFoundException { Configuration conf = new Configuration(); conf.set("textinputformat.record.delimiter", "\t\n"); + conf.setInt("mapreduce.job.maps", 1); FileSystem localFs = FileSystem.getLocal(conf); // cleanup localFs.delete(workDir, true); Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestChainMapReduce.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestChainMapReduce.java?rev=1446495&r1=1446494&r2=1446495&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestChainMapReduce.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestChainMapReduce.java Fri Feb 15 10:18:03 2013 @@ -82,6 +82,7 @@ public class TestChainMapReduce extends JobConf conf = createJobConf(); conf.setBoolean("localFS", isLocalFS()); + conf.setInt("mapreduce.job.maps", 1); cleanFlags(conf); Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestClientProtocolProviderImpls.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestClientProtocolProviderImpls.java?rev=1446495&r1=1446494&r2=1446495&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestClientProtocolProviderImpls.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestClientProtocolProviderImpls.java Fri Feb 15 10:18:03 2013 @@ -42,24 +42,10 @@ public class TestClientProtocolProviderI } - try { - conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME); - conf.set(JTConfig.JT_IPC_ADDRESS, "127.0.0.1:0"); - - new Cluster(conf); - fail("Cluster with Local Framework name should use local JT address"); - } catch (IOException e) { - - } - - try { - conf.set(JTConfig.JT_IPC_ADDRESS, "local"); - Cluster cluster = new Cluster(conf); - assertTrue(cluster.getClient() instanceof LocalJobRunner); - cluster.close(); - } catch (IOException e) { - - } + conf.set(MRConfig.FRAMEWORK_NAME, "local"); + Cluster cluster = new Cluster(conf); + assertTrue(cluster.getClient() instanceof LocalJobRunner); + cluster.close(); } @Test