Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java?rev=1079193&r1=1079192&r2=1079193&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java
Tue Mar 8 05:54:13 2011
@@ -302,6 +302,9 @@ public class JobHistoryParser {
info.launchTime = event.getLaunchTime();
info.totalMaps = event.getTotalMaps();
info.totalReduces = event.getTotalReduces();
+ info.isUber = event.getIsUber();
+ info.numUberSubMaps = event.getNumUberSubMaps();
+ info.numUberSubReduces = event.getNumUberSubReduces();
}
private void handleJobInfoChangeEvent(JobInfoChangeEvent event) {
@@ -334,6 +337,9 @@ public class JobHistoryParser {
long launchTime;
int totalMaps;
int totalReduces;
+ boolean isUber;
+ int numUberSubMaps;
+ int numUberSubReduces;
int failedMaps;
int failedReduces;
int finishedMaps;
@@ -352,8 +358,9 @@ public class JobHistoryParser {
*/
public JobInfo() {
submitTime = launchTime = finishTime = -1;
- totalMaps = totalReduces = failedMaps = failedReduces = 0;
- finishedMaps = finishedReduces = 0;
+ isUber = false;
+ totalMaps = totalReduces = numUberSubMaps = numUberSubReduces = 0;
+ failedMaps = failedReduces = finishedMaps = finishedReduces = 0;
username = jobname = jobConfPath = jobQueueName = "";
tasksMap = new HashMap<TaskID, TaskInfo>();
jobACLs = new HashMap<JobACL, AccessControlList>();
@@ -370,6 +377,7 @@ public class JobHistoryParser {
System.out.println("PRIORITY: " + priority);
System.out.println("TOTAL_MAPS: " + totalMaps);
System.out.println("TOTAL_REDUCES: " + totalReduces);
+ //GRR FIXME: add UBER_SUBMAPS and UBER_SUBREDUCES? (or only if isUber == true? coordinate
with TaskInfo printAll() changes)
System.out.println("MAP_COUNTERS:" + mapCounters.toString());
System.out.println("REDUCE_COUNTERS:" + reduceCounters.toString());
System.out.println("TOTAL_COUNTERS: " + totalCounters.toString());
@@ -395,18 +403,24 @@ public class JobHistoryParser {
public String getJobConfPath() { return jobConfPath; }
/** Get the job launch time */
public long getLaunchTime() { return launchTime; }
- /** Get the total number of maps */
- public long getTotalMaps() { return totalMaps; }
- /** Get the total number of reduces */
- public long getTotalReduces() { return totalReduces; }
+ /** Get the total number of "real" maps */
+ public int getTotalMaps() { return totalMaps; }
+ /** Get the total number of "real" reduces */
+ public int getTotalReduces() { return totalReduces; }
+ /** Was the job small enough to be converted to an UberTask? */
+ public boolean getIsUber() { return isUber; }
+ /** Get the number of sub-MapTasks within the UberTask */
+ public int getNumUberSubMaps() { return numUberSubMaps; }
+ /** Get the number of sub-ReduceTasks within the UberTask */
+ public int getNumUberSubReduces() { return numUberSubReduces; }
/** Get the total number of failed maps */
- public long getFailedMaps() { return failedMaps; }
+ public int getFailedMaps() { return failedMaps; }
/** Get the number of failed reduces */
- public long getFailedReduces() { return failedReduces; }
+ public int getFailedReduces() { return failedReduces; }
/** Get the number of finished maps */
- public long getFinishedMaps() { return finishedMaps; }
+ public int getFinishedMaps() { return finishedMaps; }
/** Get the number of finished reduces */
- public long getFinishedReduces() { return finishedReduces; }
+ public int getFinishedReduces() { return finishedReduces; }
/** Get the job status */
public String getJobStatus() { return jobStatus; }
public String getErrorInfo() { return errorInfo; }
Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/jobhistory/JobInitedEvent.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/jobhistory/JobInitedEvent.java?rev=1079193&r1=1079192&r2=1079193&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/jobhistory/JobInitedEvent.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/jobhistory/JobInitedEvent.java
Tue Mar 8 05:54:13 2011
@@ -44,11 +44,15 @@ public class JobInitedEvent implements H
* @param jobStatus
*/
public JobInitedEvent(JobID id, long launchTime, int totalMaps,
- int totalReduces, String jobStatus) {
+ int totalReduces, boolean isUber, int numUberSubMaps,
+ int numUberSubReduces, String jobStatus) {
datum.jobid = new Utf8(id.toString());
datum.launchTime = launchTime;
datum.totalMaps = totalMaps;
datum.totalReduces = totalReduces;
+ datum.isUber = isUber;
+ datum.numUberSubMaps = numUberSubMaps;
+ datum.numUberSubReduces = numUberSubReduces;
datum.jobStatus = new Utf8(jobStatus);
}
@@ -61,10 +65,16 @@ public class JobInitedEvent implements H
public JobID getJobId() { return JobID.forName(datum.jobid.toString()); }
/** Get the launch time */
public long getLaunchTime() { return datum.launchTime; }
- /** Get the total number of maps */
+ /** Get the total number of "real" maps */
public int getTotalMaps() { return datum.totalMaps; }
- /** Get the total number of reduces */
+ /** Get the total number of "real" reduces */
public int getTotalReduces() { return datum.totalReduces; }
+ /** Was the job small enough to be converted to an UberTask? */
+ public boolean getIsUber() { return datum.isUber; }
+ /** Get the number of sub-MapTasks within the UberTask */
+ public int getNumUberSubMaps() { return datum.numUberSubMaps; }
+ /** Get the number of sub-ReduceTasks within the UberTask */
+ public int getNumUberSubReduces() { return datum.numUberSubReduces; }
/** Get the status */
public String getStatus() { return datum.jobStatus.toString(); }
/** Get the event type */
Modified: hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestJobQueueTaskScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestJobQueueTaskScheduler.java?rev=1079193&r1=1079192&r2=1079193&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestJobQueueTaskScheduler.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestJobQueueTaskScheduler.java
Tue Mar 8 05:54:13 2011
@@ -241,8 +241,12 @@ public class TestJobQueueTaskScheduler e
}
@Override
+ public boolean getIsUber() {
+ return t.isUberTask();
+ }
+
+ @Override
public void addFetchFailedMap(TaskAttemptID mapTaskId) {
-
}
};
status.setRunState(TaskStatus.State.RUNNING);
Modified: hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestTaskStatus.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestTaskStatus.java?rev=1079193&r1=1079192&r2=1079193&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestTaskStatus.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestTaskStatus.java
Tue Mar 8 05:54:13 2011
@@ -103,7 +103,7 @@ public class TestTaskStatus {
// check the default case
String test = "hi";
final int maxSize = 16;
- TaskStatus status = new TaskStatus(null, 0, 0, null, test, test, null, null,
+ TaskStatus status = new TaskStatus(null, 0, 0, null, test, test, null, null,
null) {
@Override
protected int getMaxStringSize() {
@@ -118,6 +118,11 @@ public class TestTaskStatus {
public boolean getIsMap() {
return false;
}
+
+ @Override
+ public boolean getIsUber() {
+ return false;
+ }
};
assertEquals("Small diagnostic info test failed",
status.getDiagnosticInfo(), test);
@@ -198,6 +203,11 @@ public class TestTaskStatus {
public boolean getIsMap() {
return false;
}
+
+ @Override
+ public boolean getIsUber() {
+ return false;
+ }
};
assertEquals("Large diagnostic info test failed",
maxSize, status.getDiagnosticInfo().length());
Modified: hadoop/mapreduce/branches/yahoo-merge/src/tools/org/apache/hadoop/tools/rumen/Job20LineHistoryEventEmitter.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/tools/org/apache/hadoop/tools/rumen/Job20LineHistoryEventEmitter.java?rev=1079193&r1=1079192&r2=1079193&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/tools/org/apache/hadoop/tools/rumen/Job20LineHistoryEventEmitter.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/tools/org/apache/hadoop/tools/rumen/Job20LineHistoryEventEmitter.java
Tue Mar 8 05:54:13 2011
@@ -125,10 +125,15 @@ public class Job20LineHistoryEventEmitte
String status = line.get("JOB_STATUS");
String totalMaps = line.get("TOTAL_MAPS");
String totalReduces = line.get("TOTAL_REDUCES");
+ // note: UberTask playback not supported since uber data not yet logged
+ boolean isUber = false;
+ int numUberSubMaps = 0;
+ int numUberSubReduces = 0;
if (launchTime != null && totalMaps != null && totalReduces != null)
{
- return new JobInitedEvent(jobID, Long.parseLong(launchTime), Integer
- .parseInt(totalMaps), Integer.parseInt(totalReduces), status);
+ return new JobInitedEvent(jobID, Long.parseLong(launchTime),
+ Integer.parseInt(totalMaps), Integer.parseInt(totalReduces),
+ isUber, numUberSubMaps, numUberSubReduces, status);
}
return null;
Modified: hadoop/mapreduce/branches/yahoo-merge/src/webapps/job/jobdetails.jsp
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/webapps/job/jobdetails.jsp?rev=1079193&r1=1079192&r2=1079193&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/webapps/job/jobdetails.jsp (original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/webapps/job/jobdetails.jsp Tue Mar 8 05:54:13
2011
@@ -365,10 +365,16 @@
"<th>Killed</th>" +
"<th><a href=\"jobfailures.jsp?jobid=" + jobId +
"\">Failed/Killed<br>Task Attempts</a></th></tr>\n");
- printTaskSummary(out, jobId, "map", status.mapProgress(),
- job.getTasks(TaskType.MAP));
- printTaskSummary(out, jobId, "reduce", status.reduceProgress(),
- job.getTasks(TaskType.REDUCE));
+ if (job.getUberMode()) {
+ /* placeholder until true task- and attempt-level uber info available */
+ printTaskSummary(out, jobId, "uber", status.reduceProgress(),
+ job.getTasks(TaskType.REDUCE));
+ } else {
+ printTaskSummary(out, jobId, "map", status.mapProgress(),
+ job.getTasks(TaskType.MAP));
+ printTaskSummary(out, jobId, "reduce", status.reduceProgress(),
+ job.getTasks(TaskType.REDUCE));
+ }
out.print("</table>\n");
%>
@@ -421,6 +427,7 @@
%>
</table>
+<%if (job.getTasks(TaskType.MAP).length > 0) { %>
<hr>Map Completion Graph -
<%
if("off".equals(request.getParameter("map.graph"))) {
@@ -442,10 +449,14 @@ if("off".equals(session.getAttribute("ma
width="<%=TaskGraphServlet.width + 2 * TaskGraphServlet.xmargin%>"
height="<%=TaskGraphServlet.height + 3 * TaskGraphServlet.ymargin%>"
style="width:100%" type="image/svg+xml" pluginspage="http://www.adobe.com/svg/viewer/install/"
/>
-<%}%>
+<%} }%>
-<%if(job.getTasks(TaskType.REDUCE).length > 0) { %>
+<%if (job.getTasks(TaskType.REDUCE).length > 0) { %>
+<%if (job.getUberMode()) { %>
+<hr>UberTask Completion Graph -
+<%} else { %>
<hr>Reduce Completion Graph -
+<%}%>
<%if("off".equals(session.getAttribute("reduce.graph"))) { %>
<a href="/jobdetails.jsp?jobid=<%=jobId%>&refresh=<%=refresh%>&reduce.graph=on"
> open </a>
<%} else { %>
Modified: hadoop/mapreduce/branches/yahoo-merge/src/webapps/job/jobdetailshistory.jsp
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/webapps/job/jobdetailshistory.jsp?rev=1079193&r1=1079192&r2=1079193&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/webapps/job/jobdetailshistory.jsp (original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/webapps/job/jobdetailshistory.jsp Tue Mar 8
05:54:13 2011
@@ -56,6 +56,12 @@
if (job == null) {
return;
}
+ // MR-1220 FIXME (LATER): to fully integrate uberization, need task-
+ // and attempt-level JobHistory/Avro/etc. changes to multiple Event
+ // types; instead, going with top-level hack for now
+ boolean isUber = job.getIsUber();
+ int numUberSubMaps = job.getNumUberSubMaps();
+ int numUberSubReduces = job.getNumUberSubReduces();
if (job.getJobStatus().equals("FAILED"))
reasonforFailure = job.getErrorInfo();
%>
@@ -90,7 +96,7 @@
<center>
<table border="2" cellpadding="5" cellspacing="2">
<tr>
-<td>Kind</td><td>Total Tasks(successful+failed+killed)</td><td>Successful
tasks</td><td>Failed tasks</td><td>Killed tasks</td><td>Start
Time</td><td>Finish Time</td>
+<td>Kind</td><td>Total Tasks (successful+<wbr>failed+<wbr>killed)</td><td>Successful
tasks</td><td>Failed tasks</td><td>Killed tasks</td><td>Start
Time</td><td>Finish Time</td>
</tr>
<tr>
<td>Setup</td>
@@ -106,6 +112,10 @@
<td><%=StringUtils.getFormattedTimeWithDiff(dateFormat, sj.getSetupFinished(),
sj.getSetupStarted()) %></td>
</tr>
<tr>
+
+<%
+ if (!isUber) {
+%>
<td>Map</td>
<td><a href="jobtaskshistory.jsp?logFile=<%=logFile%>&taskType=MAP&status=all">
<%=sj.getTotalMaps()%></a></td>
@@ -131,6 +141,36 @@
<td><%=StringUtils.getFormattedTimeWithDiff(dateFormat, sj.getReduceStarted(),
0) %></td>
<td><%=StringUtils.getFormattedTimeWithDiff(dateFormat, sj.getReduceFinished(),
sj.getReduceStarted()) %></td>
</tr>
+
+<%
+ } else /* isUber */ {
+%>
+
+<tr>
+<td>Uber</td>
+ <td><a href="jobtaskshistory.jsp?logFile=<%=logFile%>&taskType=REDUCE&status=all">
+ <%=sj.getTotalReduces()%></a></td>
+ <td><a href="jobtaskshistory.jsp?logFile=<%=logFile%>&taskType=REDUCE&status=SUCCEEDED">
+ <%=job.getFinishedReduces()%></a></td>
+ <td><a href="jobtaskshistory.jsp?logFile=<%=logFile%>&taskType=REDUCE&status=FAILED">
+ <%=sj.getNumFailedReduces()%></a></td>
+ <td><a href="jobtaskshistory.jsp?logFile=<%=logFile%>&taskType=REDUCE&status=KILLED">
+ <%=sj.getNumKilledReduces()%></a></td>
+ <td><%=StringUtils.getFormattedTimeWithDiff(dateFormat, sj.getReduceStarted(),
0) %></td>
+ <td><%=StringUtils.getFormattedTimeWithDiff(dateFormat, sj.getReduceFinished(),
sj.getReduceStarted()) %></td>
+</tr>
+<td>Map subtasks</td>
+ <td colspan="6"><%=numUberSubMaps%></td>
+</tr>
+<tr>
+<td>Reduce subtasks</td>
+ <td colspan="6"><%=numUberSubReduces%></td>
+</tr>
+
+<%
+ }
+%>
+
<tr>
<td>Cleanup</td>
<td><a href="jobtaskshistory.jsp?logFile=<%=logFile%>&taskType=JOB_CLEANUP&status=all">
|