Author: vinodkv
Date: Wed Aug 3 11:44:01 2011
New Revision: 1153441
URL: http://svn.apache.org/viewvc?rev=1153441&view=rev
Log:
FifoScheduler works on a single node!
Added:
hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerFinishedEvent.java
Modified:
hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
Modified: hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java?rev=1153441&r1=1153440&r2=1153441&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
(original)
+++ hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
Wed Aug 3 11:44:01 2011
@@ -281,11 +281,12 @@ public class TestMRJobs {
protected Job runFailingMapperJob()
throws IOException, InterruptedException, ClassNotFoundException {
- mrCluster.getConfig().setInt(MRJobConfig.NUM_MAPS, 1);
- mrCluster.getConfig().setInt("mapreduce.task.timeout", 10*1000);//reduce the timeout
- mrCluster.getConfig().setInt(MRJobConfig.MAP_MAX_ATTEMPTS, 2); //reduce the number of
attempts
+ Configuration myConf = new Configuration(mrCluster.getConfig());
+ myConf.setInt(MRJobConfig.NUM_MAPS, 1);
+ myConf.setInt("mapreduce.task.timeout", 10*1000);//reduce the timeout
+ myConf.setInt(MRJobConfig.MAP_MAX_ATTEMPTS, 2); //reduce the number of attempts
- Job job = new Job(mrCluster.getConfig());
+ Job job = new Job(myConf);
job.setJarByClass(FailingMapper.class);
job.setJobName("failmapper");
Modified: hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java?rev=1153441&r1=1153440&r2=1153441&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
(original)
+++ hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
Wed Aug 3 11:44:01 2011
@@ -437,7 +437,6 @@ public class ResourceManager extends Com
resourceManager.start();
} catch (Throwable e) {
LOG.error("Error starting RM", e);
- } finally {
if (resourceManager != null) {
resourceManager.stop();
}
Added: hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerFinishedEvent.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerFinishedEvent.java?rev=1153441&view=auto
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerFinishedEvent.java
(added)
+++ hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerFinishedEvent.java
Wed Aug 3 11:44:01 2011
@@ -0,0 +1,19 @@
+package org.apache.hadoop.yarn.server.resourcemanager.rmcontainer;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+
+public class RMContainerFinishedEvent extends RMContainerEvent {
+
+ private final ContainerStatus remoteContainerStatus;
+
+ public RMContainerFinishedEvent(ContainerId containerId,
+ ContainerStatus containerStatus) {
+ super(containerId, RMContainerEventType.FINISHED);
+ this.remoteContainerStatus = containerStatus;
+ }
+
+ public ContainerStatus getRemoteContainerStatus() {
+ return this.remoteContainerStatus;
+ }
+}
Modified: hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java?rev=1153441&r1=1153440&r2=1153441&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
(original)
+++ hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
Wed Aug 3 11:44:01 2011
@@ -198,6 +198,14 @@ public class RMContainerImpl implements
@Override
public void transition(RMContainerImpl container, RMContainerEvent event) {
+ RMContainerFinishedEvent finishedEvent = (RMContainerFinishedEvent) event;
+
+ // Update container-status for diagnostics. Today we completely
+ // replace it on finish. We may just need to update diagnostics.
+ // ^TODO
+ container.container.setContainerStatus(finishedEvent
+ .getRemoteContainerStatus());
+
// Inform AppAttempt
container.eventHandler.handle(new RMAppAttemptContainerFinishedEvent(
container.appAttemptId, container.container));
Modified: hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java?rev=1153441&r1=1153440&r2=1153441&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
(original)
+++ hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
Wed Aug 3 11:44:01 2011
@@ -48,6 +48,7 @@ import org.apache.hadoop.yarn.server.api
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerFinishedEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
@@ -379,28 +380,29 @@ public class RMNodeImpl implements RMNod
return RMNodeState.UNHEALTHY;
}
- Map<ApplicationId, List<Container>> appConts = statusEvent
+ Map<ApplicationId, List<Container>> remoteAppContainersMap = statusEvent
.getContainersCollection();
- for (List<Container> l : appConts.values()) {
- for (Container cont : l) {
+ for (List<Container> remoteContainerList : remoteAppContainersMap
+ .values()) {
+ for (Container remoteContainer : remoteContainerList) {
// Process running containers
- ContainerId containerId = cont.getId();
- if (cont.getContainerStatus().getState() == ContainerState.RUNNING
- || cont.getContainerStatus().getState() == ContainerState.INITIALIZING) {
+ ContainerId containerId = remoteContainer.getId();
+ if (remoteContainer.getContainerStatus().getState() == ContainerState.RUNNING
+ || remoteContainer.getContainerStatus().getState() == ContainerState.INITIALIZING)
{
if (!rmNode.launchedContainers.containsKey(containerId)) {
- rmNode.launchedContainers.put(containerId, cont);
+ // Just launched container. RM knows about it the first time.
+ rmNode.launchedContainers.put(containerId, remoteContainer);
rmNode.context.getDispatcher().getEventHandler().handle(
new RMContainerEvent(containerId,
RMContainerEventType.LAUNCHED));
}
} else {
-
- rmNode.launchedContainers.remove(containerId);
+ // A finished container
// Send event to the finished container
rmNode.context.getDispatcher().getEventHandler().handle(
- new RMContainerEvent(containerId,
- RMContainerEventType.FINISHED));
+ new RMContainerFinishedEvent(containerId, remoteContainer
+ .getContainerStatus()));
}
}
}
Modified: hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java?rev=1153441&r1=1153440&r2=1153441&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
(original)
+++ hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
Wed Aug 3 11:44:01 2011
@@ -165,7 +165,9 @@ public class MiniYARNCluster extends Com
remoteLogDir.mkdir();
LOG.info("Created logDir in " + logDir.getAbsolutePath());
getConfig().set(NMConfig.NM_LOG_DIR, logDir.getAbsolutePath());
- getConfig().set(NMConfig.REMOTE_USER_LOG_DIR, remoteLogDir.getAbsolutePath());
+ getConfig().set(NMConfig.REMOTE_USER_LOG_DIR,
+ remoteLogDir.getAbsolutePath());
+ getConfig().setInt(NMConfig.NM_VMEM_GB, 4); // By default AM + 2 containers
nodeManager = new NodeManager() {
@Override
|