hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vino...@apache.org
Subject svn commit: r1153441 - in /hadoop/common/branches/MR-279/mapreduce: mr-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/ yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resou...
Date Wed, 03 Aug 2011 11:44:02 GMT
Author: vinodkv
Date: Wed Aug  3 11:44:01 2011
New Revision: 1153441

URL: http://svn.apache.org/viewvc?rev=1153441&view=rev
Log:
FifoScheduler works on a single node!

Added:
    hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerFinishedEvent.java
Modified:
    hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
    hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
    hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
    hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
    hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java

Modified: hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java?rev=1153441&r1=1153440&r2=1153441&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
(original)
+++ hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
Wed Aug  3 11:44:01 2011
@@ -281,11 +281,12 @@ public class TestMRJobs {
 
   protected Job runFailingMapperJob()
   throws IOException, InterruptedException, ClassNotFoundException {
-    mrCluster.getConfig().setInt(MRJobConfig.NUM_MAPS, 1);
-    mrCluster.getConfig().setInt("mapreduce.task.timeout", 10*1000);//reduce the timeout
-    mrCluster.getConfig().setInt(MRJobConfig.MAP_MAX_ATTEMPTS, 2); //reduce the number of
attempts
+    Configuration myConf = new Configuration(mrCluster.getConfig());
+    myConf.setInt(MRJobConfig.NUM_MAPS, 1);
+    myConf.setInt("mapreduce.task.timeout", 10*1000);//reduce the timeout
+    myConf.setInt(MRJobConfig.MAP_MAX_ATTEMPTS, 2); //reduce the number of attempts
 
-    Job job = new Job(mrCluster.getConfig());
+    Job job = new Job(myConf);
 
     job.setJarByClass(FailingMapper.class);
     job.setJobName("failmapper");

Modified: hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java?rev=1153441&r1=1153440&r2=1153441&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
(original)
+++ hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
Wed Aug  3 11:44:01 2011
@@ -437,7 +437,6 @@ public class ResourceManager extends Com
       resourceManager.start();
     } catch (Throwable e) {
       LOG.error("Error starting RM", e);
-    } finally {
       if (resourceManager != null) {
         resourceManager.stop();
       }

Added: hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerFinishedEvent.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerFinishedEvent.java?rev=1153441&view=auto
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerFinishedEvent.java
(added)
+++ hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerFinishedEvent.java
Wed Aug  3 11:44:01 2011
@@ -0,0 +1,19 @@
+package org.apache.hadoop.yarn.server.resourcemanager.rmcontainer;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+
+public class RMContainerFinishedEvent extends RMContainerEvent {
+
+  private final ContainerStatus remoteContainerStatus;
+
+  public RMContainerFinishedEvent(ContainerId containerId,
+      ContainerStatus containerStatus) {
+    super(containerId, RMContainerEventType.FINISHED);
+    this.remoteContainerStatus = containerStatus;
+  }
+
+  public ContainerStatus getRemoteContainerStatus() {
+    return this.remoteContainerStatus;
+  }
+}

Modified: hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java?rev=1153441&r1=1153440&r2=1153441&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
(original)
+++ hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
Wed Aug  3 11:44:01 2011
@@ -198,6 +198,14 @@ public class RMContainerImpl implements 
     @Override
     public void transition(RMContainerImpl container, RMContainerEvent event) {
 
+      RMContainerFinishedEvent finishedEvent = (RMContainerFinishedEvent) event;
+
+      // Update container-status for diagnostics. Today we completely
+      // replace it on finish. We may just need to update diagnostics.
+      // ^TODO
+      container.container.setContainerStatus(finishedEvent
+          .getRemoteContainerStatus());
+
       // Inform AppAttempt
       container.eventHandler.handle(new RMAppAttemptContainerFinishedEvent(
           container.appAttemptId, container.container));

Modified: hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java?rev=1153441&r1=1153440&r2=1153441&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
(original)
+++ hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
Wed Aug  3 11:44:01 2011
@@ -48,6 +48,7 @@ import org.apache.hadoop.yarn.server.api
 import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerFinishedEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
@@ -379,28 +380,29 @@ public class RMNodeImpl implements RMNod
         return RMNodeState.UNHEALTHY;
       }
 
-      Map<ApplicationId, List<Container>> appConts = statusEvent
+      Map<ApplicationId, List<Container>> remoteAppContainersMap = statusEvent
           .getContainersCollection();
-      for (List<Container> l : appConts.values()) {
-        for (Container cont : l) {
+      for (List<Container> remoteContainerList : remoteAppContainersMap
+          .values()) {
+        for (Container remoteContainer : remoteContainerList) {
 
           // Process running containers
-          ContainerId containerId = cont.getId();
-          if (cont.getContainerStatus().getState() == ContainerState.RUNNING
-              || cont.getContainerStatus().getState() == ContainerState.INITIALIZING) {
+          ContainerId containerId = remoteContainer.getId();
+          if (remoteContainer.getContainerStatus().getState() == ContainerState.RUNNING
+              || remoteContainer.getContainerStatus().getState() == ContainerState.INITIALIZING)
{
             if (!rmNode.launchedContainers.containsKey(containerId)) {
-              rmNode.launchedContainers.put(containerId, cont);
+              // Just launched container. RM knows about it the first time.
+              rmNode.launchedContainers.put(containerId, remoteContainer);
               rmNode.context.getDispatcher().getEventHandler().handle(
                   new RMContainerEvent(containerId,
                       RMContainerEventType.LAUNCHED));
             }
           } else {
-
-            rmNode.launchedContainers.remove(containerId);
+            // A finished container
             // Send event to the finished container
             rmNode.context.getDispatcher().getEventHandler().handle(
-                new RMContainerEvent(containerId,
-                    RMContainerEventType.FINISHED));
+                new RMContainerFinishedEvent(containerId, remoteContainer
+                    .getContainerStatus()));
           }
         }
       }

Modified: hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java?rev=1153441&r1=1153440&r2=1153441&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
(original)
+++ hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
Wed Aug  3 11:44:01 2011
@@ -165,7 +165,9 @@ public class MiniYARNCluster extends Com
         remoteLogDir.mkdir();
         LOG.info("Created logDir in " + logDir.getAbsolutePath());
         getConfig().set(NMConfig.NM_LOG_DIR, logDir.getAbsolutePath());
-        getConfig().set(NMConfig.REMOTE_USER_LOG_DIR, remoteLogDir.getAbsolutePath());
+        getConfig().set(NMConfig.REMOTE_USER_LOG_DIR,
+            remoteLogDir.getAbsolutePath());
+        getConfig().setInt(NMConfig.NM_VMEM_GB, 4); // By default AM + 2 containers
         nodeManager = new NodeManager() {
 
           @Override



Mime
View raw message