Author: mahadev
Date: Fri Mar 25 01:10:49 2011
New Revision: 1085233
URL: http://svn.apache.org/viewvc?rev=1085233&view=rev
Log:
Fix NodeManager expiry in the RM to notify scheduler and remove from data structures of RM
contributed by mahadev konar
Modified:
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/RMResourceTrackerImpl.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/RMResourceTrackerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/RMResourceTrackerImpl.java?rev=1085233&r1=1085232&r2=1085233&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/RMResourceTrackerImpl.java
(original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/RMResourceTrackerImpl.java
Fri Mar 25 01:10:49 2011
@@ -269,8 +269,15 @@ ResourceTracker, ResourceContext {
nmExpiryQueue.add(status);
}
}
+
protected void expireNMs(List<NodeID> nodes) {
- /* for now do nothing */
+ for (NodeID id: nodes) {
+ synchronized (nodeManagers) {
+ NodeInfo nInfo = nodeManagers.get(id).getNodeInfo();
+ nodeManagers.remove(id);
+ resourceListener.removeNode(nInfo);
+ }
+ }
}
/*
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java?rev=1085233&r1=1085232&r2=1085233&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
(original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
Fri Mar 25 01:10:49 2011
@@ -384,6 +384,7 @@ implements ResourceScheduler, CapacitySc
public synchronized void removeNode(NodeInfo nodeInfo) {
org.apache.hadoop.yarn.server.resourcemanager.resource.Resource.subtractResource(
clusterResource, nodeInfo.getTotalCapability());
+ //TODO inform the applications that the containers are completed/failed
nodes.remove(nodeInfo.getHostName());
}
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java?rev=1085233&r1=1085232&r2=1085233&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
(original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
Fri Mar 25 01:10:49 2011
@@ -454,6 +454,7 @@ public class FifoScheduler implements Re
public synchronized void removeNode(NodeInfo nodeInfo) {
org.apache.hadoop.yarn.server.resourcemanager.resource.Resource.subtractResource(
clusterResource, nodeInfo.getTotalCapability());
+ //TODO inform the the applications that the containers are completed/failed
nodes.remove(nodeInfo.getHostName());
}
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java?rev=1085233&r1=1085232&r2=1085233&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java
(original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java
Fri Mar 25 01:10:49 2011
@@ -1,20 +1,20 @@
/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.hadoop.yarn.server.resourcemanager.resourcetracker;
@@ -52,7 +52,7 @@ public class TestNMExpiry extends TestCa
ContainerTokenSecretManager containerTokenSecretManager =
new ContainerTokenSecretManager();
AtomicInteger test = new AtomicInteger();
- Object notify = new Object();
+ AtomicInteger notify = new AtomicInteger();
private static class VoidResourceListener implements ResourceListener {
@Override
@@ -89,6 +89,7 @@ public class TestNMExpiry extends TestCa
Thread.sleep(2000);
} catch(InterruptedException ie){}
synchronized(notify) {
+ notify.addAndGet(1);
notify.notifyAll();
}
}
@@ -137,7 +138,9 @@ public class TestNMExpiry extends TestCa
stopT = false;
new TestThread().start();
synchronized (notify) {
- notify.wait(10000);
+ while (notify.get() == 0) {
+ notify.wait();
+ }
}
if (test.get() != 2)
assertTrue(false);
|