From mapreduce-commits-return-1487-apmail-hadoop-mapreduce-commits-archive=hadoop.apache.org@hadoop.apache.org Tue Apr 05 08:40:42 2011 Return-Path: Delivered-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Received: (qmail 60890 invoked from network); 5 Apr 2011 08:40:42 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.3) by minotaur.apache.org with SMTP; 5 Apr 2011 08:40:42 -0000 Received: (qmail 97085 invoked by uid 500); 5 Apr 2011 08:40:41 -0000 Delivered-To: apmail-hadoop-mapreduce-commits-archive@hadoop.apache.org Received: (qmail 96977 invoked by uid 500); 5 Apr 2011 08:40:35 -0000 Mailing-List: contact mapreduce-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: mapreduce-dev@hadoop.apache.org Delivered-To: mailing list mapreduce-commits@hadoop.apache.org Received: (qmail 96956 invoked by uid 99); 5 Apr 2011 08:40:31 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 05 Apr 2011 08:40:31 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 05 Apr 2011 08:40:27 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id D7D3223889D7; Tue, 5 Apr 2011 08:40:07 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1088923 [2/2] - in /hadoop/mapreduce/branches/MR-279: mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/ mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ src/contrib/fairsc... Date: Tue, 05 Apr 2011 08:40:07 -0000 To: mapreduce-commits@hadoop.apache.org From: vinodkv@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20110405084007.D7D3223889D7@eris.apache.org> Added: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/NodeManagerImpl.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/NodeManagerImpl.java?rev=1088923&view=auto ============================================================================== --- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/NodeManagerImpl.java (added) +++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/NodeManagerImpl.java Tue Apr 5 08:40:05 2011 @@ -0,0 +1,340 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.net.Node; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerState; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.factories.RecordFactory; +import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; +import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus; +import org.apache.hadoop.yarn.server.api.records.NodeId; +import org.apache.hadoop.yarn.server.resourcemanager.resourcetracker.NodeInfo; + +/** + * This class is used by ClusterInfo to keep track of all the applications/containers + * running on a node. + * + */ +@Private +@Unstable +public class NodeManagerImpl implements NodeManager { + private static final Log LOG = LogFactory.getLog(NodeManager.class); + private static final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); + private final NodeId nodeId; + private final String hostName; + private Resource totalCapability; + private Resource availableResource = recordFactory.newRecordInstance(Resource.class); + private Resource usedResource = recordFactory.newRecordInstance(Resource.class); + private final Node node; + private final NodeHealthStatus nodeHealthStatus = recordFactory + .newRecordInstance(NodeHealthStatus.class); + + private static final Container[] EMPTY_CONTAINER_ARRAY = new Container[] {}; + private static final List EMPTY_CONTAINER_LIST = Arrays.asList(EMPTY_CONTAINER_ARRAY); + private static final ApplicationId[] EMPTY_APPLICATION_ARRAY = new ApplicationId[]{}; + private static final List EMPTY_APPLICATION_LIST = Arrays.asList(EMPTY_APPLICATION_ARRAY); + + public static final String ANY = "*"; + /* set of containers that are allocated containers */ + private final Map allocatedContainers = + new TreeMap(); + + /* set of containers that are currently active on a node manager */ + private final Map activeContainers = + new TreeMap(); + + /* set of containers that need to be cleaned */ + private final Set containersToClean = + new TreeSet(new org.apache.hadoop.yarn.server.resourcemanager.resource.Container.Comparator()); + + + /* the list of applications that have finished and need to be purged */ + private final List finishedApplications = new ArrayList(); + + private volatile int numContainers; + + public NodeManagerImpl(NodeId nodeId, String hostname, + Node node, Resource capability) { + this.nodeId = nodeId; + this.totalCapability = capability; + this.hostName = hostname; + org.apache.hadoop.yarn.server.resourcemanager.resource.Resource.addResource( + availableResource, capability); + this.node = node; + } + + /** + * NodeInfo for this node. + * @return the {@link NodeInfo} for this node. + */ + public NodeInfo getNodeInfo() { + return this; + } + + /** + * The Scheduler has allocated containers on this node to the + * given application. + * + * @param applicationId application + * @param containers allocated containers + */ + public synchronized void allocateContainer(ApplicationId applicationId, + List containers) { + if (containers == null) { + LOG.error("Adding null containers for application " + applicationId); + return; + } + for (Container container : containers) { + allocateContainer(container); + } + + LOG.info("addContainers:" + + " node=" + getHostName() + + " #containers=" + containers.size() + + " available=" + getAvailableResource().getMemory() + + " used=" + getUsedResource().getMemory()); + } + + /** + * Status update from the NodeManager + * @param nodeStatus node status + * @return the set of containers no longer should be used by the + * node manager. + */ + public synchronized NodeResponse + statusUpdate(Map> allContainers) { + + if (allContainers == null) { + return new NodeResponse(EMPTY_APPLICATION_LIST, EMPTY_CONTAINER_LIST, + EMPTY_CONTAINER_LIST); + } + + List listContainers = new ArrayList(); + // Iterate through the running containers and update their status + for (Map.Entry> e : + allContainers.entrySet()) { + listContainers.addAll(e.getValue()); + } + NodeResponse statusCheck = update(listContainers); + return statusCheck; + } + + /** + * Status update for an application running on a given node + * @param node node + * @param containers containers update. + * @return containers that are completed or need to be preempted. + */ + private synchronized NodeResponse update(List containers) { + List completedContainers = new ArrayList(); + List containersToCleanUp = new ArrayList(); + List lastfinishedApplications = new ArrayList(); + + for (Container container : containers) { + if (allocatedContainers.remove(container.getId()) != null) { + activeContainers.put(container.getId(), container); + LOG.info("Activated container " + container.getId() + " on node " + + getHostName()); + } + + if (container.getState() == ContainerState.COMPLETE) { + if (activeContainers.remove(container.getId()) != null) { + updateResource(container); + LOG.info("Completed container " + container); + } + completedContainers.add(container); + LOG.info("Removed completed container " + container.getId() + " on node " + + getHostName()); + } + else if (container.getState() != ContainerState.COMPLETE && + (!allocatedContainers.containsKey(container.getId())) && + !activeContainers.containsKey(container.getId())) { + containersToCleanUp.add(container); + } + } + containersToCleanUp.addAll(containersToClean); + /* clear out containers to clean */ + containersToClean.clear(); + lastfinishedApplications.addAll(finishedApplications); + return new NodeResponse(lastfinishedApplications, completedContainers, + containersToCleanUp); + } + + private synchronized void allocateContainer(Container container) { + deductAvailableResource(container.getResource()); + ++numContainers; + + allocatedContainers.put(container.getId(), container); + LOG.info("Allocated container " + container.getId() + + " to node " + getHostName()); + + LOG.info("Assigned container " + container.getId() + + " of capacity " + container.getResource() + " on host " + getHostName() + + ", which currently has " + numContainers + " containers, " + + getUsedResource() + " used and " + + getAvailableResource() + " available"); + } + + private synchronized boolean isValidContainer(Container c) { + if (activeContainers.containsKey(c.getId()) || allocatedContainers.containsKey(c.getId())) + return true; + return false; + } + + private synchronized void updateResource(Container container) { + addAvailableResource(container.getResource()); + --numContainers; + } + + /** + * Release an allocated container on this node. + * @param container container to be released + * @return true iff the container was unused, + * false otherwise + */ + public synchronized boolean releaseContainer(Container container) { + if (!isValidContainer(container)) { + LOG.error("Invalid container released " + container); + return false; + } + + /* remove the containers from the nodemanger */ + + // Was this container launched? + activeContainers.remove(container.getId()); + allocatedContainers.remove(container.getId()); + containersToClean.add(container); + updateResource(container); + + LOG.info("Released container " + container.getId() + + " of capacity " + container.getResource() + " on host " + getHostName() + + ", which currently has " + numContainers + " containers, " + + getUsedResource() + " used and " + getAvailableResource() + + " available" + ", release resources=" + true); + return true; + } + + @Override + public NodeId getNodeID() { + return this.nodeId; + } + + @Override + public String getHostName() { + return this.hostName; + } + + @Override + public Resource getTotalCapability() { + return this.totalCapability; + } + + @Override + public String getRackName() { + return node.getNetworkLocation(); + } + + @Override + public Node getNode() { + return this.node; + } + + @Override + public synchronized Resource getAvailableResource() { + return this.availableResource; + } + + @Override + public synchronized Resource getUsedResource() { + return this.usedResource; + } + + public synchronized void addAvailableResource(Resource resource) { + if (resource == null) { + LOG.error("Invalid resource addition of null resource for " + this.hostName); + return; + } + org.apache.hadoop.yarn.server.resourcemanager.resource.Resource.addResource( + availableResource, resource); + org.apache.hadoop.yarn.server.resourcemanager.resource.Resource.subtractResource( + usedResource, resource); + } + + public synchronized void deductAvailableResource(Resource resource) { + if (resource == null) { + LOG.error("Invalid deduction of null resource for "+ this.hostName); + } + org.apache.hadoop.yarn.server.resourcemanager.resource.Resource.subtractResource( + availableResource, resource); + org.apache.hadoop.yarn.server.resourcemanager.resource.Resource.addResource( + usedResource, resource); + } + + public synchronized void notifyFinishedApplication(ApplicationId applicationId) { + finishedApplications.add(applicationId); + /* make sure to iterate through the list and remove all the containers that + * belong to this application. + */ + } + + @Override + public int getNumContainers() { + return numContainers; + } + + @Override + public NodeHealthStatus getNodeHealthStatus() { + synchronized (this.nodeHealthStatus) { + return this.nodeHealthStatus; + } + } + + @Override + public void updateHealthStatus(NodeHealthStatus healthStatus) { + synchronized (this.nodeHealthStatus) { + this.nodeHealthStatus.setIsNodeHealthy(healthStatus.getIsNodeHealthy()); + this.nodeHealthStatus.setHealthReport(healthStatus.getHealthReport()); + this.nodeHealthStatus.setLastHealthReportTime(healthStatus + .getLastHealthReportTime()); + } + } + + @Override + public String toString() { + return "host: " + getHostName() + " #containers=" + getNumContainers() + + " available=" + getAvailableResource().getMemory() + + " used=" + getUsedResource().getMemory(); + } + } Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceListener.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceListener.java?rev=1088923&r1=1088922&r2=1088923&view=diff ============================================================================== --- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceListener.java (original) +++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceListener.java Tue Apr 5 08:40:05 2011 @@ -39,14 +39,9 @@ public interface ResourceListener { /** * add a node to the resource listener. - * @param nodeId the nodeid of the node - * @param hostName the hostname of this node. - * @param node the topology information. - * @param capability the resource capability of the node. - * @return the {@link NodeInfo} object that tracks this nodemanager. + * @param nodeManager the nodeManager view */ - public NodeInfo addNode(NodeId nodeId,String hostName, - Node node, Resource capability); + public void addNode(NodeManager nodeManager); /** * A node has been removed from the cluster. Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java?rev=1088923&r1=1088922&r2=1088923&view=diff ============================================================================== --- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java (original) +++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java Tue Apr 5 08:40:05 2011 @@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.resourcetracker.NodeInfo; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Application; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeManagerImpl; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeResponse; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager; @@ -394,13 +395,10 @@ implements ResourceScheduler, CapacitySc } @Override - public synchronized NodeInfo addNode(NodeId nodeId, - String hostName, Node node, Resource capability) { - NodeManager nodeManager = new NodeManager(nodeId, hostName, node, capability); + public synchronized void addNode(NodeManager nodeManager) { nodes.put(nodeManager.getHostName(), nodeManager); org.apache.hadoop.yarn.server.resourcemanager.resource.Resource.addResource( clusterResource, nodeManager.getTotalCapability()); - return nodeManager; } public synchronized boolean releaseContainer(ApplicationId applicationId, Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java?rev=1088923&r1=1088922&r2=1088923&view=diff ============================================================================== --- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java (original) +++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java Tue Apr 5 08:40:05 2011 @@ -468,13 +468,10 @@ public class FifoScheduler implements Re } @Override - public synchronized NodeInfo addNode(NodeId nodeId, - String hostName, Node node, Resource capability) { - NodeManager nodeManager = new NodeManager(nodeId, hostName, node, capability); + public synchronized void addNode(NodeManager nodeManager) { nodes.put(nodeManager.getHostName(), nodeManager); org.apache.hadoop.yarn.server.resourcemanager.resource.Resource.addResource( clusterResource, nodeManager.getTotalCapability()); - return nodeManager; } public synchronized boolean releaseContainer(ApplicationId applicationId, Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java?rev=1088923&r1=1088922&r2=1088923&view=diff ============================================================================== --- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java (original) +++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java Tue Apr 5 08:40:05 2011 @@ -48,6 +48,9 @@ class NodesPage extends RmView { th(".rack", "Rack"). th(".nodeid", "Node ID"). th(".host", "Host"). + th(".healthStatus", "Health-status"). + th(".lastHealthUpdate", "Last health-update"). + th(".healthReport", "Health-report"). th(".containers", "Containers"). th(".mem", "Mem Used (MB)"). th(".mem", "Mem Avail (MB)")._()._(). @@ -57,6 +60,11 @@ class NodesPage extends RmView { td(ni.getRackName()). td(String.valueOf(ni.getNodeID().getId())). td(ni.getHostName()). + td(ni.getNodeHealthStatus().getIsNodeHealthy() ? "Healthy" + : "Unhealthy"). + td(String.valueOf(ni.getNodeHealthStatus() + .getLastHealthReportTime())). + td(String.valueOf(ni.getNodeHealthStatus().getHealthReport())). td(String.valueOf(ni.getNumContainers())). td(String.valueOf(ni.getUsedResource().getMemory())). td(String.valueOf(ni.getAvailableResource().getMemory()))._(); @@ -79,8 +87,8 @@ class NodesPage extends RmView { private String nodesTableInit() { return tableInit(). - // rack, nodeid, host, containers, memused, memavail - append(", aoColumns:[null, null, null, {bSearchable:false}, "). + // rack, nodeid, host, healthStatus, containers, memused, memavail + append(", aoColumns:[null, null, null, null, {bSearchable:false}, "). append("{bSearchable:false}, {bSearchable:false}]}").toString(); } } Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java?rev=1088923&r1=1088922&r2=1088923&view=diff ============================================================================== --- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java (original) +++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java Tue Apr 5 08:40:05 2011 @@ -26,6 +26,7 @@ import org.apache.hadoop.net.Node; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; +import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus; import org.apache.hadoop.yarn.server.api.records.NodeId; import org.apache.hadoop.yarn.server.resourcemanager.resourcetracker.NodeInfo; @@ -76,6 +77,8 @@ public class MockNodes { final int nid = NODE_ID++; final NodeId nodeID = newNodeID(nid); final String hostName = "host"+ nid; + final NodeHealthStatus nodeHealthStatus = + recordFactory.newRecordInstance(NodeHealthStatus.class); final Resource used = newUsedResource(perNode); final Resource avail = newAvailResource(perNode, used); final int containers = (int)(Math.random() * 8); @@ -119,6 +122,11 @@ public class MockNodes { public int getNumContainers() { return containers; } + + @Override + public NodeHealthStatus getNodeHealthStatus() { + return nodeHealthStatus; + } }; } } Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java?rev=1088923&r1=1088922&r2=1088923&view=diff ============================================================================== --- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java (original) +++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java Tue Apr 5 08:40:05 2011 @@ -36,6 +36,7 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager.events.ApplicationMasterEvents.ApplicationEventType; import org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager.events.ApplicationMasterEvents.ApplicationTrackerEventType; import org.apache.hadoop.yarn.server.resourcemanager.resourcetracker.NodeInfo; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeResponse; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; @@ -129,9 +130,7 @@ public class TestAMRestart extends TestC private class DummyResourceScheduler implements ResourceScheduler { @Override - public NodeInfo addNode(NodeId nodeId, String hostName, Node node, - Resource capability) { - return null; + public void addNode(NodeManager nodeManager) { } @Override public void removeNode(NodeInfo node) { Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestApplicationCleanup.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestApplicationCleanup.java?rev=1088923&r1=1088922&r2=1088923&view=diff ============================================================================== --- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestApplicationCleanup.java (original) +++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestApplicationCleanup.java Tue Apr 5 08:40:05 2011 @@ -54,6 +54,8 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager.events.ApplicationMasterEvents.ApplicationTrackerEventType; import org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager.events.ApplicationMasterEvents.SNEventType; import org.apache.hadoop.yarn.server.resourcemanager.resourcetracker.NodeInfo; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeManagerImpl; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeResponse; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; @@ -259,7 +261,10 @@ public class TestApplicationCleanup exte Node node = new NodeBase(hostName, NetworkTopology.DEFAULT_RACK); Resource capability = recordFactory.newRecordInstance(Resource.class); capability.setMemory(1024); - return scheduler.addNode(nodeId, hostName, node, capability); + NodeManager nodeManager = + new NodeManagerImpl(nodeId, hostName, node, capability); + scheduler.addNode(nodeManager); + return nodeManager; } @Test Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestSchedulerNegotiator.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestSchedulerNegotiator.java?rev=1088923&r1=1088922&r2=1088923&view=diff ============================================================================== --- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestSchedulerNegotiator.java (original) +++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestSchedulerNegotiator.java Tue Apr 5 08:40:05 2011 @@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager.events.ApplicationMasterEvents.ApplicationEventType; import org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager.events.ApplicationMasterEvents.ApplicationTrackerEventType; import org.apache.hadoop.yarn.server.resourcemanager.resourcetracker.NodeInfo; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeResponse; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager; @@ -80,9 +81,7 @@ public class TestSchedulerNegotiator ext ContainerTokenSecretManager secretManager) { } @Override - public NodeInfo addNode(NodeId nodeId, String hostName, Node node, - Resource capability) { - return null; + public void addNode(NodeManager nodeManager) { } @Override public NodeResponse nodeUpdate(NodeInfo nodeInfo, Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java?rev=1088923&r1=1088922&r2=1088923&view=diff ============================================================================== --- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java (original) +++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java Tue Apr 5 08:40:05 2011 @@ -61,9 +61,7 @@ public class TestNMExpiry extends TestCa private static class VoidResourceListener implements ResourceListener { @Override - public NodeInfo addNode(NodeId nodeId, String hostName, Node node, - Resource capability) { - return new NodeManager(nodeId, hostName, node, capability); + public void addNode(NodeManager nodeManager) { } @Override public void removeNode(NodeInfo node) { @@ -111,17 +109,22 @@ public class TestNMExpiry extends TestCa resourceTracker.start(); } - private class TestThread extends Thread { + private class ThirdNodeHeartBeatThread extends Thread { public void run() { - HeartbeatResponse res = recordFactory.newRecordInstance(HeartbeatResponse.class); + int lastResponseID = 0; while (!stopT) { try { - org.apache.hadoop.yarn.server.api.records.NodeStatus nodeStatus = recordFactory.newRecordInstance(org.apache.hadoop.yarn.server.api.records.NodeStatus.class); - nodeStatus.setNodeId(response.getNodeId()); - nodeStatus.setResponseId(res.getResponseId()); - NodeHeartbeatRequest request = recordFactory.newRecordInstance(NodeHeartbeatRequest.class); + org.apache.hadoop.yarn.server.api.records.NodeStatus nodeStatus = + recordFactory + .newRecordInstance(org.apache.hadoop.yarn.server.api.records.NodeStatus.class); + nodeStatus.setNodeId(thirdNodeRegResponse.getNodeId()); + nodeStatus.setResponseId(lastResponseID); + NodeHeartbeatRequest request = + recordFactory.newRecordInstance(NodeHeartbeatRequest.class); request.setNodeStatus(nodeStatus); - res = resourceTracker.nodeHeartbeat(request).getHeartbeatResponse(); + lastResponseID = + resourceTracker.nodeHeartbeat(request).getHeartbeatResponse() + .getResponseId(); } catch(Exception e) { LOG.info("failed to heartbeat ", e); } @@ -130,7 +133,7 @@ public class TestNMExpiry extends TestCa } boolean stopT = false; - RegistrationResponse response; + RegistrationResponse thirdNodeRegResponse; @Test public void testNMExpiry() throws Exception { @@ -149,13 +152,16 @@ public class TestNMExpiry extends TestCa request3.setResource(capability); resourceTracker.registerNodeManager(request1); resourceTracker.registerNodeManager(request2); - response = resourceTracker.registerNodeManager(request3).getRegistrationResponse(); + thirdNodeRegResponse = + resourceTracker.registerNodeManager(request3) + .getRegistrationResponse(); /* test to see if hostanme 3 does not expire */ stopT = false; - new TestThread().start(); + new ThirdNodeHeartBeatThread().start(); + int timeOut = 0; synchronized (notify) { - while (notify.get() == 0) { - notify.wait(); + while (notify.get() == 0 && timeOut++ < 30) { + notify.wait(1000); } } if (test.get() != 2) Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java?rev=1088923&r1=1088922&r2=1088923&view=diff ============================================================================== --- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java (original) +++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java Tue Apr 5 08:40:05 2011 @@ -36,6 +36,7 @@ import org.apache.hadoop.yarn.server.api import org.apache.hadoop.yarn.server.resourcemanager.resourcetracker.NodeInfo; import org.apache.hadoop.yarn.server.resourcemanager.resourcetracker.RMResourceTrackerImpl; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeManagerImpl; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeResponse; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceListener; import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager; @@ -55,10 +56,8 @@ public class TestRMNMRPCResponseId exten private class DummyResourceListener implements ResourceListener { @Override - public NodeInfo addNode(NodeId nodeId, String hostName, Node node, - Resource capability) { - nodeid = nodeId; - return new NodeManager(nodeId, hostName, node, capability); + public void addNode(NodeManager nodeManager) { + nodeid = nodeManager.getNodeID(); } @Override