Author: acmurthy
Date: Tue Apr 19 07:50:39 2011
New Revision: 1094915
URL: http://svn.apache.org/viewvc?rev=1094915&view=rev
Log:
Added functionality to refresh queues at runtime via the 'bin/yarn rmadmin' command.
Added:
hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/main/java/org/apache/hadoop/yarn/security/admin/
hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/main/java/org/apache/hadoop/yarn/security/admin/AdminSecurityInfo.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/RMAdminProtocol.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/impl/
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/impl/pb/
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/impl/pb/client/
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/impl/pb/client/RMAdminProtocolPBClientImpl.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/impl/pb/service/
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/impl/pb/service/RMAdminProtocolPBServiceImpl.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/protocolrecords/
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/protocolrecords/RefreshQueuesRequest.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/protocolrecords/RefreshQueuesResponse.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/protocolrecords/impl/
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/protocolrecords/impl/pb/
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/protocolrecords/impl/pb/RefreshQueuesRequestPBImpl.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/protocolrecords/impl/pb/RefreshQueuesResponsePBImpl.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/tools/
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/tools/RMAdmin.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/proto/
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/proto/RMAdminProtocol.proto
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_service_protos.proto
Modified:
hadoop/mapreduce/branches/MR-279/CHANGES.txt
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java
hadoop/mapreduce/branches/MR-279/yarn/bin/yarn
hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-common/src/main/resources/yarn-default.xml
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/pom.xml
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/Queue.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
Modified: hadoop/mapreduce/branches/MR-279/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/CHANGES.txt?rev=1094915&r1=1094914&r2=1094915&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/MR-279/CHANGES.txt Tue Apr 19 07:50:39 2011
@@ -4,6 +4,9 @@ Trunk (unreleased changes)
MAPREDUCE-279
+ Added functionality to refresh queues at runtime via the 'bin/yarn
+ rmadmin' command. (acmurthy)
+
Fixed computation of user-limits at runtime. (acmurthy)
Implement 'bin/mapred queue [-info [-showJobs]] [-list] and enhanced
Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java?rev=1094915&r1=1094914&r2=1094915&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java Tue Apr 19 07:50:39 2011
@@ -217,8 +217,7 @@ public class TestRMContainerAllocator {
}
private FifoScheduler createScheduler() throws YarnRemoteException {
- FifoScheduler fsc = new FifoScheduler(new Configuration(),
- new ContainerTokenSecretManager()) {
+ FifoScheduler fsc = new FifoScheduler() {
//override this to copy the objects
//otherwise FifoScheduler updates the numContainers in same objects as kept by
//RMContainerAllocator
@@ -241,6 +240,7 @@ public class TestRMContainerAllocator {
}
};
try {
+ fsc.reinitialize(new Configuration(), new ContainerTokenSecretManager());
fsc.addApplication(recordFactory.newRecordInstance(ApplicationId.class), "test", null, null);
} catch(IOException ie) {
LOG.info("add application failed with ", ie);
Modified: hadoop/mapreduce/branches/MR-279/yarn/bin/yarn
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/bin/yarn?rev=1094915&r1=1094914&r2=1094915&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/bin/yarn (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/bin/yarn Tue Apr 19 07:50:39 2011
@@ -61,11 +61,10 @@ esac
if [ $# = 0 ]; then
echo "Usage: hadoop [--config confdir] COMMAND"
echo "where COMMAND is one of:"
- echo " resourcemanager run the ResourceManager"
+ echo " resourcemanager run the ResourceManager"
echo " nodemanager run a nodemanager on each slave"
echo " historyserver run job history servers as a standalone daemon"
- echo " job manipulate MapReduce jobs"
- echo " queue get information regarding JobQueues"
+ echo " rmadmin admin tools"
echo " version print the version"
echo " jar <jar> run a jar file"
echo " classpath prints the class path needed to get the"
@@ -281,6 +280,9 @@ if [ "$COMMAND" = "classpath" ] ; then
fi
echo $CLASSPATH
exit
+elif [ "$COMMAND" = "rmadmin" ] ; then
+ CLASS='org.apache.hadoop.yarn.server.resourcemanager.tools.RMAdmin'
+ YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS"
elif [ "$COMMAND" = "resourcemanager" ] ; then
CLASSPATH=${CLASSPATH}:$YARN_CONF_DIR/rm-config/log4j.properties
CLASS='org.apache.hadoop.yarn.server.resourcemanager.ResourceManager'
@@ -296,12 +298,6 @@ elif [ "$COMMAND" = "nodemanager" ] ; th
elif [ "$COMMAND" = "historyserver" ] ; then
CLASS=org.apache.hadoop.mapreduce.v2.hs.JobHistoryServer
YARN_OPTS="$YARN_OPTS $YARN_JOB_HISTORYSERVER_OPTS"
-elif [ "$COMMAND" = "job" ] ; then
- CLASS=org.apache.hadoop.mapred.JobClient
- YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS"
-elif [ "$COMMAND" = "queue" ] ; then
- CLASS=org.apache.hadoop.mapred.JobQueueClient
- YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS"
elif [ "$COMMAND" = "version" ] ; then
CLASS=org.apache.hadoop.util.VersionInfo
YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS"
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1094915&r1=1094914&r2=1094915&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Tue Apr 19 07:50:39 2011
@@ -30,6 +30,8 @@ public class YarnConfiguration extends C
public static final String SCHEDULER_ADDRESS = RM_PREFIX
+ "scheduler.address";
+ public static final String ADMIN_ADDRESS = RM_PREFIX + "admin.address";
+
public static final String AM_EXPIRY_INTERVAL = RM_PREFIX
+ "application.expiry.interval";
@@ -47,6 +49,8 @@ public class YarnConfiguration extends C
public static final String DEFAULT_SCHEDULER_BIND_ADDRESS = "0.0.0.0:8030";
+ public static final String DEFAULT_ADMIN_BIND_ADDRESS = "0.0.0.0:8141";
+
public static final String APPSMANAGER_ADDRESS = RM_PREFIX
+ "appsManager.address";
Added: hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/main/java/org/apache/hadoop/yarn/security/admin/AdminSecurityInfo.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/main/java/org/apache/hadoop/yarn/security/admin/AdminSecurityInfo.java?rev=1094915&view=auto
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/main/java/org/apache/hadoop/yarn/security/admin/AdminSecurityInfo.java (added)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/main/java/org/apache/hadoop/yarn/security/admin/AdminSecurityInfo.java Tue Apr 19 07:50:39 2011
@@ -0,0 +1,38 @@
+package org.apache.hadoop.yarn.security.admin;
+
+import java.lang.annotation.Annotation;
+
+import org.apache.hadoop.security.KerberosInfo;
+import org.apache.hadoop.security.SecurityInfo;
+import org.apache.hadoop.security.token.TokenInfo;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+
+public class AdminSecurityInfo implements SecurityInfo {
+
+ @Override
+ public KerberosInfo getKerborosInfo(Class<?> protocol) {
+ return new KerberosInfo() {
+
+ @Override
+ public Class<? extends Annotation> annotationType() {
+ return null;
+ }
+
+ @Override
+ public String serverPrincipal() {
+ return YarnConfiguration.RM_SERVER_PRINCIPAL_KEY;
+ }
+
+ @Override
+ public String clientPrincipal() {
+ return null;
+ }
+ };
+ }
+
+ @Override
+ public TokenInfo getTokenInfo(Class<?> protocol) {
+ return null;
+ }
+
+}
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-common/src/main/resources/yarn-default.xml
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-common/src/main/resources/yarn-default.xml?rev=1094915&r1=1094914&r2=1094915&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-common/src/main/resources/yarn-default.xml (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-common/src/main/resources/yarn-default.xml Tue Apr 19 07:50:39 2011
@@ -29,6 +29,11 @@
<value>0.0.0.0:8030</value>
</property>
+ <property>
+ <name>yarn.server.resourcemanager.admin.address</name>
+ <value>0.0.0.0:8141</value>
+ </property>
+
<property>
<name>yarn.server.resourcemanager.application.max.retries</name>
<value>1</value>
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/pom.xml?rev=1094915&r1=1094914&r2=1094915&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/pom.xml (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/pom.xml Tue Apr 19 07:50:39 2011
@@ -22,4 +22,67 @@
</dependency>
</dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>create-protobuf-generated-sources-directory</id>
+ <phase>initialize</phase>
+ <configuration>
+ <target>
+ <mkdir dir="target/generated-sources/proto" />
+ </target>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>generate-sources</id>
+ <phase>generate-sources</phase>
+ <configuration>
+ <executable>protoc</executable>
+ <arguments>
+ <argument>-Isrc/main/proto/</argument>
+ <argument>--java_out=target/generated-sources/proto</argument>
+ <argument>src/main/proto/yarn_server_resourcemanager_service_protos.proto</argument>
+ <argument>src/main/proto/RMAdminProtocol.proto</argument>
+ </arguments>
+ </configuration>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>add-source</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>target/generated-sources/proto</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
</project>
Added: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java?rev=1094915&view=auto
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java (added)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java Tue Apr 19 07:50:39 2011
@@ -0,0 +1,89 @@
+package org.apache.hadoop.yarn.server.resourcemanager;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import org.apache.avro.ipc.Server;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.SecurityInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.api.RMAdminProtocol;
+import org.apache.hadoop.yarn.server.resourcemanager.api.protocolrecords.RefreshQueuesRequest;
+import org.apache.hadoop.yarn.server.resourcemanager.api.protocolrecords.RefreshQueuesResponse;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
+import org.apache.hadoop.yarn.factories.RecordFactory;
+import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.ipc.RPCUtil;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.security.SchedulerSecurityInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.service.AbstractService;
+
+public class AdminService extends AbstractService implements RMAdminProtocol {
+
+ private static final Log LOG = LogFactory.getLog(AdminService.class);
+
+ private final Configuration conf;
+ private final ResourceScheduler scheduler;
+
+ private Server server;
+ private InetSocketAddress masterServiceAddress;
+
+ private final RecordFactory recordFactory =
+ RecordFactoryProvider.getRecordFactory(null);
+
+ public AdminService(Configuration conf, ResourceScheduler scheduler) {
+ super(AdminService.class.getName());
+ this.conf = conf;
+ this.scheduler = scheduler;
+ }
+
+ @Override
+ public void init(Configuration conf) {
+ String bindAddress =
+ conf.get(YarnConfiguration.ADMIN_ADDRESS,
+ YarnConfiguration.DEFAULT_ADMIN_BIND_ADDRESS);
+ masterServiceAddress = NetUtils.createSocketAddr(bindAddress);
+ super.init(conf);
+ }
+
+ public void start() {
+ YarnRPC rpc = YarnRPC.create(getConfig());
+ Configuration serverConf = new Configuration(getConfig());
+ serverConf.setClass(
+ CommonConfigurationKeys.HADOOP_SECURITY_INFO_CLASS_NAME,
+ SchedulerSecurityInfo.class, SecurityInfo.class);
+ this.server =
+ rpc.getServer(RMAdminProtocol.class, this, masterServiceAddress,
+ serverConf, null);
+ this.server.start();
+ super.start();
+ }
+
+ @Override
+ public void stop() {
+ if (this.server != null) {
+ this.server.close();
+ }
+ super.stop();
+ }
+
+ @Override
+ public RefreshQueuesResponse refreshQueues(RefreshQueuesRequest request)
+ throws YarnRemoteException {
+ try {
+ scheduler.reinitialize(conf, null); // ContainerTokenSecretManager can't
+ // be 'refreshed'
+ RefreshQueuesResponse response =
+ recordFactory.newRecordInstance(RefreshQueuesResponse.class);
+ return response;
+ } catch (IOException ioe) {
+ LOG.info("Exception refreshing queues ", ioe);
+ throw RPCUtil.getRemoteException(ioe);
+ }
+ }
+}
\ No newline at end of file
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java?rev=1094915&r1=1094914&r2=1094915&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java Tue Apr 19 07:50:39 2011
@@ -66,6 +66,7 @@ public class ResourceManager extends Com
private RMResourceTrackerImpl rmResourceTracker;
private ClientRMService clientRM;
private ApplicationMasterService masterService;
+ private AdminService adminService;
private AtomicBoolean shutdown = new AtomicBoolean(false);
private WebApp webApp;
private final ASMContext asmContext;
@@ -103,7 +104,11 @@ public class ResourceManager extends Com
conf.getClass(YarnConfiguration.RESOURCE_SCHEDULER,
FifoScheduler.class, ResourceScheduler.class),
this.conf);
- this.scheduler.reinitialize(this.conf, this.containerTokenSecretManager);
+ try {
+ this.scheduler.reinitialize(this.conf, this.containerTokenSecretManager);
+ } catch (IOException ioe) {
+ throw new RuntimeException("Failed to initialize scheduler", ioe);
+ }
/* add the scheduler to be notified of events from the applications managers */
this.asmContext.getDispatcher().register(ApplicationTrackerEventType.class, this.scheduler);
//TODO change this to be random
@@ -121,6 +126,10 @@ public class ResourceManager extends Com
masterService = createApplicationMasterService();
addService(masterService) ;
+
+ adminService = createAdminService(conf, scheduler);
+ addService(adminService);
+
super.init(conf);
}
@@ -187,6 +196,12 @@ public class ResourceManager extends Com
this.appTokenSecretManager, applicationsManager, scheduler, this.asmContext);
}
+
+ protected AdminService createAdminService(Configuration conf,
+ ResourceScheduler scheduler) {
+ return new AdminService(conf, scheduler);
+ }
+
/**
* return applications manager.
* @return
Added: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/RMAdminProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/RMAdminProtocol.java?rev=1094915&view=auto
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/RMAdminProtocol.java (added)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/RMAdminProtocol.java Tue Apr 19 07:50:39 2011
@@ -0,0 +1,10 @@
+package org.apache.hadoop.yarn.server.resourcemanager.api;
+
+import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
+import org.apache.hadoop.yarn.server.resourcemanager.api.protocolrecords.RefreshQueuesRequest;
+import org.apache.hadoop.yarn.server.resourcemanager.api.protocolrecords.RefreshQueuesResponse;
+
+public interface RMAdminProtocol {
+ public RefreshQueuesResponse refreshQueues(RefreshQueuesRequest request)
+ throws YarnRemoteException;
+}
Added: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/impl/pb/client/RMAdminProtocolPBClientImpl.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/impl/pb/client/RMAdminProtocolPBClientImpl.java?rev=1094915&view=auto
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/impl/pb/client/RMAdminProtocolPBClientImpl.java (added)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/impl/pb/client/RMAdminProtocolPBClientImpl.java Tue Apr 19 07:50:39 2011
@@ -0,0 +1,54 @@
+package org.apache.hadoop.yarn.server.resourcemanager.api.impl.pb.client;
+
+import java.io.IOException;
+import java.lang.reflect.UndeclaredThrowableException;
+import java.net.InetSocketAddress;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
+import org.apache.hadoop.yarn.ipc.ProtoOverHadoopRpcEngine;
+import org.apache.hadoop.yarn.proto.RMAdminProtocol.RMAdminProtocolService;
+import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshQueuesRequestProto;
+import org.apache.hadoop.yarn.server.resourcemanager.api.RMAdminProtocol;
+import org.apache.hadoop.yarn.server.resourcemanager.api.protocolrecords.RefreshQueuesRequest;
+import org.apache.hadoop.yarn.server.resourcemanager.api.protocolrecords.RefreshQueuesResponse;
+import org.apache.hadoop.yarn.server.resourcemanager.api.protocolrecords.impl.pb.RefreshQueuesRequestPBImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.api.protocolrecords.impl.pb.RefreshQueuesResponsePBImpl;
+
+import com.google.protobuf.ServiceException;
+
+
+public class RMAdminProtocolPBClientImpl implements RMAdminProtocol {
+
+ private RMAdminProtocolService.BlockingInterface proxy;
+
+ public RMAdminProtocolPBClientImpl(long clientVersion, InetSocketAddress addr,
+ Configuration conf) throws IOException {
+ RPC.setProtocolEngine(conf, RMAdminProtocolService.BlockingInterface.class,
+ ProtoOverHadoopRpcEngine.class);
+ proxy = (RMAdminProtocolService.BlockingInterface)RPC.getProxy(
+ RMAdminProtocolService.BlockingInterface.class, clientVersion, addr, conf);
+ }
+
+ @Override
+ public RefreshQueuesResponse refreshQueues(RefreshQueuesRequest request)
+ throws YarnRemoteException {
+ RefreshQueuesRequestProto requestProto =
+ ((RefreshQueuesRequestPBImpl)request).getProto();
+ try {
+ return new RefreshQueuesResponsePBImpl(
+ proxy.refreshQueues(null, requestProto));
+ } catch (ServiceException e) {
+ if (e.getCause() instanceof YarnRemoteException) {
+ throw (YarnRemoteException)e.getCause();
+ } else if (e.getCause() instanceof UndeclaredThrowableException) {
+ throw (UndeclaredThrowableException)e.getCause();
+ } else {
+ throw new UndeclaredThrowableException(e);
+ }
+ }
+ }
+
+
+}
Added: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/impl/pb/service/RMAdminProtocolPBServiceImpl.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/impl/pb/service/RMAdminProtocolPBServiceImpl.java?rev=1094915&view=auto
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/impl/pb/service/RMAdminProtocolPBServiceImpl.java (added)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/impl/pb/service/RMAdminProtocolPBServiceImpl.java Tue Apr 19 07:50:39 2011
@@ -0,0 +1,34 @@
+package org.apache.hadoop.yarn.server.resourcemanager.api.impl.pb.service;
+
+import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
+import org.apache.hadoop.yarn.proto.RMAdminProtocol.RMAdminProtocolService.BlockingInterface;
+import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.*;
+import org.apache.hadoop.yarn.server.resourcemanager.api.RMAdminProtocol;
+import org.apache.hadoop.yarn.server.resourcemanager.api.protocolrecords.RefreshQueuesResponse;
+import org.apache.hadoop.yarn.server.resourcemanager.api.protocolrecords.impl.pb.RefreshQueuesRequestPBImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.api.protocolrecords.impl.pb.RefreshQueuesResponsePBImpl;
+
+import com.google.protobuf.RpcController;
+import com.google.protobuf.ServiceException;
+
+public class RMAdminProtocolPBServiceImpl implements BlockingInterface {
+
+ private RMAdminProtocol real;
+
+ public RMAdminProtocolPBServiceImpl(RMAdminProtocol impl) {
+ this.real = impl;
+ }
+
+ @Override
+ public RefreshQueuesResponseProto refreshQueues(RpcController controller,
+ RefreshQueuesRequestProto proto) throws ServiceException {
+ RefreshQueuesRequestPBImpl request = new RefreshQueuesRequestPBImpl(proto);
+ try {
+ RefreshQueuesResponse response = real.refreshQueues(request);
+ return ((RefreshQueuesResponsePBImpl)response).getProto();
+ } catch (YarnRemoteException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+}
Added: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/protocolrecords/RefreshQueuesRequest.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/protocolrecords/RefreshQueuesRequest.java?rev=1094915&view=auto
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/protocolrecords/RefreshQueuesRequest.java (added)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/protocolrecords/RefreshQueuesRequest.java Tue Apr 19 07:50:39 2011
@@ -0,0 +1,5 @@
+package org.apache.hadoop.yarn.server.resourcemanager.api.protocolrecords;
+
+public interface RefreshQueuesRequest {
+
+}
Added: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/protocolrecords/RefreshQueuesResponse.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/protocolrecords/RefreshQueuesResponse.java?rev=1094915&view=auto
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/protocolrecords/RefreshQueuesResponse.java (added)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/protocolrecords/RefreshQueuesResponse.java Tue Apr 19 07:50:39 2011
@@ -0,0 +1,5 @@
+package org.apache.hadoop.yarn.server.resourcemanager.api.protocolrecords;
+
+public interface RefreshQueuesResponse {
+
+}
Added: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/protocolrecords/impl/pb/RefreshQueuesRequestPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/protocolrecords/impl/pb/RefreshQueuesRequestPBImpl.java?rev=1094915&view=auto
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/protocolrecords/impl/pb/RefreshQueuesRequestPBImpl.java (added)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/protocolrecords/impl/pb/RefreshQueuesRequestPBImpl.java Tue Apr 19 07:50:39 2011
@@ -0,0 +1,28 @@
+package org.apache.hadoop.yarn.server.resourcemanager.api.protocolrecords.impl.pb;
+
+import org.apache.hadoop.yarn.api.records.ProtoBase;
+import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshQueuesRequestProto;
+import org.apache.hadoop.yarn.server.resourcemanager.api.protocolrecords.RefreshQueuesRequest;
+
+public class RefreshQueuesRequestPBImpl extends ProtoBase<RefreshQueuesRequestProto>
+implements RefreshQueuesRequest {
+
+ RefreshQueuesRequestProto proto = RefreshQueuesRequestProto.getDefaultInstance();
+ RefreshQueuesRequestProto.Builder builder = null;
+ boolean viaProto = false;
+
+ public RefreshQueuesRequestPBImpl() {
+ builder = RefreshQueuesRequestProto.newBuilder();
+ }
+
+ public RefreshQueuesRequestPBImpl(RefreshQueuesRequestProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public RefreshQueuesRequestProto getProto() {
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+}
Added: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/protocolrecords/impl/pb/RefreshQueuesResponsePBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/protocolrecords/impl/pb/RefreshQueuesResponsePBImpl.java?rev=1094915&view=auto
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/protocolrecords/impl/pb/RefreshQueuesResponsePBImpl.java (added)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/api/protocolrecords/impl/pb/RefreshQueuesResponsePBImpl.java Tue Apr 19 07:50:39 2011
@@ -0,0 +1,28 @@
+package org.apache.hadoop.yarn.server.resourcemanager.api.protocolrecords.impl.pb;
+
+import org.apache.hadoop.yarn.api.records.ProtoBase;
+import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshQueuesResponseProto;
+import org.apache.hadoop.yarn.server.resourcemanager.api.protocolrecords.RefreshQueuesResponse;
+
+public class RefreshQueuesResponsePBImpl extends ProtoBase<RefreshQueuesResponseProto>
+implements RefreshQueuesResponse {
+
+ RefreshQueuesResponseProto proto = RefreshQueuesResponseProto.getDefaultInstance();
+ RefreshQueuesResponseProto.Builder builder = null;
+ boolean viaProto = false;
+
+ public RefreshQueuesResponsePBImpl() {
+ builder = RefreshQueuesResponseProto.newBuilder();
+ }
+
+ public RefreshQueuesResponsePBImpl(RefreshQueuesResponseProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public RefreshQueuesResponseProto getProto() {
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+}
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java?rev=1094915&r1=1094914&r2=1094915&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java Tue Apr 19 07:50:39 2011
@@ -44,5 +44,5 @@ public interface ResourceScheduler exten
* @throws IOException
*/
void reinitialize(Configuration conf,
- ContainerTokenSecretManager secretManager);
+ ContainerTokenSecretManager secretManager) throws IOException;
}
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java?rev=1094915&r1=1094914&r2=1094915&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java Tue Apr 19 07:50:39 2011
@@ -33,7 +33,6 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.net.Node;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
@@ -42,13 +41,11 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
-import org.apache.hadoop.yarn.server.api.records.NodeId;
import org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager.events.ASMEvent;
import org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager.events.ApplicationMasterEvents.ApplicationTrackerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.resourcetracker.NodeInfo;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Application;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeManager;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeManagerImpl;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeResponse;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
@@ -98,6 +95,7 @@ implements ResourceScheduler, CapacitySc
new TreeMap<ApplicationId, Application>(
new org.apache.hadoop.yarn.server.resourcemanager.resource.ApplicationID.Comparator());
+ private boolean initialized = false;
public Queue getRootQueue() {
return root;
@@ -119,13 +117,28 @@ implements ResourceScheduler, CapacitySc
}
@Override
- public void reinitialize(Configuration conf,
- ContainerTokenSecretManager containerTokenSecretManager) {
- this.conf = new CapacitySchedulerConfiguration(conf);
- this.minimumAllocation = this.conf.getMinimumAllocation();
- this.containerTokenSecretManager = containerTokenSecretManager;
+ public synchronized void reinitialize(Configuration conf,
+ ContainerTokenSecretManager containerTokenSecretManager)
+ throws IOException {
+ if (!initialized) {
+ this.conf = new CapacitySchedulerConfiguration(conf);
+ this.minimumAllocation = this.conf.getMinimumAllocation();
+ this.containerTokenSecretManager = containerTokenSecretManager;
+
+ initializeQueues(this.conf);
+ initialized = true;
+ } else {
- initializeQueues(this.conf);
+ CapacitySchedulerConfiguration oldConf = this.conf;
+ this.conf = new CapacitySchedulerConfiguration(conf);
+ try {
+ LOG.info("Re-initializing queues...");
+ reinitializeQueues(this.conf);
+ } catch (Throwable t) {
+ this.conf = oldConf;
+ throw new IOException("Failed to re-init queues", t);
+ }
+ }
}
@Private
@@ -136,13 +149,40 @@ implements ResourceScheduler, CapacitySc
CapacitySchedulerConfiguration.PREFIX + ROOT;
private void initializeQueues(CapacitySchedulerConfiguration conf) {
- root = parseQueue(conf, null, ROOT);
- queues.put(ROOT, root);
+ root = parseQueue(conf, null, ROOT, queues);
LOG.info("Initialized root queue " + root);
}
+ private synchronized void reinitializeQueues(CapacitySchedulerConfiguration conf)
+ throws IOException {
+ // Parse new queues
+ Map<String, Queue> newQueues = new HashMap<String, Queue>();
+ Queue newRoot = parseQueue(conf, null, ROOT, newQueues);
+
+ // Ensure all existing queues are still present
+ validateExistingQueues(queues, newQueues);
+
+ // Re-configure queues
+ root.reinitialize(newRoot, clusterResource);
+ }
+
+ /**
+ * Ensure all existing queues are present. Queues cannot be deleted
+ * @param queues existing queues
+ * @param newQueues new queues
+ */
+ private void validateExistingQueues(
+ Map<String, Queue> queues, Map<String, Queue> newQueues)
+ throws IOException {
+ for (String queue : queues.keySet()) {
+ if (!newQueues.containsKey(queue)) {
+ throw new IOException(queue + " cannot be found during refresh!");
+ }
+ }
+ }
+
private Queue parseQueue(CapacitySchedulerConfiguration conf,
- Queue parent, String queueName) {
+ Queue parent, String queueName, Map<String, Queue> queues) {
Queue queue;
String[] childQueueNames =
conf.getQueues((parent == null) ?
@@ -155,19 +195,16 @@ implements ResourceScheduler, CapacitySc
List<Queue> childQueues = new ArrayList<Queue>();
for (String childQueueName : childQueueNames) {
Queue childQueue =
- parseQueue(
- conf,
- parentQueue,
- childQueueName);
+ parseQueue(conf, parentQueue, childQueueName, queues);
childQueues.add(childQueue);
-
- queues.put(childQueueName, childQueue);
}
parentQueue.setChildQueues(childQueues);
queue = parentQueue;
}
+ queues.put(queueName, queue);
+
LOG.info("Initialized queue: " + queue);
return queue;
}
@@ -268,10 +305,14 @@ implements ResourceScheduler, CapacitySc
}
@Override
- public synchronized QueueInfo getQueueInfo(String queueName,
+ public QueueInfo getQueueInfo(String queueName,
boolean includeApplications, boolean includeChildQueues, boolean recursive)
throws IOException {
- Queue queue = this.queues.get(queueName);
+ Queue queue = null;
+
+ synchronized (this) {
+ queue = this.queues.get(queueName);
+ }
if (queue == null) {
throw new IOException("Unknown queue: " + queueName);
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1094915&r1=1094914&r2=1094915&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java Tue Apr 19 07:50:39 2011
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Comparator;
@@ -56,15 +57,15 @@ public class LeafQueue implements Queue
private final String queueName;
private final Queue parent;
- private final float capacity;
- private final float absoluteCapacity;
- private final float maximumCapacity;
- private final float absoluteMaxCapacity;
- private final int userLimit;
- private final float userLimitFactor;
+ private float capacity;
+ private float absoluteCapacity;
+ private float maximumCapacity;
+ private float absoluteMaxCapacity;
+ private int userLimit;
+ private float userLimitFactor;
- private final int maxApplications;
- private final int maxApplicationsPerUser;
+ private int maxApplications;
+ private int maxApplicationsPerUser;
private Resource usedResources =
org.apache.hadoop.yarn.server.resourcemanager.resource.Resource.createResource(0);
@@ -96,28 +97,25 @@ public class LeafQueue implements Queue
this.minimumAllocation = cs.getMinimumAllocation();
this.containerTokenSecretManager = cs.getContainerTokenSecretManager();
- this.capacity =
+ float capacity =
(float)cs.getConfiguration().getCapacity(getQueuePath()) / 100;
- this.absoluteCapacity = parent.getAbsoluteCapacity() * capacity;
+ float absoluteCapacity = parent.getAbsoluteCapacity() * capacity;
- this.maximumCapacity = cs.getConfiguration().getMaximumCapacity(getQueuePath());
- this.absoluteMaxCapacity =
+ float maximumCapacity = cs.getConfiguration().getMaximumCapacity(getQueuePath());
+ float absoluteMaxCapacity =
(maximumCapacity == CapacitySchedulerConfiguration.UNDEFINED) ?
Float.MAX_VALUE : (parent.getAbsoluteCapacity() * maximumCapacity) / 100;
- this.userLimit = cs.getConfiguration().getUserLimit(getQueuePath());
-
- this.userLimitFactor =
+ int userLimit = cs.getConfiguration().getUserLimit(getQueuePath());
+ float userLimitFactor =
cs.getConfiguration().getUserLimitFactor(getQueuePath());
int maxSystemJobs = cs.getConfiguration().getMaximumSystemApplications();
- this.maxApplications = (int)(maxSystemJobs * absoluteCapacity);
- this.maxApplicationsPerUser =
+ int maxApplications = (int)(maxSystemJobs * absoluteCapacity);
+ int maxApplicationsPerUser =
(int)(maxApplications * (userLimit / 100.0f) * userLimitFactor);
this.queueInfo = recordFactory.newRecordInstance(QueueInfo.class);
- this.queueInfo.setCapacity(capacity);
- this.queueInfo.setMaximumCapacity(maximumCapacity);
this.queueInfo.setQueueName(queueName);
this.queueInfo.setChildQueues(new ArrayList<QueueInfo>());
@@ -125,20 +123,50 @@ public class LeafQueue implements Queue
new HashMap<ApplicationId,
org.apache.hadoop.yarn.api.records.Application>();
+ setupQueueConfigs(capacity, absoluteCapacity,
+ maximumCapacity, absoluteMaxCapacity,
+ userLimit, userLimitFactor,
+ maxApplications, maxApplicationsPerUser);
+
LOG.info("DEBUG --- LeafQueue:" +
" name=" + queueName +
- ", fullname=" + getQueuePath() +
- ", capacity=" + capacity +
- ", asboluteCapacity=" + absoluteCapacity +
- ", maxCapacity=" + maximumCapacity +
- ", asboluteMaxCapacity=" + absoluteMaxCapacity +
- ", userLimit=" + userLimit + ", userLimitFactor=" + userLimitFactor +
- ", maxApplications=" + maxApplications +
- ", maxApplicationsPerUser=" + maxApplicationsPerUser);
+ ", fullname=" + getQueuePath());
this.applications = new TreeSet<Application>(applicationComparator);
}
+ private synchronized void setupQueueConfigs(
+ float capacity, float absoluteCapacity,
+ float maxCapacity, float absoluteMaxCapacity,
+ int userLimit, float userLimitFactor,
+ int maxApplications, int maxApplicationsPerUser)
+ {
+ this.capacity = capacity;
+ this.absoluteCapacity = parent.getAbsoluteCapacity() * capacity;
+
+ this.maximumCapacity = maxCapacity;
+ this.absoluteMaxCapacity = absoluteMaxCapacity;
+
+ this.userLimit = userLimit;
+ this.userLimitFactor = userLimitFactor;
+
+ this.maxApplications = maxApplications;
+ this.maxApplicationsPerUser = maxApplicationsPerUser;
+
+ this.queueInfo.setCapacity(capacity);
+ this.queueInfo.setMaximumCapacity(maximumCapacity);
+
+ LOG.info(queueName +
+ ", capacity=" + capacity +
+ ", asboluteCapacity=" + absoluteCapacity +
+ ", maxCapacity=" + maxCapacity +
+ ", asboluteMaxCapacity=" + absoluteMaxCapacity +
+ ", userLimit=" + userLimit + ", userLimitFactor=" + userLimitFactor +
+ ", maxApplications=" + maxApplications +
+ ", maxApplicationsPerUser=" + maxApplicationsPerUser);
+ }
+
+
@Override
public float getCapacity() {
return capacity;
@@ -248,6 +276,25 @@ public class LeafQueue implements Queue
}
@Override
+ public synchronized void reinitialize(Queue queue, Resource clusterResource)
+ throws IOException {
+ // Sanity check
+ if (!(queue instanceof LeafQueue) ||
+ !queue.getQueuePath().equals(getQueuePath())) {
+ throw new IOException("Trying to reinitialize " + getQueuePath() +
+ " from " + queue.getQueuePath());
+ }
+
+ LeafQueue leafQueue = (LeafQueue)queue;
+ setupQueueConfigs(leafQueue.capacity, leafQueue.absoluteCapacity,
+ leafQueue.maximumCapacity, leafQueue.absoluteMaxCapacity,
+ leafQueue.userLimit, leafQueue.userLimitFactor,
+ leafQueue.maxApplications, leafQueue.maxApplicationsPerUser);
+
+ update(clusterResource);
+ }
+
+ @Override
public void submitApplication(Application application, String userName,
String queue, Priority priority)
throws AccessControlException {
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java?rev=1094915&r1=1094914&r2=1094915&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java Tue Apr 19 07:50:39 2011
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
@@ -52,15 +53,17 @@ public class ParentQueue implements Queu
private final Queue parent;
private final String queueName;
- private final float capacity;
- private final float maximumCapacity;
- private final float absoluteCapacity;
- private final float absoluteMaxCapacity;
+
+ private float capacity;
+ private float maximumCapacity;
+ private float absoluteCapacity;
+ private float absoluteMaxCapacity;
private float usedCapacity = 0.0f;
private float utilization = 0.0f;
private final Set<Queue> childQueues;
+ private final Comparator<Queue> queueComparator;
private Resource usedResources =
org.apache.hadoop.yarn.server.resourcemanager.resource.Resource.createResource(0);
@@ -87,55 +90,74 @@ public class ParentQueue implements Queu
this.queueName = queueName;
this.rootQueue = (parent == null);
- LOG.info("PQ: parent=" + parent + ", qName=" + queueName +
- " qPath=" + getQueuePath() + ", root=" + rootQueue);
- this.capacity =
+ float capacity =
(float)cs.getConfiguration().getCapacity(getQueuePath()) / 100;
float parentAbsoluteCapacity =
(parent == null) ? 1.0f : parent.getAbsoluteCapacity();
- this.absoluteCapacity = parentAbsoluteCapacity * capacity;
+ float absoluteCapacity = parentAbsoluteCapacity * capacity;
- this.maximumCapacity =
+ float maximumCapacity =
cs.getConfiguration().getMaximumCapacity(getQueuePath());
- this.absoluteMaxCapacity =
+ float absoluteMaxCapacity =
(maximumCapacity == CapacitySchedulerConfiguration.UNDEFINED) ?
Float.MAX_VALUE : (parentAbsoluteCapacity * maximumCapacity) / 100;
- this.childQueues = new TreeSet<Queue>(comparator);
-
this.queueInfo = recordFactory.newRecordInstance(QueueInfo.class);
- this.queueInfo.setCapacity(capacity);
- this.queueInfo.setMaximumCapacity(maximumCapacity);
this.queueInfo.setQueueName(queueName);
this.queueInfo.setChildQueues(new ArrayList<QueueInfo>());
+
+ setupQueueConfigs(capacity, absoluteCapacity,
+ maximumCapacity, absoluteMaxCapacity);
+ this.queueComparator = comparator;
+ this.childQueues = new TreeSet<Queue>(comparator);
+
this.applicationInfos =
new HashMap<ApplicationId,
org.apache.hadoop.yarn.api.records.Application>();
+
LOG.info("Initialized parent-queue " + queueName +
" name=" + queueName +
- ", fullname=" + getQueuePath() +
- ", capacity=" + capacity +
- ", asboluteCapacity=" + absoluteCapacity +
+ ", fullname=" + getQueuePath());
+ }
+
+ private synchronized void setupQueueConfigs(
+ float capacity, float absoluteCapacity,
+ float maximumCapacity, float absoluteMaxCapacity
+ ) {
+ this.capacity = capacity;
+ this.absoluteCapacity = absoluteCapacity;
+ this.maximumCapacity = maximumCapacity;
+ this.absoluteMaxCapacity = absoluteMaxCapacity;
+
+ this.queueInfo.setCapacity(capacity);
+ this.queueInfo.setMaximumCapacity(maximumCapacity);
+
+ LOG.info(queueName +
+ ", capacity=" + capacity +
+ ", asboluteCapacity=" + absoluteCapacity +
", maxCapacity=" + maximumCapacity +
", asboluteMaxCapacity=" + absoluteMaxCapacity);
}
- public void setChildQueues(Collection<Queue> childQueues) {
+ private static float PRECISION = 0.005f; // 0.05% precision
+ void setChildQueues(Collection<Queue> childQueues) {
// Validate
float childCapacities = 0;
for (Queue queue : childQueues) {
childCapacities += queue.getCapacity();
}
- if (childCapacities != 1.0f) {
+ float delta = Math.abs(1.0f - childCapacities); // crude way to check
+ if (delta > PRECISION) {
throw new IllegalArgumentException("Illegal" +
" capacity of " + childCapacities +
" for children of queue " + queueName);
}
+ this.childQueues.clear();
this.childQueues.addAll(childQueues);
LOG.info("DEBUG --- setChildQueues: " + getChildQueuesToPrint());
}
@@ -246,6 +268,56 @@ public class ParentQueue implements Queu
}
@Override
+ public synchronized void reinitialize(Queue queue, Resource clusterResource)
+ throws IOException {
+ // Sanity check
+ if (!(queue instanceof ParentQueue) ||
+ !queue.getQueuePath().equals(getQueuePath())) {
+ throw new IOException("Trying to reinitialize " + getQueuePath() +
+ " from " + queue.getQueuePath());
+ }
+
+ ParentQueue parentQueue = (ParentQueue)queue;
+
+ // Re-configure existing child queues and add new ones
+ // The CS has already checked to ensure all existing child queues are present!
+ Map<String, Queue> currentChildQueues = getQueues(childQueues);
+ Map<String, Queue> newChildQueues = getQueues(parentQueue.childQueues);
+ for (Map.Entry<String, Queue> e : newChildQueues.entrySet()) {
+ String newChildQueueName = e.getKey();
+ Queue newChildQueue = e.getValue();
+
+ Queue childQueue = currentChildQueues.get(newChildQueueName);
+ if (childQueue != null){
+ childQueue.reinitialize(newChildQueue, clusterResource);
+ LOG.info(getQueueName() + ": re-configured queue: " + childQueue);
+ } else {
+ currentChildQueues.put(newChildQueueName, newChildQueue);
+ LOG.info(getQueueName() + ": added new child queue: " + newChildQueue);
+ }
+ }
+
+ // Re-sort all queues
+ childQueues.clear();
+ childQueues.addAll(currentChildQueues.values());
+
+ // Set new configs
+ setupQueueConfigs(parentQueue.capacity, parentQueue.absoluteCapacity,
+ parentQueue.maximumCapacity, parentQueue.absoluteMaxCapacity);
+
+ // Update
+ update(clusterResource);
+ }
+
+ Map<String, Queue> getQueues(Set<Queue> queues) {
+ Map<String, Queue> queuesMap = new HashMap<String, Queue>();
+ for (Queue queue : queues) {
+ queuesMap.put(queue.getQueueName(), queue);
+ }
+ return queuesMap;
+ }
+
+ @Override
public void submitApplication(Application application, String user,
String queue, Priority priority)
throws AccessControlException {
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/Queue.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/Queue.java?rev=1094915&r1=1094914&r2=1094915&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/Queue.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/Queue.java Tue Apr 19 07:50:39 2011
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+import java.io.IOException;
import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -160,4 +161,13 @@ extends org.apache.hadoop.yarn.server.re
* @return number of applications
*/
public int getNumApplications();
+
+
+ /**
+ * Reinitialize the queue.
+ * @param queue new queue to re-initalize from
+ * @param clusterResource resources in the cluster
+ */
+ public void reinitialize(Queue queue, Resource clusterResource)
+ throws IOException;
}
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java?rev=1094915&r1=1094914&r2=1094915&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java Tue Apr 19 07:50:39 2011
@@ -103,16 +103,10 @@ public class FifoScheduler implements Re
public FifoScheduler() {}
- public FifoScheduler(Configuration conf,
- ContainerTokenSecretManager containerTokenSecretManager)
- {
- reinitialize(conf, containerTokenSecretManager);
- }
-
-
@Override
public void reinitialize(Configuration conf,
ContainerTokenSecretManager containerTokenSecretManager)
+ throws IOException
{
this.conf = conf;
this.containerTokenSecretManager = containerTokenSecretManager;
Added: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/tools/RMAdmin.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/tools/RMAdmin.java?rev=1094915&view=auto
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/tools/RMAdmin.java (added)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/tools/RMAdmin.java Tue Apr 19 07:50:39 2011
@@ -0,0 +1,183 @@
+package org.apache.hadoop.yarn.server.resourcemanager.tools;
+
+import java.io.IOException;
+import java.security.PrivilegedAction;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.SecurityInfo;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.factories.RecordFactory;
+import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.security.admin.AdminSecurityInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.api.RMAdminProtocol;
+import org.apache.hadoop.yarn.server.resourcemanager.api.protocolrecords.RefreshQueuesRequest;
+
+public class RMAdmin extends Configured implements Tool {
+
+ private final RecordFactory recordFactory =
+ RecordFactoryProvider.getRecordFactory(null);
+
+ public RMAdmin() {
+ super();
+ }
+
+ public RMAdmin(Configuration conf) {
+ super(conf);
+ }
+
+ private static void printHelp(String cmd) {
+ String summary = "rmadmin is the command to execute Map-Reduce administrative commands.\n" +
+ "The full syntax is: \n\n" +
+ "hadoop rmadmin [-refreshQueues] " +
+ "[-help [cmd]]\n";
+
+ String refreshQueues =
+ "-refreshQueues: Reload the queues' acls, states and "
+ + "scheduler specific properties.\n"
+ + "\t\tJobTracker will reload the mapred-queues configuration file.\n";
+
+ String help = "-help [cmd]: \tDisplays help for the given command or all commands if none\n" +
+ "\t\tis specified.\n";
+
+ if ("refreshQueues".equals(cmd)) {
+ System.out.println(refreshQueues);
+ } else if ("help".equals(cmd)) {
+ System.out.println(help);
+ } else {
+ System.out.println(summary);
+ System.out.println(refreshQueues);
+ System.out.println(help);
+ System.out.println();
+ ToolRunner.printGenericCommandUsage(System.out);
+ }
+ }
+
+ /**
+ * Displays format of commands.
+ * @param cmd The command that is being executed.
+ */
+ private static void printUsage(String cmd) {
+ if ("-refreshQueues".equals(cmd)) {
+ System.err.println("Usage: java RMAdmin" + " [-refreshQueues]");
+ } else {
+ System.err.println("Usage: java RMAdmin");
+ System.err.println(" [-refreshQueues]");
+ System.err.println(" [-help [cmd]]");
+ System.err.println();
+ ToolRunner.printGenericCommandUsage(System.err);
+ }
+ }
+
+ private static UserGroupInformation getUGI(Configuration conf
+ ) throws IOException {
+ return UserGroupInformation.getCurrentUser();
+ }
+
+ private int refreshQueues() throws IOException {
+ // Get the current configuration
+ final YarnConfiguration conf = new YarnConfiguration(getConf());
+
+ // Create the client
+ final String adminAddress =
+ conf.get(YarnConfiguration.ADMIN_ADDRESS,
+ YarnConfiguration.DEFAULT_ADMIN_BIND_ADDRESS);
+ final YarnRPC rpc = YarnRPC.create(conf);
+
+ if (UserGroupInformation.isSecurityEnabled()) {
+ conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_INFO_CLASS_NAME,
+ AdminSecurityInfo.class, SecurityInfo.class);
+ }
+
+ RMAdminProtocol adminProtocol =
+ getUGI(conf).doAs(new PrivilegedAction<RMAdminProtocol>() {
+ @Override
+ public RMAdminProtocol run() {
+ return (RMAdminProtocol) rpc.getProxy(RMAdminProtocol.class,
+ NetUtils.createSocketAddr(adminAddress), conf);
+ }
+ });
+
+ // Refresh the queue properties
+ RefreshQueuesRequest request =
+ recordFactory.newRecordInstance(RefreshQueuesRequest.class);
+ adminProtocol.refreshQueues(request);
+
+ return 0;
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ if (args.length < 1) {
+ printUsage("");
+ return -1;
+ }
+
+ int exitCode = -1;
+ int i = 0;
+ String cmd = args[i++];
+ //
+ // verify that we have enough command line parameters
+ //
+ if ("-refreshQueues".equals(cmd)) {
+ if (args.length != 1) {
+ printUsage(cmd);
+ return exitCode;
+ }
+ }
+
+ exitCode = 0;
+ try {
+ if ("-refreshQueues".equals(cmd)) {
+ exitCode = refreshQueues();
+ } else if ("-help".equals(cmd)) {
+ if (i < args.length) {
+ printUsage(args[i]);
+ } else {
+ printHelp("");
+ }
+ } else {
+ exitCode = -1;
+ System.err.println(cmd.substring(1) + ": Unknown command");
+ printUsage("");
+ printUsage("");
+ }
+
+ } catch (IllegalArgumentException arge) {
+ exitCode = -1;
+ System.err.println(cmd.substring(1) + ": " + arge.getLocalizedMessage());
+ printUsage(cmd);
+ } catch (RemoteException e) {
+ //
+ // This is a error returned by hadoop server. Print
+ // out the first line of the error mesage, ignore the stack trace.
+ exitCode = -1;
+ try {
+ String[] content;
+ content = e.getLocalizedMessage().split("\n");
+ System.err.println(cmd.substring(1) + ": "
+ + content[0]);
+ } catch (Exception ex) {
+ System.err.println(cmd.substring(1) + ": "
+ + ex.getLocalizedMessage());
+ }
+ } catch (Exception e) {
+ exitCode = -1;
+ System.err.println(cmd.substring(1) + ": "
+ + e.getLocalizedMessage());
+ }
+ return exitCode;
+ }
+
+ public static void main(String[] args) throws Exception {
+ int result = ToolRunner.run(new RMAdmin(), args);
+ System.exit(result);
+ }
+}
Added: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/proto/RMAdminProtocol.proto
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/proto/RMAdminProtocol.proto?rev=1094915&view=auto
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/proto/RMAdminProtocol.proto (added)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/proto/RMAdminProtocol.proto Tue Apr 19 07:50:39 2011
@@ -0,0 +1,10 @@
+option java_package = "org.apache.hadoop.yarn.proto";
+option java_outer_classname = "RMAdminProtocol";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+
+import "yarn_server_resourcemanager_service_protos.proto";
+
+service RMAdminProtocolService {
+ rpc refreshQueues(RefreshQueuesRequestProto) returns (RefreshQueuesResponseProto);
+}
Added: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_service_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_service_protos.proto?rev=1094915&view=auto
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_service_protos.proto (added)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_service_protos.proto Tue Apr 19 07:50:39 2011
@@ -0,0 +1,10 @@
+option java_package = "org.apache.hadoop.yarn.proto";
+option java_outer_classname = "YarnServerResourceManagerServiceProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+
+
+message RefreshQueuesRequestProto {
+}
+message RefreshQueuesResponseProto {
+}
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java?rev=1094915&r1=1094914&r2=1094915&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java Tue Apr 19 07:50:39 2011
@@ -27,8 +27,6 @@ import org.apache.hadoop.net.NetworkTopo
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.factories.RecordFactory;
-import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.resourcemanager.Application;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.Task;
@@ -45,7 +43,6 @@ import junit.framework.TestCase;
public class TestCapacityScheduler extends TestCase {
private static final Log LOG = LogFactory.getLog(TestCapacityScheduler.class);
- private static final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
private ResourceManager resourceManager = null;
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java?rev=1094915&r1=1094914&r2=1094915&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java Tue Apr 19 07:50:39 2011
@@ -29,7 +29,7 @@ public class TestQueueParsing {
private static final Log LOG = LogFactory.getLog(TestQueueParsing.class);
@Test
- public void testQueueParsing() {
+ public void testQueueParsing() throws Exception {
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
setupQueueConfiguration(conf);
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java?rev=1094915&r1=1094914&r2=1094915&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java Tue Apr 19 07:50:39 2011
@@ -26,8 +26,6 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.net.NetworkTopology;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.factories.RecordFactory;
-import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.resourcemanager.Application;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.Task;
@@ -41,7 +39,6 @@ import junit.framework.TestCase;
public class TestFifoScheduler extends TestCase {
private static final Log LOG = LogFactory.getLog(TestFifoScheduler.class);
- private static final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
private ResourceManager resourceManager = null;
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java?rev=1094915&r1=1094914&r2=1094915&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java Tue Apr 19 07:50:39 2011
@@ -136,7 +136,8 @@ public class TestRMWebApp {
}
public static ResourceManager mockRm(int apps, int racks, int nodes,
- int mbsPerNode) {
+ int mbsPerNode)
+ throws Exception {
ResourceManager rm = mock(ResourceManager.class);
ApplicationsManager asm = mockAsm(apps);
RMResourceTrackerImpl rt = mockResource(racks, nodes, mbsPerNode);
@@ -147,7 +148,7 @@ public class TestRMWebApp {
return rm;
}
- public static CapacityScheduler mockCS() {
+ public static CapacityScheduler mockCS() throws Exception {
// stolen from TestCapacityScheduler
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
setupQueueConfiguration(conf);
@@ -206,7 +207,7 @@ public class TestRMWebApp {
conf.setCapacity(C13, 40);
}
- public static void main(String[] args) {
+ public static void main(String[] args) throws Exception {
// For manual testing
WebApps.$for("yarn", new TestRMWebApp()).at(8888).inDevMode().
start(new RMWebApp(mockRm(42, 8, 8, 8*GiB))).joinThread();
|