http://git-wip-us.apache.org/repos/asf/helix/blob/713586c4/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/YarnProvisioner.java
----------------------------------------------------------------------
diff --cc helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/YarnProvisioner.java
index 0000000,ce6b1bc..a3bdb0f
mode 000000,100644..100644
--- a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/YarnProvisioner.java
+++ b/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/YarnProvisioner.java
@@@ -1,0 -1,392 +1,411 @@@
+ package org.apache.helix.provisioning.yarn;
+
++/*
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements. See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership. The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing,
++ * software distributed under the License is distributed on an
++ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
++ * KIND, either express or implied. See the License for the
++ * specific language governing permissions and limitations
++ * under the License.
++ */
++
+ import java.io.File;
+ import java.util.ArrayList;
+ import java.util.Collection;
+ import java.util.HashMap;
+ import java.util.HashSet;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.Set;
+ import java.util.Vector;
+ import java.util.concurrent.Executors;
+
+ import org.apache.commons.logging.Log;
+ import org.apache.commons.logging.LogFactory;
+ import org.apache.hadoop.fs.FileStatus;
+ import org.apache.hadoop.fs.FileSystem;
+ import org.apache.hadoop.fs.Path;
+ import org.apache.hadoop.yarn.api.ApplicationConstants;
+ import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
+ import org.apache.hadoop.yarn.api.records.Container;
+ import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+ import org.apache.hadoop.yarn.api.records.LocalResource;
+ import org.apache.hadoop.yarn.api.records.LocalResourceType;
+ import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+ import org.apache.hadoop.yarn.api.records.Priority;
+ import org.apache.hadoop.yarn.api.records.Resource;
+ import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
+ import org.apache.hadoop.yarn.conf.YarnConfiguration;
+ import org.apache.hadoop.yarn.util.ConverterUtils;
+ import org.apache.hadoop.yarn.util.Records;
+ import org.apache.helix.HelixManager;
+ import org.apache.helix.api.Cluster;
+ import org.apache.helix.api.Participant;
+ import org.apache.helix.api.config.ContainerConfig;
+ import org.apache.helix.api.config.ResourceConfig;
+ import org.apache.helix.api.id.ParticipantId;
+ import org.apache.helix.api.id.ResourceId;
+ import org.apache.helix.controller.provisioner.ContainerId;
+ import org.apache.helix.controller.provisioner.ContainerProvider;
+ import org.apache.helix.controller.provisioner.ContainerSpec;
+ import org.apache.helix.controller.provisioner.ContainerState;
+ import org.apache.helix.controller.provisioner.Provisioner;
+ import org.apache.helix.controller.provisioner.TargetProvider;
+ import org.apache.helix.controller.provisioner.TargetProviderResponse;
+ import org.apache.helix.model.InstanceConfig;
+ import org.apache.helix.provisioning.ApplicationSpec;
+ import org.apache.helix.provisioning.ContainerAskResponse;
+ import org.apache.helix.provisioning.ContainerLaunchResponse;
+ import org.apache.helix.provisioning.ContainerReleaseResponse;
+ import org.apache.helix.provisioning.ContainerStopResponse;
+ import org.apache.helix.provisioning.ParticipantLauncher;
+
+ import com.google.common.base.Function;
+ import com.google.common.collect.Lists;
+ import com.google.common.collect.Maps;
+ import com.google.common.util.concurrent.Futures;
+ import com.google.common.util.concurrent.ListenableFuture;
+ import com.google.common.util.concurrent.ListeningExecutorService;
+ import com.google.common.util.concurrent.MoreExecutors;
+
+ public class YarnProvisioner implements Provisioner, TargetProvider, ContainerProvider {
+
+ private static final Log LOG = LogFactory.getLog(YarnProvisioner.class);
+ static GenericApplicationMaster applicationMaster;
+ static ListeningExecutorService service = MoreExecutors.listeningDecorator(Executors
+ .newCachedThreadPool());
+ public static AppMasterConfig applicationMasterConfig;
+ public static ApplicationSpec applicationSpec;
+ Map<ContainerId, Container> allocatedContainersMap = new HashMap<ContainerId, Container>();
+ private HelixManager _helixManager;
+ private ResourceConfig _resourceConfig;
+
+ public YarnProvisioner() {
+
+ }
+
+ @Override
+ public ListenableFuture<ContainerId> allocateContainer(ContainerSpec spec) {
+ ContainerRequest containerAsk = setupContainerAskForRM(spec);
+ ListenableFuture<ContainerAskResponse> requestNewContainer =
+ applicationMaster.acquireContainer(containerAsk);
+ return Futures.transform(requestNewContainer,
+ new Function<ContainerAskResponse, ContainerId>() {
+ @Override
+ public ContainerId apply(ContainerAskResponse containerAskResponse) {
+ ContainerId helixContainerId =
+ ContainerId.from(containerAskResponse.getContainer().getId().toString());
+ allocatedContainersMap.put(helixContainerId, containerAskResponse.getContainer());
+ return helixContainerId;
+ }
+ });
+
+ }
+
+ @Override
+ public ListenableFuture<Boolean> deallocateContainer(final ContainerId containerId) {
+ ListenableFuture<ContainerReleaseResponse> releaseContainer =
+ applicationMaster.releaseContainer(allocatedContainersMap.get(containerId));
+ return Futures.transform(releaseContainer, new Function<ContainerReleaseResponse, Boolean>() {
+ @Override
+ public Boolean apply(ContainerReleaseResponse response) {
+ return response != null;
+ }
+ }, service);
+
+ }
+
+ @Override
+ public ListenableFuture<Boolean> startContainer(final ContainerId containerId,
+ Participant participant) {
+ Container container = allocatedContainersMap.get(containerId);
+ ContainerLaunchContext launchContext;
+ try {
+ launchContext = createLaunchContext(containerId, container, participant);
+ } catch (Exception e) {
+ LOG.error("Exception while creating context to launch container:" + containerId, e);
+ return null;
+ }
+ ListenableFuture<ContainerLaunchResponse> future =
+ applicationMaster.launchContainer(container, launchContext);
+ return Futures.transform(future, new Function<ContainerLaunchResponse, Boolean>() {
+ @Override
+ public Boolean apply(ContainerLaunchResponse response) {
+ return response != null;
+ }
+ }, service);
+ }
+
+ private ContainerLaunchContext createLaunchContext(ContainerId containerId, Container container,
+ Participant participant) throws Exception {
+
+ ContainerLaunchContext participantContainer = Records.newRecord(ContainerLaunchContext.class);
+
+ // Map<String, String> envs = System.getenv();
+ String appName = applicationMasterConfig.getAppName();
+ int appId = applicationMasterConfig.getAppId();
+ String serviceName = _resourceConfig.getId().stringify();
+ String serviceClasspath = applicationMasterConfig.getClassPath(serviceName);
+ String mainClass = applicationMasterConfig.getMainClass(serviceName);
+ String zkAddress = applicationMasterConfig.getZKAddress();
+
+ // set the localresources needed to launch container
+ Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
+
+ LocalResource servicePackageResource = Records.newRecord(LocalResource.class);
+ YarnConfiguration conf = new YarnConfiguration();
+ FileSystem fs;
+ fs = FileSystem.get(conf);
+ String pathSuffix = appName + "/" + appId + "/" + serviceName + ".tar";
+ Path dst = new Path(fs.getHomeDirectory(), pathSuffix);
+ FileStatus destStatus = fs.getFileStatus(dst);
+
+ // Set the type of resource - file or archive
+ // archives are untarred at destination
+ // we don't need the jar file to be untarred for now
+ servicePackageResource.setType(LocalResourceType.ARCHIVE);
+ // Set visibility of the resource
+ // Setting to most private option
+ servicePackageResource.setVisibility(LocalResourceVisibility.APPLICATION);
+ // Set the resource to be copied over
+ servicePackageResource.setResource(ConverterUtils.getYarnUrlFromPath(dst));
+ // Set timestamp and length of file so that the framework
+ // can do basic sanity checks for the local resource
+ // after it has been copied over to ensure it is the same
+ // resource the client intended to use with the application
+ servicePackageResource.setTimestamp(destStatus.getModificationTime());
+ servicePackageResource.setSize(destStatus.getLen());
+ LOG.info("Setting local resource:" + servicePackageResource + " for service" + serviceName);
+ localResources.put(serviceName, servicePackageResource);
+
+ // Set local resource info into app master container launch context
+ participantContainer.setLocalResources(localResources);
+
+ // Set the necessary security tokens as needed
+ // amContainer.setContainerTokens(containerToken);
+
+ // Set the env variables to be setup in the env where the application master will be run
+ LOG.info("Set the environment for the application master");
+ Map<String, String> env = new HashMap<String, String>();
+ env.put(serviceName, dst.getName());
+ // Add AppMaster.jar location to classpath
+ // At some point we should not be required to add
+ // the hadoop specific classpaths to the env.
+ // It should be provided out of the box.
+ // For now setting all required classpaths including
+ // the classpath to "." for the application jar
+ StringBuilder classPathEnv =
+ new StringBuilder(Environment.CLASSPATH.$()).append(File.pathSeparatorChar).append("./*");
+ classPathEnv.append(File.pathSeparatorChar);
+ classPathEnv.append(serviceClasspath);
+ for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
+ YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
+ classPathEnv.append(File.pathSeparatorChar);
+ classPathEnv.append(c.trim());
+ }
+ classPathEnv.append(File.pathSeparatorChar).append("./log4j.properties");
+ LOG.info("Setting classpath for service:\n" + classPathEnv.toString());
+ env.put("CLASSPATH", classPathEnv.toString());
+
+ participantContainer.setEnvironment(env);
+
+ if (applicationMaster.allTokens != null) {
+ LOG.info("Setting tokens: " + applicationMaster.allTokens);
+ participantContainer.setTokens(applicationMaster.allTokens);
+ }
+
+ // Set the necessary command to execute the application master
+ Vector<CharSequence> vargs = new Vector<CharSequence>(30);
+
+ // Set java executable command
+ LOG.info("Setting up app master command");
+ vargs.add(Environment.JAVA_HOME.$() + "/bin/java");
+ // Set Xmx based on am memory size
+ vargs.add("-Xmx" + 1024 + "m");
+ // Set class name
+ vargs.add(ParticipantLauncher.class.getCanonicalName());
+ // Set params for container participant
+ vargs.add("--zkAddress " + zkAddress);
+ vargs.add("--cluster " + appName);
+ vargs.add("--participantId " + participant.getId().stringify());
+ vargs.add("--participantClass " + mainClass);
+
+ vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/ContainerParticipant.stdout");
+ vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/ContainerParticipant.stderr");
+
+ // Get final commmand
+ StringBuilder command = new StringBuilder();
+ for (CharSequence str : vargs) {
+ command.append(str).append(" ");
+ }
+
+ LOG.info("Completed setting up container launch command " + command.toString()
+ + " with arguments \n" + vargs);
+ List<String> commands = new ArrayList<String>();
+ commands.add(command.toString());
+ participantContainer.setCommands(commands);
+ return participantContainer;
+ }
+
+ @Override
+ public ListenableFuture<Boolean> stopContainer(final ContainerId containerId) {
+ Container container = allocatedContainersMap.get(containerId);
+ ListenableFuture<ContainerStopResponse> future = applicationMaster.stopContainer(container);
+ return Futures.transform(future, new Function<ContainerStopResponse, Boolean>() {
+ @Override
+ public Boolean apply(ContainerStopResponse response) {
+ return response != null;
+ }
+ }, service);
+ }
+
+ @Override
+ public void init(HelixManager helixManager, ResourceConfig resourceConfig) {
+ _helixManager = helixManager;
+ _resourceConfig = resourceConfig;
+ }
+
+ @Override
+ public TargetProviderResponse evaluateExistingContainers(Cluster cluster, ResourceId resourceId,
+ Collection<Participant> participants) {
+ TargetProviderResponse response = new TargetProviderResponse();
+ // ask for two containers at a time
+ List<ContainerSpec> containersToAcquire = Lists.newArrayList();
+ List<Participant> containersToStart = Lists.newArrayList();
+ List<Participant> containersToRelease = Lists.newArrayList();
+ List<Participant> containersToStop = Lists.newArrayList();
+ YarnProvisionerConfig provisionerConfig =
+ (YarnProvisionerConfig) cluster.getConfig().getResourceMap().get(resourceId)
+ .getProvisionerConfig();
+ int targetNumContainers = provisionerConfig.getNumContainers();
+
+ // Any container that is in a state should be put in this set
+ Set<ParticipantId> existingContainersIdSet = new HashSet<ParticipantId>();
+
+ // Cache halted containers to determine which to restart and which to release
+ Map<ParticipantId, Participant> excessHaltedContainers = Maps.newHashMap();
+
+ // Cache participants to ensure that excess participants are stopped
+ Map<ParticipantId, Participant> excessActiveContainers = Maps.newHashMap();
+
+ for (Participant participant : participants) {
+ ContainerConfig containerConfig = participant.getContainerConfig();
+ if (containerConfig != null && containerConfig.getState() != null) {
+ ContainerState state = containerConfig.getState();
+ switch (state) {
+ case ACQUIRING:
+ existingContainersIdSet.add(participant.getId());
+ break;
+ case ACQUIRED:
+ // acquired containers are ready to start
+ existingContainersIdSet.add(participant.getId());
+ containersToStart.add(participant);
+ break;
+ case CONNECTING:
+ existingContainersIdSet.add(participant.getId());
+ break;
+ case CONNECTED:
+ // active containers can be stopped or kept active
+ existingContainersIdSet.add(participant.getId());
+ excessActiveContainers.put(participant.getId(), participant);
+ break;
+ case DISCONNECTED:
+ // disconnected containers must be stopped
+ existingContainersIdSet.add(participant.getId());
+ containersToStop.add(participant);
+ case HALTING:
+ existingContainersIdSet.add(participant.getId());
+ break;
+ case HALTED:
+ // halted containers can be released or restarted
+ existingContainersIdSet.add(participant.getId());
+ excessHaltedContainers.put(participant.getId(), participant);
+ break;
+ case FINALIZING:
+ existingContainersIdSet.add(participant.getId());
+ break;
+ case FINALIZED:
+ break;
+ case FAILED:
+ // remove the failed instance
+ _helixManager.getClusterManagmentTool().dropInstance(cluster.getId().toString(),
+ new InstanceConfig(participant.getId()));
+ break;
+ default:
+ break;
+ }
+ }
+ }
+
+ for (int i = 0; i < targetNumContainers; i++) {
+ ParticipantId participantId = ParticipantId.from(resourceId + "_container_" + (i));
+ excessActiveContainers.remove(participantId); // don't stop this container if active
+ if (excessHaltedContainers.containsKey(participantId)) {
+ // Halted containers can be restarted if necessary
+ // Participant participant = excessHaltedContainers.get(participantId);
+ // containersToStart.add(participant);
+ // excessHaltedContainers.remove(participantId); // don't release this container
+ } else if (!existingContainersIdSet.contains(participantId)) {
+ // Unallocated containers must be allocated
+ ContainerSpec containerSpec = new ContainerSpec(participantId);
+ containerSpec.setMemory(_resourceConfig.getUserConfig().getIntField("memory", 1024));
+ containersToAcquire.add(containerSpec);
+ }
+ }
+
+ // Add all the containers that should be stopped because they fall outside the target range
+ containersToStop.addAll(excessActiveContainers.values());
+
+ // Add halted containers that should not be restarted
+ containersToRelease.addAll(excessHaltedContainers.values());
+
+ response.setContainersToAcquire(containersToAcquire);
+ response.setContainersToStart(containersToStart);
+ response.setContainersToRelease(containersToRelease);
+ response.setContainersToStop(containersToStop);
+ LOG.info("target provider response containers to acquire:" + response.getContainersToAcquire());
+ LOG.info("target provider response containers to start:" + response.getContainersToStart());
+ return response;
+ }
+
+ private ContainerRequest setupContainerAskForRM(ContainerSpec spec) {
+ // setup requirements for hosts
+ // using * as any host will do for the distributed shell app
+ // set the priority for the request
+ Priority pri = Records.newRecord(Priority.class);
+ int requestPriority = 0;
+ // TODO - what is the range for priority? how to decide?
+ pri.setPriority(requestPriority);
+
+ // Set up resource type requirements
+ // For now, only memory is supported so we set memory requirements
+ Resource capability = Records.newRecord(Resource.class);
+ int memory = spec.getMemory();
+ capability.setMemory(memory);
+
+ ContainerRequest request = new ContainerRequest(capability, null, null, pri);
+ LOG.info("Requested container ask: " + request.toString());
+ return request;
+ }
+
+ @Override
+ public ContainerProvider getContainerProvider() {
+ return this;
+ }
+
+ @Override
+ public TargetProvider getTargetProvider() {
+ return this;
+ }
+
+ }
http://git-wip-us.apache.org/repos/asf/helix/blob/713586c4/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/YarnProvisionerConfig.java
----------------------------------------------------------------------
diff --cc helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/YarnProvisionerConfig.java
index 0000000,67dd679..69aa2fb
mode 000000,100644..100644
--- a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/YarnProvisionerConfig.java
+++ b/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/YarnProvisionerConfig.java
@@@ -1,0 -1,54 +1,73 @@@
+ package org.apache.helix.provisioning.yarn;
+
++/*
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements. See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership. The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing,
++ * software distributed under the License is distributed on an
++ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
++ * KIND, either express or implied. See the License for the
++ * specific language governing permissions and limitations
++ * under the License.
++ */
++
+ import org.apache.helix.api.id.ResourceId;
+ import org.apache.helix.controller.provisioner.ProvisionerConfig;
+ import org.apache.helix.controller.provisioner.ProvisionerRef;
+ import org.apache.helix.controller.serializer.DefaultStringSerializer;
+ import org.apache.helix.controller.serializer.StringSerializer;
+ import org.codehaus.jackson.annotate.JsonProperty;
+
+ public class YarnProvisionerConfig implements ProvisionerConfig {
+
+ private ResourceId _resourceId;
+ private Class<? extends StringSerializer> _serializerClass;
+ private ProvisionerRef _provisionerRef;
+ private Integer _numContainers;
+
+ public YarnProvisionerConfig(@JsonProperty("resourceId") ResourceId resourceId) {
+ _resourceId = resourceId;
+ _serializerClass = DefaultStringSerializer.class;
+ _provisionerRef = ProvisionerRef.from(YarnProvisioner.class.getName());
+ }
+
+ public void setNumContainers(int numContainers) {
+ _numContainers = numContainers;
+ }
+
+ public Integer getNumContainers() {
+ return _numContainers;
+ }
+
+ @Override
+ public ResourceId getResourceId() {
+ return _resourceId;
+ }
+
+ @Override
+ public ProvisionerRef getProvisionerRef() {
+ return _provisionerRef;
+ }
+
+ public void setProvisionerRef(ProvisionerRef provisionerRef) {
+ _provisionerRef = provisionerRef;
+ }
+
+ @Override
+ public Class<? extends StringSerializer> getSerializerClass() {
+ return _serializerClass;
+ }
+
+ public void setSerializerClass(Class<? extends StringSerializer> serializerClass) {
+ _serializerClass = serializerClass;
+ }
+
+ }
http://git-wip-us.apache.org/repos/asf/helix/blob/713586c4/helix-provisioning/src/main/resources/sample_application.yaml
----------------------------------------------------------------------
diff --cc helix-provisioning/src/main/resources/sample_application.yaml
index 0000000,f45faa3..5d6d88f
mode 000000,100644..100644
--- a/helix-provisioning/src/main/resources/sample_application.yaml
+++ b/helix-provisioning/src/main/resources/sample_application.yaml
@@@ -1,0 -1,26 +1,42 @@@
-===
++#
++# Licensed to the Apache Software Foundation (ASF) under one
++# or more contributor license agreements. See the NOTICE file
++# distributed with this work for additional information
++# regarding copyright ownership. The ASF licenses this file
++# to you under the Apache License, Version 2.0 (the
++# "License"); you may not use this file except in compliance
++# with the License. You may obtain a copy of the License at
++#
++# http://www.apache.org/licenses/LICENSE-2.0
++#
++# Unless required by applicable law or agreed to in writing,
++# software distributed under the License is distributed on an
++# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
++# KIND, either express or implied. See the License for the
++# specific language governing permissions and limitations
++# under the License.
++#
+ appName: test
+ configs:
+ k1: v1
+ services:
+ - name: myservice
- participantClass: org.apache.helix.myApp.SimpleWebserver
++ participantClass: org.apache.helix.myApp.SimpleWebserver
+ minContainers:3
- maxContainers:3
++ maxContainers:3
+ configs:
- - participantId: myservice_0
++ - participantId: myservice_0
+ port: 9500
- - participantId: myservice_1
- port: 9501
- - participantId: myservice_2
++ - participantId: myservice_1
++ port: 9501
++ - participantId: myservice_2
+ port: 9502
+ resources:
+ - name: distributedLock
+ numPartitions: 6
+ numReplicas: 2
+ rebalanceMode: FULL_AUTO
+ stateModel: OnlineOffline
- tag: myservice
- configs:
++ tag: myservice
++ configs:
+ k1: v1
-
http://git-wip-us.apache.org/repos/asf/helix/blob/713586c4/pom.xml
----------------------------------------------------------------------
diff --cc pom.xml
index 2675d45,f3d2e61..acb8589
--- a/pom.xml
+++ b/pom.xml
@@@ -197,9 -197,10 +197,10 @@@ under the License
<module>helix-core</module>
<module>helix-admin-webapp</module>
<module>helix-agent</module>
+ <module>helix-provisioning</module>
<module>helix-examples</module>
<module>recipes</module>
- <module>site-releases</module>
+ <module>website</module>
</modules>
<mailingLists>
@@@ -342,6 -352,6 +343,11 @@@
<version>${project.version}</version>
</dependency>
<dependency>
++ <groupId>org.apache.helix</groupId>
++ <artifactId>helix-provisioning</artifactId>
++ <version>${project.version}</version>
++ </dependency>
++ <dependency>
<groupId>org.apache.helix</groupId>
<artifactId>helix-core</artifactId>
<type>test-jar</type>
http://git-wip-us.apache.org/repos/asf/helix/blob/713586c4/recipes/helloworld-provisioning-yarn/pom.xml
----------------------------------------------------------------------
diff --cc recipes/helloworld-provisioning-yarn/pom.xml
index 0000000,4cef9a7..bf5a89c
mode 000000,100644..100644
--- a/recipes/helloworld-provisioning-yarn/pom.xml
+++ b/recipes/helloworld-provisioning-yarn/pom.xml
@@@ -1,0 -1,159 +1,158 @@@
+ <?xml version="1.0" encoding="UTF-8" ?>
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+ -->
+ <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.helix.recipes</groupId>
+ <artifactId>recipes</artifactId>
+ <version>0.7.1-incubating-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>helloworld-provisioning-yarn</artifactId>
+ <packaging>bundle</packaging>
+ <name>Apache Helix :: Recipes :: Provisioning :: YARN :: Hello World</name>
+
+ <properties>
+ <osgi.import>
+ org.apache.helix*,
+ org.apache.log4j,
+ *
+ </osgi.import>
+ <osgi.export>org.apache.helix.provisioning.yarn.example*;version="${project.version};-noimport:=true</osgi.export>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.testng</groupId>
+ <artifactId>testng</artifactId>
+ <version>6.0.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.helix</groupId>
+ <artifactId>helix-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.helix</groupId>
+ <artifactId>helix-provisioning</artifactId>
- <version>0.7.1-incubating-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.mail</groupId>
+ <artifactId>mail</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.jms</groupId>
+ <artifactId>jms</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jdmk</groupId>
+ <artifactId>jmxtools</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jmx</groupId>
+ <artifactId>jmxri</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+ <build>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>appassembler-maven-plugin</artifactId>
+ <configuration>
+ <!-- Set the target configuration directory to be used in the bin scripts -->
+ <!-- <configurationDirectory>conf</configurationDirectory> -->
+ <!-- Copy the contents from "/src/main/config" to the target configuration
+ directory in the assembled application -->
+ <!-- <copyConfigurationDirectory>true</copyConfigurationDirectory> -->
+ <!-- Include the target configuration directory in the beginning of
+ the classpath declaration in the bin scripts -->
+ <includeConfigurationDirectoryInClasspath>true</includeConfigurationDirectoryInClasspath>
+ <assembleDirectory>${project.build.directory}/${project.artifactId}-pkg</assembleDirectory>
+ <!-- Extra JVM arguments that will be included in the bin scripts -->
+ <extraJvmArguments>-Xms512m -Xmx512m</extraJvmArguments>
+ <!-- Generate bin scripts for windows and unix pr default -->
+ <platforms>
+ <platform>windows</platform>
+ <platform>unix</platform>
+ </platforms>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>assemble</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <excludes combine.children="append">
+ </excludes>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>appassembler-maven-plugin</artifactId>
+ <configuration>
+ <programs>
+ <program>
+ <mainClass>org.apache.helix.provisioning.yarn.Client</mainClass>
+ <name>yarn-job-launcher</name>
+ </program>
+ <program>
+ <mainClass>org.apache.helix.provisioning.yarn.AppLauncher</mainClass>
+ <name>app-launcher</name>
+ </program>
+ </programs>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <descriptors>
+ <descriptor>src/assemble/assembly.xml</descriptor>
+ </descriptors>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </project>
http://git-wip-us.apache.org/repos/asf/helix/blob/713586c4/recipes/helloworld-provisioning-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/HelloWordAppSpecFactory.java
----------------------------------------------------------------------
diff --cc recipes/helloworld-provisioning-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/HelloWordAppSpecFactory.java
index 0000000,e9163d3..40fed23
mode 000000,100644..100644
--- a/recipes/helloworld-provisioning-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/HelloWordAppSpecFactory.java
+++ b/recipes/helloworld-provisioning-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/HelloWordAppSpecFactory.java
@@@ -1,0 -1,36 +1,48 @@@
+ package org.apache.helix.provisioning.yarn.example;
+
-import java.io.File;
++/*
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements. See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership. The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing,
++ * software distributed under the License is distributed on an
++ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
++ * KIND, either express or implied. See the License for the
++ * specific language governing permissions and limitations
++ * under the License.
++ */
++
+ import java.io.InputStream;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
+
-import org.apache.helix.provisioning.AppConfig;
+ import org.apache.helix.provisioning.ApplicationSpec;
+ import org.apache.helix.provisioning.ApplicationSpecFactory;
-import org.apache.helix.provisioning.yarn.example.HelloWorldService;
+ import org.apache.helix.provisioning.yarn.example.HelloworldAppSpec;
-import org.yaml.snakeyaml.DumperOptions;
+ import org.yaml.snakeyaml.Yaml;
+
+ public class HelloWordAppSpecFactory implements ApplicationSpecFactory {
+
+ @Override
+ public ApplicationSpec fromYaml(InputStream inputstream) {
+ return (ApplicationSpec) new Yaml().load(inputstream);
+ // return data;
+ }
+
+ public static void main(String[] args) {
+
+ Yaml yaml = new Yaml();
+ InputStream resourceAsStream =
+ ClassLoader.getSystemClassLoader().getResourceAsStream("hello_world_app_spec.yaml");
+ HelloworldAppSpec spec = yaml.loadAs(resourceAsStream, HelloworldAppSpec.class);
+ String dump = yaml.dump(spec);
+ System.out.println(dump);
+ System.out.println(spec.getServiceConfig("HelloWorld").getStringField("num_containers", "1"));
+
+ }
+ }
http://git-wip-us.apache.org/repos/asf/helix/blob/713586c4/recipes/helloworld-provisioning-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/HelloWorldService.java
----------------------------------------------------------------------
diff --cc recipes/helloworld-provisioning-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/HelloWorldService.java
index 0000000,269ae0c..3d604eb
mode 000000,100644..100644
--- a/recipes/helloworld-provisioning-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/HelloWorldService.java
+++ b/recipes/helloworld-provisioning-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/HelloWorldService.java
@@@ -1,0 -1,37 +1,56 @@@
+ package org.apache.helix.provisioning.yarn.example;
+
++/*
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements. See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership. The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing,
++ * software distributed under the License is distributed on an
++ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
++ * KIND, either express or implied. See the License for the
++ * specific language governing permissions and limitations
++ * under the License.
++ */
++
+ import org.apache.helix.HelixConnection;
+ import org.apache.helix.api.id.ClusterId;
+ import org.apache.helix.api.id.ParticipantId;
+ import org.apache.helix.participant.AbstractParticipantService;
+ import org.apache.helix.provisioning.ServiceConfig;
+ import org.apache.helix.provisioning.participant.StatelessParticipantService;
+ import org.apache.log4j.Logger;
+
+ public class HelloWorldService extends StatelessParticipantService {
+
+ private static Logger LOG = Logger.getLogger(AbstractParticipantService.class);
+
+ static String SERVICE_NAME = "HelloWorld";
+
+ public HelloWorldService(HelixConnection connection, ClusterId clusterId,
+ ParticipantId participantId) {
+ super(connection, clusterId, participantId, SERVICE_NAME);
+ }
+
+ @Override
+ protected void init(ServiceConfig serviceConfig) {
+ LOG.info("Initialized service with config " + serviceConfig);
+ }
+
+ @Override
+ protected void goOnline() {
+ LOG.info("HelloWorld service is told to go online");
+ }
+
+ @Override
+ protected void goOffine() {
+ LOG.info("HelloWorld service is told to go offline");
+ }
+
+ }
http://git-wip-us.apache.org/repos/asf/helix/blob/713586c4/recipes/helloworld-provisioning-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/HelloworldAppSpec.java
----------------------------------------------------------------------
diff --cc recipes/helloworld-provisioning-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/HelloworldAppSpec.java
index 0000000,4fda91e..e2a63f2
mode 000000,100644..100644
--- a/recipes/helloworld-provisioning-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/HelloworldAppSpec.java
+++ b/recipes/helloworld-provisioning-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/HelloworldAppSpec.java
@@@ -1,0 -1,153 +1,167 @@@
+ package org.apache.helix.provisioning.yarn.example;
+
++/*
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements. See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership. The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing,
++ * software distributed under the License is distributed on an
++ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
++ * KIND, either express or implied. See the License for the
++ * specific language governing permissions and limitations
++ * under the License.
++ */
++
+ import java.net.URI;
+ import java.net.URISyntaxException;
+ import java.util.List;
+ import java.util.Map;
+
+ import org.apache.helix.api.Scope;
-import org.apache.helix.api.config.ParticipantConfig;
-import org.apache.helix.api.config.ResourceConfig;
-import org.apache.helix.api.config.ResourceConfig.Builder;
-import org.apache.helix.api.config.UserConfig;
-import org.apache.helix.api.id.ParticipantId;
+ import org.apache.helix.api.id.ResourceId;
+ import org.apache.helix.provisioning.AppConfig;
+ import org.apache.helix.provisioning.ApplicationSpec;
+ import org.apache.helix.provisioning.ServiceConfig;
+ import org.apache.helix.provisioning.TaskConfig;
+
+ import com.google.common.collect.Maps;
+
+ public class HelloworldAppSpec implements ApplicationSpec {
+
+ public String _appName;
+
+ public AppConfig _appConfig;
+
+ public List<String> _services;
+
+ private String _appMasterPackageUri;
+
+ private Map<String, String> _servicePackageURIMap;
+
+ private Map<String, String> _serviceMainClassMap;
+
+ private Map<String, ServiceConfig> _serviceConfigMap;
+
+ private List<TaskConfig> _taskConfigs;
+
+ public AppConfig getAppConfig() {
+ return _appConfig;
+ }
+
+ public void setAppConfig(AppConfig appConfig) {
+ _appConfig = appConfig;
+ }
+
+ public String getAppMasterPackageUri() {
+ return _appMasterPackageUri;
+ }
+
+ public void setAppMasterPackageUri(String appMasterPackageUri) {
+ _appMasterPackageUri = appMasterPackageUri;
+ }
+
+ public Map<String, String> getServicePackageURIMap() {
+ return _servicePackageURIMap;
+ }
+
+ public void setServicePackageURIMap(Map<String, String> servicePackageURIMap) {
+ _servicePackageURIMap = servicePackageURIMap;
+ }
+
+ public Map<String, String> getServiceMainClassMap() {
+ return _serviceMainClassMap;
+ }
+
+ public void setServiceMainClassMap(Map<String, String> serviceMainClassMap) {
+ _serviceMainClassMap = serviceMainClassMap;
+ }
+
+ public Map<String, Map<String, String>> getServiceConfigMap() {
+ Map<String,Map<String,String>> map = Maps.newHashMap();
+ for(String service:_serviceConfigMap.keySet()){
+ map.put(service, _serviceConfigMap.get(service).getSimpleFields());
+ }
+ return map;
+ }
+
+ public void setServiceConfigMap(Map<String, Map<String, Object>> map) {
+ _serviceConfigMap = Maps.newHashMap();
+
+ for(String service:map.keySet()){
+ ServiceConfig serviceConfig = new ServiceConfig(Scope.resource(ResourceId.from(service)));
+ Map<String, Object> simpleFields = map.get(service);
+ for(String key:simpleFields.keySet()){
+ serviceConfig.setSimpleField(key, simpleFields.get(key).toString());
+ }
+ _serviceConfigMap.put(service, serviceConfig);
+ }
+ }
+
+ public void setAppName(String appName) {
+ _appName = appName;
+ }
+
+ public void setServices(List<String> services) {
+ _services = services;
+ }
+
+ public void setTaskConfigs(List<TaskConfig> taskConfigs) {
+ _taskConfigs = taskConfigs;
+ }
+
+ @Override
+ public String getAppName() {
+ return _appName;
+ }
+
+ @Override
+ public AppConfig getConfig() {
+ return _appConfig;
+ }
+
+ @Override
+ public List<String> getServices() {
+ return _services;
+ }
+
+ @Override
+ public URI getAppMasterPackage() {
+ try {
+ return new URI(_appMasterPackageUri);
+ } catch (URISyntaxException e) {
+ return null;
+ }
+ }
+
+ @Override
+ public URI getServicePackage(String serviceName) {
+ try {
+ return new URI(_servicePackageURIMap.get(serviceName));
+ } catch (URISyntaxException e) {
+ return null;
+ }
+ }
+
+ @Override
+ public String getServiceMainClass(String service) {
+ return _serviceMainClassMap.get(service);
+ }
+
+ @Override
+ public ServiceConfig getServiceConfig(String serviceName) {
+ return _serviceConfigMap.get(serviceName);
+ }
+
+ @Override
+ public List<TaskConfig> getTaskConfigs() {
+ return _taskConfigs;
+ }
+
+ }
http://git-wip-us.apache.org/repos/asf/helix/blob/713586c4/recipes/helloworld-provisioning-yarn/src/main/resources/hello_world_app_spec.yaml
----------------------------------------------------------------------
diff --cc recipes/helloworld-provisioning-yarn/src/main/resources/hello_world_app_spec.yaml
index 0000000,baaddb5..761c97e
mode 000000,100755..100755
--- a/recipes/helloworld-provisioning-yarn/src/main/resources/hello_world_app_spec.yaml
+++ b/recipes/helloworld-provisioning-yarn/src/main/resources/hello_world_app_spec.yaml
@@@ -1,0 -1,24 +1,42 @@@
++#
++# Licensed to the Apache Software Foundation (ASF) under one
++# or more contributor license agreements. See the NOTICE file
++# distributed with this work for additional information
++# regarding copyright ownership. The ASF licenses this file
++# to you under the Apache License, Version 2.0 (the
++# "License"); you may not use this file except in compliance
++# with the License. You may obtain a copy of the License at
++#
++# http://www.apache.org/licenses/LICENSE-2.0
++#
++# Unless required by applicable law or agreed to in writing,
++# software distributed under the License is distributed on an
++# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
++# KIND, either express or implied. See the License for the
++# specific language governing permissions and limitations
++# under the License.
++#
+ !!org.apache.helix.provisioning.yarn.example.HelloworldAppSpec
+ appConfig:
+ config: {
+ k1: v1
+ }
+ appMasterPackageUri: 'file:///Users/kgopalak/Documents/projects/incubator-helix/recipes/helloworld-provisioning-yarn/target/helloworld-provisioning-yarn-0.7.1-incubating-SNAPSHOT-pkg.tar'
+ appName: testApp
+ serviceConfigMap:
+ HelloWorld: {
+ num_containers: 3,
+ memory: 1024
+ }
+ serviceMainClassMap: {
+ HelloWorld: org.apache.helix.provisioning.yarn.example.HelloWorldService
+ }
+ servicePackageURIMap: {
+ HelloWorld: 'file:///Users/kgopalak/Documents/projects/incubator-helix/recipes/helloworld-provisioning-yarn/target/helloworld-provisioning-yarn-0.7.1-incubating-SNAPSHOT-pkg.tar'
+ }
+ services: [
+ HelloWorld]
+ taskConfigs: null
+
+
+
http://git-wip-us.apache.org/repos/asf/helix/blob/713586c4/recipes/jobrunner-yarn/pom.xml
----------------------------------------------------------------------
diff --cc recipes/jobrunner-yarn/pom.xml
index 0000000,f067a56..434fd8d
mode 000000,100644..100644
--- a/recipes/jobrunner-yarn/pom.xml
+++ b/recipes/jobrunner-yarn/pom.xml
@@@ -1,0 -1,159 +1,158 @@@
+ <?xml version="1.0" encoding="UTF-8" ?>
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+ -->
+ <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.helix.recipes</groupId>
+ <artifactId>recipes</artifactId>
+ <version>0.7.1-incubating-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>jobrunner-yarn</artifactId>
+ <packaging>bundle</packaging>
+ <name>Apache Helix :: Recipes :: Provisioning :: YARN :: Job Runner</name>
+
+ <properties>
+ <osgi.import>
+ org.apache.helix*,
+ org.apache.log4j,
+ *
+ </osgi.import>
+ <osgi.export>org.apache.helix.provisioning.yarn.example*;version="${project.version};-noimport:=true</osgi.export>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.testng</groupId>
+ <artifactId>testng</artifactId>
+ <version>6.0.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.helix</groupId>
+ <artifactId>helix-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.helix</groupId>
+ <artifactId>helix-provisioning</artifactId>
- <version>0.7.1-incubating-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.mail</groupId>
+ <artifactId>mail</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.jms</groupId>
+ <artifactId>jms</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jdmk</groupId>
+ <artifactId>jmxtools</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jmx</groupId>
+ <artifactId>jmxri</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+ <build>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>appassembler-maven-plugin</artifactId>
+ <configuration>
+ <!-- Set the target configuration directory to be used in the bin scripts -->
+ <!-- <configurationDirectory>conf</configurationDirectory> -->
+ <!-- Copy the contents from "/src/main/config" to the target configuration
+ directory in the assembled application -->
+ <!-- <copyConfigurationDirectory>true</copyConfigurationDirectory> -->
+ <!-- Include the target configuration directory in the beginning of
+ the classpath declaration in the bin scripts -->
+ <includeConfigurationDirectoryInClasspath>true</includeConfigurationDirectoryInClasspath>
+ <assembleDirectory>${project.build.directory}/${project.artifactId}-pkg</assembleDirectory>
+ <!-- Extra JVM arguments that will be included in the bin scripts -->
+ <extraJvmArguments>-Xms512m -Xmx512m</extraJvmArguments>
+ <!-- Generate bin scripts for windows and unix pr default -->
+ <platforms>
+ <platform>windows</platform>
+ <platform>unix</platform>
+ </platforms>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>assemble</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <excludes combine.children="append">
+ </excludes>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>appassembler-maven-plugin</artifactId>
+ <configuration>
+ <programs>
+ <program>
+ <mainClass>org.apache.helix.provisioning.yarn.AppLauncher</mainClass>
+ <name>app-launcher</name>
+ </program>
+ <program>
+ <mainClass>org.apache.helix.provisioning.yarn.example.JobRunnerMain</mainClass>
+ <name>job-runner</name>
+ </program>
+ </programs>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <descriptors>
+ <descriptor>src/assemble/assembly.xml</descriptor>
+ </descriptors>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </project>
http://git-wip-us.apache.org/repos/asf/helix/blob/713586c4/recipes/jobrunner-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/JobRunnerMain.java
----------------------------------------------------------------------
diff --cc recipes/jobrunner-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/JobRunnerMain.java
index 0000000,78266cf..3b92f33
mode 000000,100644..100644
--- a/recipes/jobrunner-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/JobRunnerMain.java
+++ b/recipes/jobrunner-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/JobRunnerMain.java
@@@ -1,0 -1,132 +1,151 @@@
+ package org.apache.helix.provisioning.yarn.example;
+
++/*
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements. See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership. The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing,
++ * software distributed under the License is distributed on an
++ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
++ * KIND, either express or implied. See the License for the
++ * specific language governing permissions and limitations
++ * under the License.
++ */
++
+ import java.io.File;
+ import java.util.Collection;
+ import java.util.List;
+
+ import org.apache.commons.cli.CommandLine;
+ import org.apache.commons.cli.GnuParser;
+ import org.apache.commons.cli.Option;
+ import org.apache.commons.cli.Options;
+ import org.apache.helix.ClusterMessagingService;
+ import org.apache.helix.HelixConnection;
+ import org.apache.helix.HelixManager;
+ import org.apache.helix.HelixRole;
+ import org.apache.helix.InstanceType;
+ import org.apache.helix.api.Participant;
+ import org.apache.helix.api.RunningInstance;
+ import org.apache.helix.api.accessor.ClusterAccessor;
+ import org.apache.helix.api.config.ContainerConfig;
+ import org.apache.helix.api.id.ClusterId;
+ import org.apache.helix.api.id.Id;
+ import org.apache.helix.manager.zk.HelixConnectionAdaptor;
+ import org.apache.helix.provisioning.ApplicationSpec;
+ import org.apache.helix.provisioning.ApplicationSpecFactory;
+ import org.apache.helix.provisioning.HelixYarnUtil;
+ import org.apache.helix.provisioning.TaskConfig;
+ import org.apache.helix.provisioning.yarn.AppLauncher;
+ import org.apache.helix.task.TaskDriver;
+ import org.apache.helix.task.Workflow;
+
+ public class JobRunnerMain {
+ public static void main(String[] args) throws Exception {
+ Options opts = new Options();
+ opts.addOption(new Option("app_spec_provider", true,
+ "Application Spec Factory Class that will parse the app_config_spec file"));
+ opts.addOption(new Option("app_config_spec", true,
+ "YAML config file that provides the app specifications"));
+ CommandLine cliParser = new GnuParser().parse(opts, args);
+ String appSpecFactoryClass = cliParser.getOptionValue("app_spec_provider");
+ String yamlConfigFileName = cliParser.getOptionValue("app_config_spec");
+
+ ApplicationSpecFactory applicationSpecFactory =
+ HelixYarnUtil.createInstance(appSpecFactoryClass);
+ File yamlConfigFile = new File(yamlConfigFileName);
+ if (!yamlConfigFile.exists()) {
+ throw new IllegalArgumentException("YAML app_config_spec file: '" + yamlConfigFileName
+ + "' does not exist");
+ }
+ final AppLauncher launcher = new AppLauncher(applicationSpecFactory, yamlConfigFile);
+ launcher.launch();
+ Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
+
+ @Override
+ public void run() {
+ launcher.cleanup();
+ }
+ }));
+
+ final ApplicationSpec appSpec = launcher.getApplicationSpec();
+
+ // Repeatedly print status
+ final HelixConnection connection = launcher.pollForConnection();
+ final ClusterId clusterId = ClusterId.from(appSpec.getAppName());
+ // TODO: this is a hack -- TaskDriver should accept a connection instead of a manager
+ HelixManager manager = new HelixConnectionAdaptor(new HelixRole() {
+ @Override
+ public HelixConnection getConnection() {
+ return connection;
+ }
+
+ @Override
+ public ClusterId getClusterId() {
+ return clusterId;
+ }
+
+ @Override
+ public Id getId() {
+ return null;
+ }
+
+ @Override
+ public InstanceType getType() {
+ return InstanceType.ADMINISTRATOR;
+ }
+
+ @Override
+ public ClusterMessagingService getMessagingService() {
+ return null;
+ }
+ });
+
+ // Get all submitted jobs
+ String workflow = null;
+ List<TaskConfig> taskConfigs = appSpec.getTaskConfigs();
+ if (taskConfigs != null) {
+ for (TaskConfig taskConfig : taskConfigs) {
+ String yamlFile = taskConfig.getValue("yamlFile");
+ if (yamlFile != null) {
+ Workflow flow = Workflow.parse(new File(yamlFile));
+ workflow = flow.getName();
+ }
+ }
+ }
+
+ // Repeatedly poll for status
+ if (workflow != null) {
+ ClusterAccessor accessor = connection.createClusterAccessor(clusterId);
+ TaskDriver driver = new TaskDriver(manager);
+ while (true) {
+ System.out.println("CONTAINER STATUS");
+ System.out.println("----------------");
+ Collection<Participant> participants = accessor.readParticipants().values();
+ for (Participant participant : participants) {
+ ContainerConfig containerConfig = participant.getContainerConfig();
+ if (containerConfig != null) {
+ System.out.println(participant.getId() + "[" + containerConfig.getId() + "]: "
+ + containerConfig.getState());
+ }
+ if (participant.isAlive()) {
+ RunningInstance runningInstance = participant.getRunningInstance();
+ System.out.println("\tProcess: " + runningInstance.getPid());
+ }
+ }
+ System.out.println("----------------");
+ System.out.println("TASK STATUS");
+ System.out.println("----------------");
+ driver.list(workflow);
+ Thread.sleep(5000);
+ }
+ }
+ }
+ }
http://git-wip-us.apache.org/repos/asf/helix/blob/713586c4/recipes/jobrunner-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/MyTask.java
----------------------------------------------------------------------
diff --cc recipes/jobrunner-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/MyTask.java
index 0000000,584550d..650bb14
mode 000000,100644..100644
--- a/recipes/jobrunner-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/MyTask.java
+++ b/recipes/jobrunner-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/MyTask.java
@@@ -1,0 -1,53 +1,72 @@@
+ package org.apache.helix.provisioning.yarn.example;
+
++/*
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements. See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership. The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing,
++ * software distributed under the License is distributed on an
++ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
++ * KIND, either express or implied. See the License for the
++ * specific language governing permissions and limitations
++ * under the License.
++ */
++
+ import org.apache.helix.task.Task;
+ import org.apache.helix.task.TaskCallbackContext;
+ import org.apache.helix.task.TaskResult;
+ import org.apache.log4j.Logger;
+
+ /**
+ * Callbacks for task execution - THIS INTERFACE IS SUBJECT TO CHANGE
+ */
+ public class MyTask implements Task {
+ private static final Logger LOG = Logger.getLogger(MyTask.class);
+ private static final long DEFAULT_DELAY = 60000L;
+ private final long _delay;
+ private volatile boolean _canceled;
+
+ public MyTask(TaskCallbackContext context) {
+ LOG.info("Job config" + context.getJobConfig().getJobConfigMap());
+ if (context.getTaskConfig() != null) {
+ LOG.info("Task config: " + context.getTaskConfig().getConfigMap());
+ }
+ _delay = DEFAULT_DELAY;
+ }
+
+ @Override
+ public TaskResult run() {
+ long expiry = System.currentTimeMillis() + _delay;
+ long timeLeft;
+ while (System.currentTimeMillis() < expiry) {
+ if (_canceled) {
+ timeLeft = expiry - System.currentTimeMillis();
+ return new TaskResult(TaskResult.Status.CANCELED, String.valueOf(timeLeft < 0 ? 0
+ : timeLeft));
+ }
+ sleep(50);
+ }
+ timeLeft = expiry - System.currentTimeMillis();
+ return new TaskResult(TaskResult.Status.COMPLETED, String.valueOf(timeLeft < 0 ? 0 : timeLeft));
+ }
+
+ @Override
+ public void cancel() {
+ _canceled = true;
+ }
+
+ private static void sleep(long d) {
+ try {
+ Thread.sleep(d);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ }
http://git-wip-us.apache.org/repos/asf/helix/blob/713586c4/recipes/jobrunner-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/MyTaskAppSpec.java
----------------------------------------------------------------------
diff --cc recipes/jobrunner-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/MyTaskAppSpec.java
index 0000000,a20994c..50fb3de
mode 000000,100644..100644
--- a/recipes/jobrunner-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/MyTaskAppSpec.java
+++ b/recipes/jobrunner-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/MyTaskAppSpec.java
@@@ -1,0 -1,148 +1,167 @@@
+ package org.apache.helix.provisioning.yarn.example;
+
++/*
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements. See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership. The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing,
++ * software distributed under the License is distributed on an
++ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
++ * KIND, either express or implied. See the License for the
++ * specific language governing permissions and limitations
++ * under the License.
++ */
++
+ import java.net.URI;
+ import java.net.URISyntaxException;
+ import java.util.List;
+ import java.util.Map;
+
+ import org.apache.helix.api.Scope;
+ import org.apache.helix.api.id.ResourceId;
+ import org.apache.helix.provisioning.AppConfig;
+ import org.apache.helix.provisioning.ApplicationSpec;
+ import org.apache.helix.provisioning.ServiceConfig;
+ import org.apache.helix.provisioning.TaskConfig;
+
+ import com.google.common.collect.Maps;
+
+ public class MyTaskAppSpec implements ApplicationSpec {
+
+ public String _appName;
+
+ public AppConfig _appConfig;
+
+ public List<String> _services;
+
+ private String _appMasterPackageUri;
+
+ private Map<String, String> _servicePackageURIMap;
+
+ private Map<String, String> _serviceMainClassMap;
+
+ private Map<String, ServiceConfig> _serviceConfigMap;
+
+ private List<TaskConfig> _taskConfigs;
+
+ public AppConfig getAppConfig() {
+ return _appConfig;
+ }
+
+ public void setAppConfig(AppConfig appConfig) {
+ _appConfig = appConfig;
+ }
+
+ public String getAppMasterPackageUri() {
+ return _appMasterPackageUri;
+ }
+
+ public void setAppMasterPackageUri(String appMasterPackageUri) {
+ _appMasterPackageUri = appMasterPackageUri;
+ }
+
+ public Map<String, String> getServicePackageURIMap() {
+ return _servicePackageURIMap;
+ }
+
+ public void setServicePackageURIMap(Map<String, String> servicePackageURIMap) {
+ _servicePackageURIMap = servicePackageURIMap;
+ }
+
+ public Map<String, String> getServiceMainClassMap() {
+ return _serviceMainClassMap;
+ }
+
+ public void setServiceMainClassMap(Map<String, String> serviceMainClassMap) {
+ _serviceMainClassMap = serviceMainClassMap;
+ }
+
+ public Map<String, Map<String, String>> getServiceConfigMap() {
+ Map<String, Map<String, String>> map = Maps.newHashMap();
+ for (String service : _serviceConfigMap.keySet()) {
+ map.put(service, _serviceConfigMap.get(service).getSimpleFields());
+ }
+ return map;
+ }
+
+ public void setServiceConfigMap(Map<String, Map<String, Object>> map) {
+ _serviceConfigMap = Maps.newHashMap();
+
+ for (String service : map.keySet()) {
+ ServiceConfig serviceConfig = new ServiceConfig(Scope.resource(ResourceId.from(service)));
+ Map<String, Object> simpleFields = map.get(service);
+ for (String key : simpleFields.keySet()) {
+ serviceConfig.setSimpleField(key, simpleFields.get(key).toString());
+ }
+ _serviceConfigMap.put(service, serviceConfig);
+ }
+ }
+
+ public void setAppName(String appName) {
+ _appName = appName;
+ }
+
+ public void setServices(List<String> services) {
+ _services = services;
+ }
+
+ public void setTaskConfigs(List<TaskConfig> taskConfigs) {
+ _taskConfigs = taskConfigs;
+ }
+
+ @Override
+ public String getAppName() {
+ return _appName;
+ }
+
+ @Override
+ public AppConfig getConfig() {
+ return _appConfig;
+ }
+
+ @Override
+ public List<String> getServices() {
+ return _services;
+ }
+
+ @Override
+ public URI getAppMasterPackage() {
+ try {
+ return new URI(_appMasterPackageUri);
+ } catch (URISyntaxException e) {
+ return null;
+ }
+ }
+
+ @Override
+ public URI getServicePackage(String serviceName) {
+ try {
+ return new URI(_servicePackageURIMap.get(serviceName));
+ } catch (URISyntaxException e) {
+ return null;
+ }
+ }
+
+ @Override
+ public String getServiceMainClass(String service) {
+ return _serviceMainClassMap.get(service);
+ }
+
+ @Override
+ public ServiceConfig getServiceConfig(String serviceName) {
+ return _serviceConfigMap.get(serviceName);
+ }
+
+ @Override
+ public List<TaskConfig> getTaskConfigs() {
+ return _taskConfigs;
+ }
+
+ }
http://git-wip-us.apache.org/repos/asf/helix/blob/713586c4/recipes/jobrunner-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/MyTaskAppSpecFactory.java
----------------------------------------------------------------------
diff --cc recipes/jobrunner-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/MyTaskAppSpecFactory.java
index 0000000,17601ba..d5f486a
mode 000000,100644..100644
--- a/recipes/jobrunner-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/MyTaskAppSpecFactory.java
+++ b/recipes/jobrunner-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/MyTaskAppSpecFactory.java
@@@ -1,0 -1,28 +1,47 @@@
+ package org.apache.helix.provisioning.yarn.example;
+
++/*
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements. See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership. The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing,
++ * software distributed under the License is distributed on an
++ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
++ * KIND, either express or implied. See the License for the
++ * specific language governing permissions and limitations
++ * under the License.
++ */
++
+ import java.io.InputStream;
+
+ import org.apache.helix.provisioning.ApplicationSpec;
+ import org.apache.helix.provisioning.ApplicationSpecFactory;
+ import org.yaml.snakeyaml.Yaml;
+
+ public class MyTaskAppSpecFactory implements ApplicationSpecFactory {
+
+ @Override
+ public ApplicationSpec fromYaml(InputStream inputstream) {
+ return (ApplicationSpec) new Yaml().load(inputstream);
+ // return data;
+ }
+
+ public static void main(String[] args) {
+
+ Yaml yaml = new Yaml();
+ InputStream resourceAsStream =
+ ClassLoader.getSystemClassLoader().getResourceAsStream("job_runner_app_spec.yaml");
+ MyTaskAppSpec spec = yaml.loadAs(resourceAsStream, MyTaskAppSpec.class);
+ String dump = yaml.dump(spec);
+ System.out.println(dump);
+ System.out.println(spec.getServiceConfig("JobRunner").getStringField("num_containers", "1"));
+
+ }
+ }
http://git-wip-us.apache.org/repos/asf/helix/blob/713586c4/recipes/jobrunner-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/MyTaskService.java
----------------------------------------------------------------------
diff --cc recipes/jobrunner-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/MyTaskService.java
index 0000000,22c3ab0..7c50e53
mode 000000,100644..100644
--- a/recipes/jobrunner-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/MyTaskService.java
+++ b/recipes/jobrunner-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/MyTaskService.java
@@@ -1,0 -1,62 +1,81 @@@
+ package org.apache.helix.provisioning.yarn.example;
+
++/*
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements. See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership. The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing,
++ * software distributed under the License is distributed on an
++ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
++ * KIND, either express or implied. See the License for the
++ * specific language governing permissions and limitations
++ * under the License.
++ */
++
+ import java.util.HashMap;
+ import java.util.Map;
+
+ import org.apache.helix.HelixConnection;
+ import org.apache.helix.HelixManager;
+ import org.apache.helix.api.id.ClusterId;
+ import org.apache.helix.api.id.ParticipantId;
+ import org.apache.helix.api.id.StateModelDefId;
+ import org.apache.helix.manager.zk.HelixConnectionAdaptor;
+ import org.apache.helix.participant.AbstractParticipantService;
+ import org.apache.helix.provisioning.ServiceConfig;
+ import org.apache.helix.provisioning.participant.StatelessParticipantService;
+ import org.apache.helix.task.Task;
+ import org.apache.helix.task.TaskCallbackContext;
+ import org.apache.helix.task.TaskFactory;
+ import org.apache.helix.task.TaskStateModelFactory;
+ import org.apache.log4j.Logger;
+
+ /**
+ * A simple "service" for task callback registration.
+ */
+ public class MyTaskService extends StatelessParticipantService {
+
+ private static Logger LOG = Logger.getLogger(AbstractParticipantService.class);
+
+ static String SERVICE_NAME = "JobRunner";
+
+ public MyTaskService(HelixConnection connection, ClusterId clusterId,
+ ParticipantId participantId) {
+ super(connection, clusterId, participantId, SERVICE_NAME);
+ }
+
+ @Override
+ protected void init(ServiceConfig serviceConfig) {
+ LOG.info("Initialized service with config " + serviceConfig);
+
+ // Register for callbacks for tasks
+ HelixManager manager = new HelixConnectionAdaptor(getParticipant());
+ Map<String, TaskFactory> taskFactoryReg = new HashMap<String, TaskFactory>();
+ taskFactoryReg.put("RunTask", new TaskFactory() {
+ @Override
+ public Task createNewTask(TaskCallbackContext context) {
+ return new MyTask(context);
+ }
+ });
+ getParticipant().getStateMachineEngine().registerStateModelFactory(
+ StateModelDefId.from("Task"), new TaskStateModelFactory(manager, taskFactoryReg));
+ }
+
+ @Override
+ protected void goOnline() {
+ LOG.info("JobRunner service is told to go online");
+ }
+
+ @Override
+ protected void goOffine() {
+ LOG.info("JobRunner service is told to go offline");
+ }
+
+ }
http://git-wip-us.apache.org/repos/asf/helix/blob/713586c4/recipes/jobrunner-yarn/src/main/resources/dummy_job.yaml
----------------------------------------------------------------------
diff --cc recipes/jobrunner-yarn/src/main/resources/dummy_job.yaml
index 0000000,0187fd1..ff44243
mode 000000,100644..100644
--- a/recipes/jobrunner-yarn/src/main/resources/dummy_job.yaml
+++ b/recipes/jobrunner-yarn/src/main/resources/dummy_job.yaml
@@@ -1,0 -1,18 +1,36 @@@
++#
++# Licensed to the Apache Software Foundation (ASF) under one
++# or more contributor license agreements. See the NOTICE file
++# distributed with this work for additional information
++# regarding copyright ownership. The ASF licenses this file
++# to you under the Apache License, Version 2.0 (the
++# "License"); you may not use this file except in compliance
++# with the License. You may obtain a copy of the License at
++#
++# http://www.apache.org/licenses/LICENSE-2.0
++#
++# Unless required by applicable law or agreed to in writing,
++# software distributed under the License is distributed on an
++# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
++# KIND, either express or implied. See the License for the
++# specific language governing permissions and limitations
++# under the License.
++#
+ name: myJob1234
+ jobs:
+ - name: myJob1234
+ command: RunTask
+ jobConfigMap: {
+ k1: "v1",
+ k2: "v2"
+ }
+ tasks:
+ - taskConfigMap: {
+ k3: "v3"
+ }
+ - taskConfigMap: {
+ k4: "v4"
+ }
+ - taskConfigMap: {
+ k5: "v5"
+ }
http://git-wip-us.apache.org/repos/asf/helix/blob/713586c4/recipes/jobrunner-yarn/src/main/resources/job_runner_app_spec.yaml
----------------------------------------------------------------------
diff --cc recipes/jobrunner-yarn/src/main/resources/job_runner_app_spec.yaml
index 0000000,0945690..83c7edf
mode 000000,100755..100755
--- a/recipes/jobrunner-yarn/src/main/resources/job_runner_app_spec.yaml
+++ b/recipes/jobrunner-yarn/src/main/resources/job_runner_app_spec.yaml
@@@ -1,0 -1,23 +1,41 @@@
++#
++# Licensed to the Apache Software Foundation (ASF) under one
++# or more contributor license agreements. See the NOTICE file
++# distributed with this work for additional information
++# regarding copyright ownership. The ASF licenses this file
++# to you under the Apache License, Version 2.0 (the
++# "License"); you may not use this file except in compliance
++# with the License. You may obtain a copy of the License at
++#
++# http://www.apache.org/licenses/LICENSE-2.0
++#
++# Unless required by applicable law or agreed to in writing,
++# software distributed under the License is distributed on an
++# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
++# KIND, either express or implied. See the License for the
++# specific language governing permissions and limitations
++# under the License.
++#
+ !!org.apache.helix.provisioning.yarn.example.MyTaskAppSpec
+ appConfig:
+ config: {
+ k1: v1
+ }
+ appMasterPackageUri: 'file:///Users/kbiscuit/helix/incubator-helix/recipes/jobrunner-yarn/target/jobrunner-yarn-0.7.1-incubating-SNAPSHOT-pkg.tar'
+ appName: testApp
+ serviceConfigMap:
+ JobRunner: {
+ num_containers: 3,
+ memory: 1024
+ }
+ serviceMainClassMap: {
+ JobRunner: org.apache.helix.provisioning.yarn.example.MyTaskService
+ }
+ servicePackageURIMap: {
+ JobRunner: 'file:///Users/kbiscuit/helix/incubator-helix/recipes/jobrunner-yarn/target/jobrunner-yarn-0.7.1-incubating-SNAPSHOT-pkg.tar'
+ }
+ services: [
+ JobRunner]
+ taskConfigs:
+ - name: JobRunnerWorkflow
+ yamlFile: 'file:///Users/kbiscuit/helix/incubator-helix/recipes/jobrunner-yarn/src/main/resources/dummy_job.yaml'
|