Adding new Helloworld recipe to launch containers on YARN
Project: http://git-wip-us.apache.org/repos/asf/helix/repo
Commit: http://git-wip-us.apache.org/repos/asf/helix/commit/b47e3299
Tree: http://git-wip-us.apache.org/repos/asf/helix/tree/b47e3299
Diff: http://git-wip-us.apache.org/repos/asf/helix/diff/b47e3299
Branch: refs/heads/master
Commit: b47e329905d853b8566231946bb9be04ad2b0685
Parents: 7118717
Author: Kishore Gopalakrishna <g.kishore@gmail.com>
Authored: Sat Feb 22 00:05:41 2014 -0800
Committer: Kishore Gopalakrishna <g.kishore@gmail.com>
Committed: Sat Feb 22 00:05:41 2014 -0800
----------------------------------------------------------------------
.../helix/provisioning/yarn/AppLauncher.java | 10 +++--
.../yarn/HelixYarnApplicationMasterMain.java | 28 ++++++++-----
.../helix/provisioning/yarn/HelixYarnUtil.java | 42 ++++++++++++++++++++
recipes/provisioning/yarn/helloworld/pom.xml | 2 +-
.../main/resources/hello_world_app_spec.yaml | 4 +-
5 files changed, 70 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/helix/blob/b47e3299/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/AppLauncher.java
----------------------------------------------------------------------
diff --git a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/AppLauncher.java
b/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/AppLauncher.java
index 3bba018..1fe0a28 100644
--- a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/AppLauncher.java
+++ b/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/AppLauncher.java
@@ -42,7 +42,6 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.client.api.YarnClientApplication;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Records;
@@ -375,6 +374,9 @@ public class AppLauncher {
+ ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop");
return false;
}
+ if (YarnApplicationState.RUNNING == state) {
+
+ }
prevReport = reportMessage;
Thread.sleep(10000);
} catch (Exception e) {
@@ -404,7 +406,8 @@ public class AppLauncher {
LOG.info("Cleaning up");
try {
ApplicationReport applicationReport = yarnClient.getApplicationReport(_appId);
- LOG.info("Killing application:"+ _appId + " \n Application report" + generateReport(applicationReport));
+ LOG.info("Killing application:" + _appId + " \n Application report"
+ + generateReport(applicationReport));
} catch (Exception e) {
e.printStackTrace();
}
@@ -416,8 +419,7 @@ public class AppLauncher {
* @throws Exception
*/
public static void main(String[] args) throws Exception {
- ApplicationSpecFactory applicationSpecFactory =
- (ApplicationSpecFactory) Class.forName(args[0]).newInstance();
+ ApplicationSpecFactory applicationSpecFactory = HelixYarnUtil.createInstance(args[0]);
File yamlConfigFile = new File(args[1]);
final AppLauncher launcher = new AppLauncher(applicationSpecFactory, yamlConfigFile);
launcher.launch();
http://git-wip-us.apache.org/repos/asf/helix/blob/b47e3299/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/HelixYarnApplicationMasterMain.java
----------------------------------------------------------------------
diff --git a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/HelixYarnApplicationMasterMain.java
b/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/HelixYarnApplicationMasterMain.java
index 33183c7..5884a35 100644
--- a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/HelixYarnApplicationMasterMain.java
+++ b/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/HelixYarnApplicationMasterMain.java
@@ -2,6 +2,7 @@ package org.apache.helix.provisioning.yarn;
import java.io.File;
import java.io.FileInputStream;
+import java.io.IOException;
import java.util.Arrays;
import java.util.Map;
@@ -11,10 +12,12 @@ import org.I0Itec.zkclient.ZkServer;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.helix.HelixController;
import org.apache.helix.api.accessor.ClusterAccessor;
@@ -43,7 +46,8 @@ import org.apache.log4j.Logger;
public class HelixYarnApplicationMasterMain {
public static Logger LOG = Logger.getLogger(HelixYarnApplicationMasterMain.class);
- public static void main(String[] args) throws Exception {
+ @SuppressWarnings("unchecked")
+ public static void main(String[] args) throws Exception{
Map<String, String> env = System.getenv();
LOG.info("Starting app master with the following environment variables");
for (String key : env.keySet()) {
@@ -71,8 +75,12 @@ public class HelixYarnApplicationMasterMain {
}
};
- FileUtils.deleteDirectory(new File(dataDir));
- FileUtils.deleteDirectory(new File(logDir));
+ try {
+ FileUtils.deleteDirectory(new File(dataDir));
+ FileUtils.deleteDirectory(new File(logDir));
+ } catch (IOException e) {
+ LOG.error(e);
+ }
final ZkServer server = new ZkServer(dataDir, logDir, defaultNameSpace);
server.start();
@@ -84,13 +92,15 @@ public class HelixYarnApplicationMasterMain {
ApplicationAttemptId appAttemptID = containerId.getApplicationAttemptId();
String configFile = AppMasterConfig.AppEnvironment.APP_SPEC_FILE.toString();
- ApplicationSpecFactory factory =
- (ApplicationSpecFactory) Class.forName(appMasterConfig.getApplicationSpecFactory())
- .newInstance();
-
+ String className = appMasterConfig.getApplicationSpecFactory();
+
GenericApplicationMaster genericApplicationMaster = new GenericApplicationMaster(appAttemptID);
- genericApplicationMaster.start();
-
+ try {
+ genericApplicationMaster.start();
+ } catch (Exception e) {
+ LOG.error("Unable to start application master: ", e);
+ }
+ ApplicationSpecFactory factory = HelixYarnUtil.createInstance(className);
YarnProvisioner.applicationMaster = genericApplicationMaster;
YarnProvisioner.applicationMasterConfig = appMasterConfig;
ApplicationSpec applicationSpec = factory.fromYaml(new FileInputStream(configFile));
http://git-wip-us.apache.org/repos/asf/helix/blob/b47e3299/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/HelixYarnUtil.java
----------------------------------------------------------------------
diff --git a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/HelixYarnUtil.java
b/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/HelixYarnUtil.java
new file mode 100644
index 0000000..ad606ba
--- /dev/null
+++ b/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/HelixYarnUtil.java
@@ -0,0 +1,42 @@
+package org.apache.helix.provisioning.yarn;
+
+import org.apache.log4j.Logger;
+
+public class HelixYarnUtil {
+ private static Logger LOG = Logger.getLogger(HelixYarnUtil.class);
+
+ @SuppressWarnings("unchecked")
+ public static <T extends ApplicationSpecFactory> T createInstance(String className)
{
+ Class<ApplicationSpecFactory> factoryClazz = null;
+ {
+ try {
+ factoryClazz =
+ (Class<ApplicationSpecFactory>) Thread.currentThread().getContextClassLoader()
+ .loadClass(className);
+ } catch (ClassNotFoundException e) {
+ try {
+ factoryClazz =
+ (Class<ApplicationSpecFactory>) ClassLoader.getSystemClassLoader().loadClass(
+ className);
+ } catch (ClassNotFoundException e1) {
+ try {
+ factoryClazz = (Class<ApplicationSpecFactory>) Class.forName(className);
+ } catch (ClassNotFoundException e2) {
+
+ }
+ }
+ }
+ }
+ System.out.println(System.getProperty("java.class.path"));
+ if (factoryClazz == null) {
+ LOG.error("Unable to find class:" + className);
+ }
+ ApplicationSpecFactory factory = null;
+ try {
+ factory = factoryClazz.newInstance();
+ } catch (Exception e) {
+ LOG.error("Unable to create instance of class: " + className, e);
+ }
+ return (T) factory;
+ }
+}
http://git-wip-us.apache.org/repos/asf/helix/blob/b47e3299/recipes/provisioning/yarn/helloworld/pom.xml
----------------------------------------------------------------------
diff --git a/recipes/provisioning/yarn/helloworld/pom.xml b/recipes/provisioning/yarn/helloworld/pom.xml
index bc6aca2..6c30679 100644
--- a/recipes/provisioning/yarn/helloworld/pom.xml
+++ b/recipes/provisioning/yarn/helloworld/pom.xml
@@ -36,7 +36,7 @@ under the License.
org.apache.log4j,
*
</osgi.import>
- <osgi.export>org.apache.helix.recipes.provisioning.yarn.helloworld*;version="${project.version};-noimport:=true</osgi.export>
+ <osgi.export>org.apache.helix.provisioning.yarn.example*;version="${project.version};-noimport:=true</osgi.export>
</properties>
<dependencies>
http://git-wip-us.apache.org/repos/asf/helix/blob/b47e3299/recipes/provisioning/yarn/helloworld/src/main/resources/hello_world_app_spec.yaml
----------------------------------------------------------------------
diff --git a/recipes/provisioning/yarn/helloworld/src/main/resources/hello_world_app_spec.yaml
b/recipes/provisioning/yarn/helloworld/src/main/resources/hello_world_app_spec.yaml
index 535bece..d8d1dd2 100644
--- a/recipes/provisioning/yarn/helloworld/src/main/resources/hello_world_app_spec.yaml
+++ b/recipes/provisioning/yarn/helloworld/src/main/resources/hello_world_app_spec.yaml
@@ -3,7 +3,7 @@ appConfig:
config: {
k1: v1
}
-appMasterPackageUri: 'file:///Users/kbiscuit/helix/incubator-helix/recipes/provisioning/yarn/helloworld/target/helloworld-0.7.1-incubating-SNAPSHOT-pkg.tar'
+appMasterPackageUri: 'file:///Users/kgopalak/Documents/projects/incubator-helix/recipes/provisioning/yarn/helloworld/target/helloworld-0.7.1-incubating-SNAPSHOT-pkg.tar'
appName: testApp
serviceConfigMap:
HelloWorld: {
@@ -14,7 +14,7 @@ serviceMainClassMap: {
HelloWorld: org.apache.helix.provisioning.yarn.example.HelloWorldService
}
servicePackageURIMap: {
- HelloWorld: 'file:///Users/kbiscuit/helix/incubator-helix/recipes/provisioning/yarn/helloworld/target/helloworld-0.7.1-incubating-SNAPSHOT-pkg.tar'
+ HelloWorld: 'file:///Users/kgopalak/Documents/projects/incubator-helix/recipes/provisioning/yarn/helloworld/target/helloworld-0.7.1-incubating-SNAPSHOT-pkg.tar'
}
services: [
HelloWorld]
|