helix-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ka...@apache.org
Subject [15/50] [abbrv] git commit: Adding new Helloworld recipe to launch containers on YARN
Date Thu, 10 Jul 2014 17:04:58 GMT
Adding new Helloworld recipe to launch containers on YARN


Project: http://git-wip-us.apache.org/repos/asf/helix/repo
Commit: http://git-wip-us.apache.org/repos/asf/helix/commit/b47e3299
Tree: http://git-wip-us.apache.org/repos/asf/helix/tree/b47e3299
Diff: http://git-wip-us.apache.org/repos/asf/helix/diff/b47e3299

Branch: refs/heads/master
Commit: b47e329905d853b8566231946bb9be04ad2b0685
Parents: 7118717
Author: Kishore Gopalakrishna <g.kishore@gmail.com>
Authored: Sat Feb 22 00:05:41 2014 -0800
Committer: Kishore Gopalakrishna <g.kishore@gmail.com>
Committed: Sat Feb 22 00:05:41 2014 -0800

----------------------------------------------------------------------
 .../helix/provisioning/yarn/AppLauncher.java    | 10 +++--
 .../yarn/HelixYarnApplicationMasterMain.java    | 28 ++++++++-----
 .../helix/provisioning/yarn/HelixYarnUtil.java  | 42 ++++++++++++++++++++
 recipes/provisioning/yarn/helloworld/pom.xml    |  2 +-
 .../main/resources/hello_world_app_spec.yaml    |  4 +-
 5 files changed, 70 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/helix/blob/b47e3299/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/AppLauncher.java
----------------------------------------------------------------------
diff --git a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/AppLauncher.java
b/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/AppLauncher.java
index 3bba018..1fe0a28 100644
--- a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/AppLauncher.java
+++ b/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/AppLauncher.java
@@ -42,7 +42,6 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 import org.apache.hadoop.yarn.client.api.YarnClient;
 import org.apache.hadoop.yarn.client.api.YarnClientApplication;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.hadoop.yarn.util.Records;
 
@@ -375,6 +374,9 @@ public class AppLauncher {
               + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop");
           return false;
         }
+        if (YarnApplicationState.RUNNING == state) {
+
+        }
         prevReport = reportMessage;
         Thread.sleep(10000);
       } catch (Exception e) {
@@ -404,7 +406,8 @@ public class AppLauncher {
     LOG.info("Cleaning up");
     try {
       ApplicationReport applicationReport = yarnClient.getApplicationReport(_appId);
-      LOG.info("Killing application:"+ _appId + " \n Application report" + generateReport(applicationReport));
+      LOG.info("Killing application:" + _appId + " \n Application report"
+          + generateReport(applicationReport));
     } catch (Exception e) {
       e.printStackTrace();
     }
@@ -416,8 +419,7 @@ public class AppLauncher {
    * @throws Exception
    */
   public static void main(String[] args) throws Exception {
-    ApplicationSpecFactory applicationSpecFactory =
-        (ApplicationSpecFactory) Class.forName(args[0]).newInstance();
+    ApplicationSpecFactory applicationSpecFactory = HelixYarnUtil.createInstance(args[0]);
     File yamlConfigFile = new File(args[1]);
     final AppLauncher launcher = new AppLauncher(applicationSpecFactory, yamlConfigFile);
     launcher.launch();

http://git-wip-us.apache.org/repos/asf/helix/blob/b47e3299/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/HelixYarnApplicationMasterMain.java
----------------------------------------------------------------------
diff --git a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/HelixYarnApplicationMasterMain.java
b/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/HelixYarnApplicationMasterMain.java
index 33183c7..5884a35 100644
--- a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/HelixYarnApplicationMasterMain.java
+++ b/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/HelixYarnApplicationMasterMain.java
@@ -2,6 +2,7 @@ package org.apache.helix.provisioning.yarn;
 
 import java.io.File;
 import java.io.FileInputStream;
+import java.io.IOException;
 import java.util.Arrays;
 import java.util.Map;
 
@@ -11,10 +12,12 @@ import org.I0Itec.zkclient.ZkServer;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.GnuParser;
 import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
 import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.helix.HelixController;
 import org.apache.helix.api.accessor.ClusterAccessor;
@@ -43,7 +46,8 @@ import org.apache.log4j.Logger;
 public class HelixYarnApplicationMasterMain {
   public static Logger LOG = Logger.getLogger(HelixYarnApplicationMasterMain.class);
 
-  public static void main(String[] args) throws Exception {
+  @SuppressWarnings("unchecked")
+  public static void main(String[] args) throws Exception{
     Map<String, String> env = System.getenv();
     LOG.info("Starting app master with the following environment variables");
     for (String key : env.keySet()) {
@@ -71,8 +75,12 @@ public class HelixYarnApplicationMasterMain {
 
       }
     };
-    FileUtils.deleteDirectory(new File(dataDir));
-    FileUtils.deleteDirectory(new File(logDir));
+    try {
+      FileUtils.deleteDirectory(new File(dataDir));
+      FileUtils.deleteDirectory(new File(logDir));
+    } catch (IOException e) {
+      LOG.error(e);
+    }
 
     final ZkServer server = new ZkServer(dataDir, logDir, defaultNameSpace);
     server.start();
@@ -84,13 +92,15 @@ public class HelixYarnApplicationMasterMain {
     ApplicationAttemptId appAttemptID = containerId.getApplicationAttemptId();
 
     String configFile = AppMasterConfig.AppEnvironment.APP_SPEC_FILE.toString();
-    ApplicationSpecFactory factory =
-        (ApplicationSpecFactory) Class.forName(appMasterConfig.getApplicationSpecFactory())
-            .newInstance();
-
+    String className = appMasterConfig.getApplicationSpecFactory();
+   
     GenericApplicationMaster genericApplicationMaster = new GenericApplicationMaster(appAttemptID);
-    genericApplicationMaster.start();
-
+    try {
+      genericApplicationMaster.start();
+    } catch (Exception e) {
+      LOG.error("Unable to start application master: ", e);
+    }
+    ApplicationSpecFactory factory = HelixYarnUtil.createInstance(className);
     YarnProvisioner.applicationMaster = genericApplicationMaster;
     YarnProvisioner.applicationMasterConfig = appMasterConfig;
     ApplicationSpec applicationSpec = factory.fromYaml(new FileInputStream(configFile));

http://git-wip-us.apache.org/repos/asf/helix/blob/b47e3299/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/HelixYarnUtil.java
----------------------------------------------------------------------
diff --git a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/HelixYarnUtil.java
b/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/HelixYarnUtil.java
new file mode 100644
index 0000000..ad606ba
--- /dev/null
+++ b/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/HelixYarnUtil.java
@@ -0,0 +1,42 @@
+package org.apache.helix.provisioning.yarn;
+
+import org.apache.log4j.Logger;
+
+public class HelixYarnUtil {
+  private static Logger LOG = Logger.getLogger(HelixYarnUtil.class);
+
+  @SuppressWarnings("unchecked")
+  public static <T extends ApplicationSpecFactory> T createInstance(String className)
{
+    Class<ApplicationSpecFactory> factoryClazz = null;
+    {
+      try {
+        factoryClazz =
+            (Class<ApplicationSpecFactory>) Thread.currentThread().getContextClassLoader()
+                .loadClass(className);
+      } catch (ClassNotFoundException e) {
+        try {
+          factoryClazz =
+              (Class<ApplicationSpecFactory>) ClassLoader.getSystemClassLoader().loadClass(
+                  className);
+        } catch (ClassNotFoundException e1) {
+          try {
+            factoryClazz = (Class<ApplicationSpecFactory>) Class.forName(className);
+          } catch (ClassNotFoundException e2) {
+
+          }
+        }
+      }
+    }
+    System.out.println(System.getProperty("java.class.path"));
+    if (factoryClazz == null) {
+      LOG.error("Unable to find class:" + className);
+    }
+    ApplicationSpecFactory factory = null;
+    try {
+      factory = factoryClazz.newInstance();
+    } catch (Exception e) {
+      LOG.error("Unable to create instance of class: " + className, e);
+    }
+    return (T) factory;
+  }
+}

http://git-wip-us.apache.org/repos/asf/helix/blob/b47e3299/recipes/provisioning/yarn/helloworld/pom.xml
----------------------------------------------------------------------
diff --git a/recipes/provisioning/yarn/helloworld/pom.xml b/recipes/provisioning/yarn/helloworld/pom.xml
index bc6aca2..6c30679 100644
--- a/recipes/provisioning/yarn/helloworld/pom.xml
+++ b/recipes/provisioning/yarn/helloworld/pom.xml
@@ -36,7 +36,7 @@ under the License.
       org.apache.log4j,
       *
     </osgi.import>
-    <osgi.export>org.apache.helix.recipes.provisioning.yarn.helloworld*;version="${project.version};-noimport:=true</osgi.export>
+    <osgi.export>org.apache.helix.provisioning.yarn.example*;version="${project.version};-noimport:=true</osgi.export>
   </properties>
 
   <dependencies>

http://git-wip-us.apache.org/repos/asf/helix/blob/b47e3299/recipes/provisioning/yarn/helloworld/src/main/resources/hello_world_app_spec.yaml
----------------------------------------------------------------------
diff --git a/recipes/provisioning/yarn/helloworld/src/main/resources/hello_world_app_spec.yaml
b/recipes/provisioning/yarn/helloworld/src/main/resources/hello_world_app_spec.yaml
index 535bece..d8d1dd2 100644
--- a/recipes/provisioning/yarn/helloworld/src/main/resources/hello_world_app_spec.yaml
+++ b/recipes/provisioning/yarn/helloworld/src/main/resources/hello_world_app_spec.yaml
@@ -3,7 +3,7 @@ appConfig:
   config: {
     k1: v1
   }
-appMasterPackageUri: 'file:///Users/kbiscuit/helix/incubator-helix/recipes/provisioning/yarn/helloworld/target/helloworld-0.7.1-incubating-SNAPSHOT-pkg.tar'
+appMasterPackageUri: 'file:///Users/kgopalak/Documents/projects/incubator-helix/recipes/provisioning/yarn/helloworld/target/helloworld-0.7.1-incubating-SNAPSHOT-pkg.tar'
 appName: testApp
 serviceConfigMap:
   HelloWorld: {
@@ -14,7 +14,7 @@ serviceMainClassMap: {
   HelloWorld: org.apache.helix.provisioning.yarn.example.HelloWorldService
 }
 servicePackageURIMap: {
-  HelloWorld: 'file:///Users/kbiscuit/helix/incubator-helix/recipes/provisioning/yarn/helloworld/target/helloworld-0.7.1-incubating-SNAPSHOT-pkg.tar'
+  HelloWorld: 'file:///Users/kgopalak/Documents/projects/incubator-helix/recipes/provisioning/yarn/helloworld/target/helloworld-0.7.1-incubating-SNAPSHOT-pkg.tar'
 }
 services: [
   HelloWorld]


Mime
View raw message