incubator-s4-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mmo...@apache.org
Subject git commit: S4-59 fixed s4-tools commands with new configuration scheme - also fixed some parameters names that needed an update + doc
Date Tue, 22 Jan 2013 17:02:41 GMT
Updated Branches:
  refs/heads/S4-59 a941dfd0e -> 7548ba53b


S4-59 fixed s4-tools commands with new configuration scheme
- also fixed some parameters names that needed an update + doc


Project: http://git-wip-us.apache.org/repos/asf/incubator-s4/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s4/commit/7548ba53
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s4/tree/7548ba53
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s4/diff/7548ba53

Branch: refs/heads/S4-59
Commit: 7548ba53b70e9131dd30e437e392d2c0b6e30731
Parents: a941dfd
Author: Matthieu Morel <mmorel@apache.org>
Authored: Tue Jan 22 17:42:46 2013 +0100
Committer: Matthieu Morel <mmorel@apache.org>
Committed: Tue Jan 22 17:49:48 2013 +0100

----------------------------------------------------------------------
 .../main/java/org/apache/s4/core/S4Bootstrap.java  |    4 +-
 .../src/main/java/org/apache/s4/core/S4Node.java   |    4 +-
 .../java/org/apache/s4/core/util/AppConfig.java    |   12 ++++++--
 .../s4/deploy/prodcon/TestProducerConsumer.java    |    2 +-
 .../src/main/java/org/apache/s4/tools/Deploy.java  |   23 ++++++++++++--
 .../src/main/java/org/apache/s4/tools/Status.java  |   14 +++++----
 .../main/java/org/apache/s4/tools/ZKServer.java    |    4 +-
 .../src/main/resources/templates/newApp.README     |    7 +++-
 .../s4-tools/src/main/resources/templates/s4       |    8 ++--
 test-apps/twitter-adapter/build.gradle             |    2 +-
 10 files changed, 53 insertions(+), 27 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/7548ba53/subprojects/s4-core/src/main/java/org/apache/s4/core/S4Bootstrap.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-core/src/main/java/org/apache/s4/core/S4Bootstrap.java b/subprojects/s4-core/src/main/java/org/apache/s4/core/S4Bootstrap.java
index deb14ad..e469121 100644
--- a/subprojects/s4-core/src/main/java/org/apache/s4/core/S4Bootstrap.java
+++ b/subprojects/s4-core/src/main/java/org/apache/s4/core/S4Bootstrap.java
@@ -107,7 +107,7 @@ public class S4Bootstrap {
         // can be null
         final AppConfig appConfig = new AppConfig(appData);
 
-        String appName = appData.getSimpleField("name");
+        String appName = appConfig.getAppName();
 
         List<File> modulesLocalCopies = new ArrayList<File>();
 
@@ -120,7 +120,7 @@ public class S4Bootstrap {
 
             @Override
             public void run() {
-                // load Main class through modules classloader and start it
+                // load app class through modules classloader and start it
                 S4Bootstrap.startS4App(appConfig, parentInjector, modulesLoader);
                 signalOneAppLoaded.countDown();
             }

http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/7548ba53/subprojects/s4-core/src/main/java/org/apache/s4/core/S4Node.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-core/src/main/java/org/apache/s4/core/S4Node.java b/subprojects/s4-core/src/main/java/org/apache/s4/core/S4Node.java
index 4bd929b..907fcb9 100644
--- a/subprojects/s4-core/src/main/java/org/apache/s4/core/S4Node.java
+++ b/subprojects/s4-core/src/main/java/org/apache/s4/core/S4Node.java
@@ -39,7 +39,7 @@ public class S4Node {
 
     }
 
-    private static void startNode(S4NodeArgs mainArgs) throws InterruptedException, IOException
{
+    private static void startNode(S4NodeArgs nodeArgs) throws InterruptedException, IOException
{
         Thread.setDefaultUncaughtExceptionHandler(new UncaughtExceptionHandler() {
 
             @Override
@@ -50,7 +50,7 @@ public class S4Node {
         });
 
         Injector injector = Guice.createInjector(new Module[] { new BaseModule(Resources.getResource(
-                "default.s4.base.properties").openStream(), mainArgs.clusterName) });
+                "default.s4.base.properties").openStream(), nodeArgs.clusterName) });
         S4Bootstrap bootstrap = injector.getInstance(S4Bootstrap.class);
         try {
             bootstrap.start(injector);

http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/7548ba53/subprojects/s4-core/src/main/java/org/apache/s4/core/util/AppConfig.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-core/src/main/java/org/apache/s4/core/util/AppConfig.java b/subprojects/s4-core/src/main/java/org/apache/s4/core/util/AppConfig.java
index 9bd09c7..a31ff17 100644
--- a/subprojects/s4-core/src/main/java/org/apache/s4/core/util/AppConfig.java
+++ b/subprojects/s4-core/src/main/java/org/apache/s4/core/util/AppConfig.java
@@ -137,17 +137,23 @@ public class AppConfig {
         }
 
         public Builder customModulesNames(List<String> customModulesNames) {
-            config.customModulesNames = customModulesNames;
+            if (customModulesNames != null) {
+                config.customModulesNames = customModulesNames;
+            }
             return this;
         }
 
         public Builder customModulesURIs(List<String> customModulesURIs) {
-            config.customModulesURIs = customModulesURIs;
+            if (customModulesURIs != null) {
+                config.customModulesURIs = customModulesURIs;
+            }
             return this;
         }
 
         public Builder namedParameters(Map<String, String> namedParameters) {
-            config.namedParameters = namedParameters;
+            if (namedParameters != null) {
+                config.namedParameters = namedParameters;
+            }
             return this;
         }
 

http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/7548ba53/subprojects/s4-core/src/test/java/org/apache/s4/deploy/prodcon/TestProducerConsumer.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-core/src/test/java/org/apache/s4/deploy/prodcon/TestProducerConsumer.java
b/subprojects/s4-core/src/test/java/org/apache/s4/deploy/prodcon/TestProducerConsumer.java
index b48d371..31c6a34 100644
--- a/subprojects/s4-core/src/test/java/org/apache/s4/deploy/prodcon/TestProducerConsumer.java
+++ b/subprojects/s4-core/src/test/java/org/apache/s4/deploy/prodcon/TestProducerConsumer.java
@@ -147,7 +147,7 @@ public class TestProducerConsumer {
                 "localhost:2181");
 
         // that may be a bit long to complete...
-        Assert.assertTrue(signalConsumptionComplete.await(30, TimeUnit.SECONDS));
+        Assert.assertTrue(signalConsumptionComplete.await(40, TimeUnit.SECONDS));
 
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/7548ba53/subprojects/s4-tools/src/main/java/org/apache/s4/tools/Deploy.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-tools/src/main/java/org/apache/s4/tools/Deploy.java b/subprojects/s4-tools/src/main/java/org/apache/s4/tools/Deploy.java
index e39d00f..69ebaa6 100644
--- a/subprojects/s4-tools/src/main/java/org/apache/s4/tools/Deploy.java
+++ b/subprojects/s4-tools/src/main/java/org/apache/s4/tools/Deploy.java
@@ -70,9 +70,10 @@ public class Deploy extends S4ArgsBase {
                 System.exit(1);
             }
 
-            File s4rToDeploy;
+            File s4rToDeploy = null;
 
             if (deployArgs.s4rPath != null) {
+                // 1. if there is an application archive, we use it
                 s4rToDeploy = new File(deployArgs.s4rPath);
                 if (!s4rToDeploy.exists()) {
                     logger.error("Specified S4R file does not exist in {}", s4rToDeploy.getAbsolutePath());
@@ -82,7 +83,10 @@ public class Deploy extends S4ArgsBase {
                             "Using specified S4R [{}], the S4R archive will not be built
from source (and corresponding parameters are ignored)",
                             s4rToDeploy.getAbsolutePath());
                 }
-            } else {
+            } else if (deployArgs.gradleBuildFile != null) {
+
+                // 2. otherwise if there is a build file, we create the S4R archive from
that
+
                 List<String> params = new ArrayList<String>();
                 // prepare gradle -P parameters, including passed gradle opts
                 params.addAll(deployArgs.gradleOpts);
@@ -103,11 +107,22 @@ public class Deploy extends S4ArgsBase {
                 } else {
                     s4rToDeploy = tmpS4R;
                 }
+            } else {
+                if (!Strings.isNullOrEmpty(deployArgs.appClass)) {
+                    // 3. otherwise if there is at least an app class specified (e.g. for
running "s4 adapter"), we use
+                    // it and won't use an S4R
+                    logger.info("No S4R path specified, nor build file specified: this assumes
the app is in the classpath");
+                } else {
+                    logger.error("You must specify an S4R file, a build file to create an
S4R from, or an appClass that will be in the classpath");
+                    System.exit(1);
+                }
+
             }
 
             DeploymentUtils.initAppConfig(
-                    new AppConfig.Builder().appName(deployArgs.appName).appURI(s4rToDeploy.toURI().toString())
-                            .customModulesNames(deployArgs.modulesClassesNames)
+                    new AppConfig.Builder().appName(deployArgs.appName)
+                            .appURI(s4rToDeploy != null ? s4rToDeploy.toURI().toString()
: null)
+                            .appClassName(deployArgs.appClass).customModulesNames(deployArgs.modulesClassesNames)
                             .customModulesURIs(deployArgs.modulesURIs)
                             .namedParameters(convertListArgsToMap(deployArgs.extraNamedParameters)).build(),
                     deployArgs.clusterName, false, deployArgs.zkConnectionString);

http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/7548ba53/subprojects/s4-tools/src/main/java/org/apache/s4/tools/Status.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-tools/src/main/java/org/apache/s4/tools/Status.java b/subprojects/s4-tools/src/main/java/org/apache/s4/tools/Status.java
index 824aa03..09ae37d 100644
--- a/subprojects/s4-tools/src/main/java/org/apache/s4/tools/Status.java
+++ b/subprojects/s4-tools/src/main/java/org/apache/s4/tools/Status.java
@@ -30,7 +30,7 @@ import org.apache.s4.comm.topology.ClusterNode;
 import org.apache.s4.comm.topology.ZNRecord;
 import org.apache.s4.comm.topology.ZNRecordSerializer;
 import org.apache.s4.comm.topology.ZkClient;
-import org.apache.s4.tools.S4ArgsBase.GradleOptsConverter;
+import org.apache.s4.core.util.AppConfig;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -219,7 +219,7 @@ public class Status extends S4ArgsBase {
 
     /**
      * show as cluster1(app1), cluster2(app2)
-     *
+     * 
      * @param clusters
      *            cluster list
      * @param clusterAppMap
@@ -298,8 +298,9 @@ public class Status extends S4ArgsBase {
         private static String getApp(String clusterName, ZkClient zkClient) {
             String appPath = "/s4/clusters/" + clusterName + "/app/s4App";
             if (zkClient.exists(appPath)) {
-                ZNRecord appRecord = zkClient.readData("/s4/clusters/" + clusterName + "/app/s4App");
-                return appRecord.getSimpleField("name");
+                AppConfig appConfig = new AppConfig((ZNRecord) zkClient.readData("/s4/clusters/"
+ clusterName
+                        + "/app/s4App"));
+                return appConfig.getAppName();
             }
             return NONE;
         }
@@ -357,8 +358,9 @@ public class Status extends S4ArgsBase {
             app.cluster = clusterName;
             try {
                 ZNRecord appRecord = zkClient.readData(appPath);
-                app.name = appRecord.getSimpleField("name");
-                app.uri = appRecord.getSimpleField("s4r_uri");
+                AppConfig appConfig = new AppConfig(appRecord);
+                app.name = appConfig.getAppName();
+                app.uri = appConfig.getAppURI();
             } catch (ZkNoNodeException e) {
                 logger.warn(appPath + " doesn't exist");
             }

http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/7548ba53/subprojects/s4-tools/src/main/java/org/apache/s4/tools/ZKServer.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-tools/src/main/java/org/apache/s4/tools/ZKServer.java b/subprojects/s4-tools/src/main/java/org/apache/s4/tools/ZKServer.java
index 4893928..580872b 100644
--- a/subprojects/s4-tools/src/main/java/org/apache/s4/tools/ZKServer.java
+++ b/subprojects/s4-tools/src/main/java/org/apache/s4/tools/ZKServer.java
@@ -107,8 +107,8 @@ public class ZKServer {
         String dataDir = new File(System.getProperty("java.io.tmpdir") + File.separator +
"tmp" + File.separator
                 + "zookeeper" + File.separator + "data").getAbsolutePath();
 
-        @Parameter(names = "-clean", description = "clean zookeeper data (arity=0) (make
sure you specify correct directories...)")
-        boolean clean = false;
+        @Parameter(names = "-clean", description = "clean zookeeper data (make sure you specify
correct directories...)")
+        boolean clean = true;
 
         @Parameter(names = "-logDir", description = "log directory")
         String logDir = new File(System.getProperty("java.io.tmpdir") + File.separator +
"tmp" + File.separator

http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/7548ba53/subprojects/s4-tools/src/main/resources/templates/newApp.README
----------------------------------------------------------------------
diff --git a/subprojects/s4-tools/src/main/resources/templates/newApp.README b/subprojects/s4-tools/src/main/resources/templates/newApp.README
index 052e303..e572737 100644
--- a/subprojects/s4-tools/src/main/resources/templates/newApp.README
+++ b/subprojects/s4-tools/src/main/resources/templates/newApp.README
@@ -32,6 +32,9 @@
  - application 2 must define an output stream with the same name (stream1)
 
  If you want to use a simple adapter process, listening to an external source, converting
incoming data into S4 events, and sending that to S4 apps, you can define
- your own app that extends the AdapterApp class. Then to start it (for instance):
- - ./s4 adapter -cluster=c1 -appClass=org.apache.s4.example.twitter.TwitterInputAdapter -p=s4.adapter.output.stream=stream1
+ your own app that extends the AdapterApp class, for instance:
+ ./s4 deploy -appClass=hello.HelloInputAdapter -p=s4.adapter.output.stream=names -cluster=c2
-appName=adapter
+ 
+ Then in order to start a node and automatically using the classpath of the current project,
use the "s4 adapter" command:
+ - ./s4 adapter -cluster=c2
 

http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/7548ba53/subprojects/s4-tools/src/main/resources/templates/s4
----------------------------------------------------------------------
diff --git a/subprojects/s4-tools/src/main/resources/templates/s4 b/subprojects/s4-tools/src/main/resources/templates/s4
index 45c5efd..69e5c78 100644
--- a/subprojects/s4-tools/src/main/resources/templates/s4
+++ b/subprojects/s4-tools/src/main/resources/templates/s4
@@ -3,12 +3,12 @@
 case "$1" in
 "adapter")
     # we need something different in order to pass the classpath of the current project
-    # you must specify: appClassName (extends AdapterApp class) , cluster name, output stream
name
-    # current syntax is not yet straightforward:
-    # example : ./s4 adapter -cluster=c1 -appClass=org.apache.s4.example.twitter.TwitterInputAdapter
-p=s4.adapter.output.stream:s1
+    # you must specify: cluster name (and -zk param if different from localhost:2181) 
+    
+    # example : ./s4 adapter -cluster=c1
 	shift 1
 	<s4_install_dir>/gradlew cp
-	java -cp `cat classpath.txt` org.apache.s4.core.Main $@
+	java -cp `cat classpath.txt` org.apache.s4.core.S4Node $@
 ;;
 *)
 	echo "calling referenced s4 script : <s4_script_path>"

http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/7548ba53/test-apps/twitter-adapter/build.gradle
----------------------------------------------------------------------
diff --git a/test-apps/twitter-adapter/build.gradle b/test-apps/twitter-adapter/build.gradle
index 9b6a2d4..9272fa1 100644
--- a/test-apps/twitter-adapter/build.gradle
+++ b/test-apps/twitter-adapter/build.gradle
@@ -55,7 +55,7 @@ apply plugin: 'eclipse'
 apply plugin: 'idea'
 apply plugin:'application'
 
-mainClassName = "org.apache.s4.core.Main"
+mainClassName = "org.apache.s4.core.S4Node"
 
 /* The app classname is set automatically from the source files. */
 def appClassname = ''


Mime
View raw message