incubator-s4-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mmo...@apache.org
Subject [45/50] [abbrv] git commit: S4-130 Update and document parameters initialization from nodes - applicable to csv metrics logging configuration - also: license update for some xml files
Date Sat, 01 Jun 2013 11:33:35 GMT
S4-130 Update and document parameters initialization from nodes
- applicable to csv metrics logging configuration
- also: license update for some xml files


Project: http://git-wip-us.apache.org/repos/asf/incubator-s4/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s4/commit/394622b2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s4/tree/394622b2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s4/diff/394622b2

Branch: refs/heads/master
Commit: 394622b224ebc8eabf26311466cc8f06f27e503e
Parents: 2a5f92c
Author: Matthieu Morel <mmorel@apache.org>
Authored: Fri Mar 15 17:28:59 2013 +0100
Committer: Matthieu Morel <mmorel@apache.org>
Committed: Fri Mar 15 17:29:51 2013 +0100

----------------------------------------------------------------------
 .../src/main/java/org/apache/s4/core/S4Node.java   |   21 ++++-
 .../java/org/apache/s4/core/util/ParsingUtils.java |   66 +++++++++++++++
 subprojects/s4-edsl/src/main/diezel/s4/s4-impl.xml |   16 ++++
 subprojects/s4-edsl/src/main/diezel/s4/s4.xml      |   16 ++++
 .../src/main/java/org/apache/s4/tools/Deploy.java  |   42 +--------
 website/content/doc/0.6.0/metrics.md               |    9 ++-
 6 files changed, 126 insertions(+), 44 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/394622b2/subprojects/s4-core/src/main/java/org/apache/s4/core/S4Node.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-core/src/main/java/org/apache/s4/core/S4Node.java b/subprojects/s4-core/src/main/java/org/apache/s4/core/S4Node.java
index 7aafe90..fb8f6e4 100644
--- a/subprojects/s4-core/src/main/java/org/apache/s4/core/S4Node.java
+++ b/subprojects/s4-core/src/main/java/org/apache/s4/core/S4Node.java
@@ -19,16 +19,20 @@ package org.apache.s4.core;
 
 import java.io.IOException;
 import java.lang.Thread.UncaughtExceptionHandler;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
 
 import org.apache.s4.core.util.ArchiveFetchException;
 import org.apache.s4.core.util.ParametersInjectionModule;
+import org.apache.s4.core.util.ParsingUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.beust.jcommander.JCommander;
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.Parameters;
-import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
 import com.google.common.io.Resources;
 import com.google.inject.Guice;
 import com.google.inject.Injector;
@@ -68,10 +72,15 @@ public class S4Node {
             }
         });
 
+        // inject parameter from the command line, including zk string
+        Map<String, String> inlineParameters = Maps.newHashMap(ParsingUtils
+                .convertListArgsToMap(nodeArgs.extraNamedParameters));
+        inlineParameters.put("s4.cluster.zk_address", nodeArgs.zkConnectionString);
+
         Injector injector = Guice.createInjector(Modules.override(
                 new BaseModule(Resources.getResource("default.s4.base.properties").openStream(),
nodeArgs.clusterName))
-                .with(new ParametersInjectionModule(ImmutableMap.of("s4.cluster.zk_address",
-                        nodeArgs.zkConnectionString))));
+                .with(new ParametersInjectionModule(inlineParameters)));
+
         S4Bootstrap bootstrap = injector.getInstance(S4Bootstrap.class);
         try {
             bootstrap.start(injector);
@@ -96,5 +105,11 @@ public class S4Node {
         @Parameter(names = "-zk", description = "Zookeeper connection string", required =
false)
         String zkConnectionString = "localhost:2181";
 
+        @Parameter(names = { "-namedStringParameters", "-p" }, description = "Comma-separated
list of "
+                + "inline configuration parameters, taking precedence over homonymous configuration
parameters from "
+                + "configuration files. Syntax: '-p=name1=value1,name2=value2 '. "
+                + "NOTE: application parameters should be injected in the application configuration/deployment
step."
+                + "Only parameters relevant to the node should be injected here, e.g. metrics
logging configuration", hidden = false, converter = ParsingUtils.InlineConfigParameterConverter.class)
+        List<String> extraNamedParameters = new ArrayList<String>();
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/394622b2/subprojects/s4-core/src/main/java/org/apache/s4/core/util/ParsingUtils.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-core/src/main/java/org/apache/s4/core/util/ParsingUtils.java b/subprojects/s4-core/src/main/java/org/apache/s4/core/util/ParsingUtils.java
new file mode 100644
index 0000000..37e18de
--- /dev/null
+++ b/subprojects/s4-core/src/main/java/org/apache/s4/core/util/ParsingUtils.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.s4.core.util;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.beust.jcommander.IStringConverter;
+import com.beust.jcommander.internal.Maps;
+
+/**
+ * Utilities for parsing command line parameters
+ */
+public class ParsingUtils {
+
+    public static Map<String, String> convertListArgsToMap(List<String> args)
{
+        Map<String, String> result = Maps.newHashMap();
+        for (String arg : args) {
+            String[] split = arg.split("[=]");
+            if (!(split.length == 2)) {
+                throw new RuntimeException("Invalid args: " + Arrays.toString(args.toArray(new
String[] {})));
+            }
+            result.put(split[0], split[1]);
+        }
+        return result;
+    }
+
+    public static class InlineConfigParameterConverter implements IStringConverter<String>
{
+
+        private static Logger logger = LoggerFactory.getLogger(InlineConfigParameterConverter.class);
+
+        @Override
+        public String convert(String arg) {
+            Pattern parameterPattern = Pattern.compile("(\\S+=\\S+)");
+            logger.info("processing inline configuration parameter {}", arg);
+            Matcher parameterMatcher = parameterPattern.matcher(arg);
+            if (!parameterMatcher.find()) {
+                throw new IllegalArgumentException("Cannot understand parameter " + arg);
+            }
+            return parameterMatcher.group(1);
+        }
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/394622b2/subprojects/s4-edsl/src/main/diezel/s4/s4-impl.xml
----------------------------------------------------------------------
diff --git a/subprojects/s4-edsl/src/main/diezel/s4/s4-impl.xml b/subprojects/s4-edsl/src/main/diezel/s4/s4-impl.xml
index 796a781..e562800 100644
--- a/subprojects/s4-edsl/src/main/diezel/s4/s4-impl.xml
+++ b/subprojects/s4-edsl/src/main/diezel/s4/s4-impl.xml
@@ -1,4 +1,20 @@
 <?xml version="1.0" encoding="UTF-8"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one
+       or more contributor license agreements.  See the NOTICE file
+       distributed with this work for additional information
+       regarding copyright ownership.  The ASF licenses this file
+       to you under the Apache License, Version 2.0 (the
+       "License"); you may not use this file except in compliance
+       with the License.  You may obtain a copy of the License at
+
+         http://www.apache.org/licenses/LICENSE-2.0
+
+       Unless required by applicable law or agreed to in writing,
+       software distributed under the License is distributed on an
+       "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+       KIND, either express or implied.  See the License for the
+       specific language governing permissions and limitations
+       under the License.-->
 <diezelImplementation xmlns="http://diezel.ericaro.net/2.0.0/">
 	<package>org.apache.s4.edsl</package>
 	<name>Builder</name>

http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/394622b2/subprojects/s4-edsl/src/main/diezel/s4/s4.xml
----------------------------------------------------------------------
diff --git a/subprojects/s4-edsl/src/main/diezel/s4/s4.xml b/subprojects/s4-edsl/src/main/diezel/s4/s4.xml
index 90764b2..c019141 100644
--- a/subprojects/s4-edsl/src/main/diezel/s4/s4.xml
+++ b/subprojects/s4-edsl/src/main/diezel/s4/s4.xml
@@ -1,4 +1,20 @@
 <?xml version="1.0" encoding="UTF-8"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one
+       or more contributor license agreements.  See the NOTICE file
+       distributed with this work for additional information
+       regarding copyright ownership.  The ASF licenses this file
+       to you under the Apache License, Version 2.0 (the
+       "License"); you may not use this file except in compliance
+       with the License.  You may obtain a copy of the License at
+
+         http://www.apache.org/licenses/LICENSE-2.0
+
+       Unless required by applicable law or agreed to in writing,
+       software distributed under the License is distributed on an
+       "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+       KIND, either express or implied.  See the License for the
+       specific language governing permissions and limitations
+       under the License. -->
 <diezel xmlns="http://diezel.ericaro.net/2.0.0/">
 	<package>org.apache.s4.edsl</package>
 	<name>S4DSL</name>

http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/394622b2/subprojects/s4-tools/src/main/java/org/apache/s4/tools/Deploy.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-tools/src/main/java/org/apache/s4/tools/Deploy.java b/subprojects/s4-tools/src/main/java/org/apache/s4/tools/Deploy.java
index f78b866..695faf3 100644
--- a/subprojects/s4-tools/src/main/java/org/apache/s4/tools/Deploy.java
+++ b/subprojects/s4-tools/src/main/java/org/apache/s4/tools/Deploy.java
@@ -23,13 +23,11 @@ import java.net.URI;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
-import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 
 import org.I0Itec.zkclient.ZkClient;
 import org.apache.s4.comm.topology.ZNRecordSerializer;
 import org.apache.s4.core.util.AppConfig;
+import org.apache.s4.core.util.ParsingUtils;
 import org.apache.s4.deploy.DeploymentUtils;
 import org.gradle.tooling.BuildLauncher;
 import org.gradle.tooling.GradleConnector;
@@ -37,10 +35,8 @@ import org.gradle.tooling.ProgressListener;
 import org.gradle.tooling.ProjectConnection;
 import org.slf4j.LoggerFactory;
 
-import com.beust.jcommander.IStringConverter;
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.Parameters;
-import com.beust.jcommander.internal.Maps;
 import com.google.common.base.Strings;
 
 /**
@@ -86,8 +82,8 @@ public class Deploy extends S4ArgsBase {
                             .appURI(s4rURI == null ? null : s4rURI.toString())
                             .customModulesNames(deployArgs.modulesClassesNames)
                             .customModulesURIs(deployArgs.modulesURIs).appClassName(deployArgs.appClass)
-                            .namedParameters(convertListArgsToMap(deployArgs.extraNamedParameters)).build(),
-                    deployArgs.clusterName, false, deployArgs.zkConnectionString);
+                            .namedParameters(ParsingUtils.convertListArgsToMap(deployArgs.extraNamedParameters))
+                            .build(), deployArgs.clusterName, false, deployArgs.zkConnectionString);
             // Explicitly shutdown the JVM since Gradle leaves non-daemon threads running
that delay the termination
             if (!deployArgs.testMode) {
                 System.exit(0);
@@ -98,18 +94,6 @@ public class Deploy extends S4ArgsBase {
 
     }
 
-    private static Map<String, String> convertListArgsToMap(List<String> args)
{
-        Map<String, String> result = Maps.newHashMap();
-        for (String arg : args) {
-            String[] split = arg.split("[=]");
-            if (!(split.length == 2)) {
-                throw new RuntimeException("Invalid args: " + Arrays.toString(args.toArray(new
String[] {})));
-            }
-            result.put(split[0], split[1]);
-        }
-        return result;
-    }
-
     @Parameters(commandNames = "s4 deploy", commandDescription = "Package and deploy application
to S4 cluster", separators = "=")
     static class DeployAppArgs extends S4ArgsBase {
 
@@ -137,7 +121,7 @@ public class Deploy extends S4ArgsBase {
         @Parameter(names = { "-modulesClasses", "-emc", "-mc" }, description = "Fully qualified
class names of custom modules")
         List<String> modulesClassesNames = new ArrayList<String>();
 
-        @Parameter(names = { "-namedStringParameters", "-p" }, description = "Comma-separated
list of inline configuration parameters, taking precedence over homonymous configuration parameters
from configuration files. Syntax: '-p=name1=value1,name2=value2 '", hidden = false, converter
= InlineConfigParameterConverter.class)
+        @Parameter(names = { "-namedStringParameters", "-p" }, description = "Comma-separated
list of inline configuration parameters, taking precedence over homonymous configuration parameters
from configuration files. Syntax: '-p=name1=value1,name2=value2 '", hidden = false, converter
= ParsingUtils.InlineConfigParameterConverter.class)
         List<String> extraNamedParameters = new ArrayList<String>();
 
         @Parameter(names = "-testMode", description = "Special mode for regression testing",
hidden = true)
@@ -147,24 +131,6 @@ public class Deploy extends S4ArgsBase {
         boolean debug = false;
     }
 
-    /**
-     * Parameters parsing utility.
-     * 
-     */
-    public static class InlineConfigParameterConverter implements IStringConverter<String>
{
-
-        @Override
-        public String convert(String arg) {
-            Pattern parameterPattern = Pattern.compile("(\\S+=\\S+)");
-            logger.info("processing inline configuration parameter {}", arg);
-            Matcher parameterMatcher = parameterPattern.matcher(arg);
-            if (!parameterMatcher.find()) {
-                throw new IllegalArgumentException("Cannot understand parameter " + arg);
-            }
-            return parameterMatcher.group(1);
-        }
-    }
-
     static class ExecGradle {
 
         public static void exec(File buildFile, String taskName, String[] params, boolean
debug) throws Exception {

http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/394622b2/website/content/doc/0.6.0/metrics.md
----------------------------------------------------------------------
diff --git a/website/content/doc/0.6.0/metrics.md b/website/content/doc/0.6.0/metrics.md
index 70d08d5..cdc2469 100644
--- a/website/content/doc/0.6.0/metrics.md
+++ b/website/content/doc/0.6.0/metrics.md
@@ -24,14 +24,14 @@ You can also monitor your own PEs. Simply add new probes (`Meter`, `Gauge`,
etc.
 
 By default, metrics are exposed by each node through JMX.
 
-The `s4.metrics.config` parameter enables periodic dumps of aggregated statistics to the
**console** or to **files** in csv format. This parameter is specified as an application parameter,
and must match the following regular expression: 
+The `s4.metrics.config` parameter enables periodic dumps of aggregated statistics to the
**console** or to **files** in csv format. This parameter is specified as a node or application
parameter [^1], and must match the following regular expression: 
 
 	(csv:.+|console):(\d+):(DAYS|HOURS|MICROSECONDS|MILLISECONDS|MINUTES|NANOSECONDS|SECONDS)
 
 Examples:
 	
 	# dump metrics to csv files to /path/to/directory every 10 seconds
-	# (recommendation: use a clean directory)
+	# (recommendation: use a different and clean directory for each of the nodes)
 	csv:/path/to/directory:10:SECONDS
 	
 	# dump metrics to the console every minute
@@ -39,4 +39,7 @@ Examples:
 	
 	
 
-Reporting to Ganglia or Graphite is not provided out of the box with S4, but it's quite easy
to add. You simply have to add the corresponding dependencies to your project and enable reporting
to these systems during the initialization of your application. See the [metrics](http://metrics.codahale.com)
documentation for more information.
\ No newline at end of file
+Reporting to Ganglia or Graphite is not provided out of the box with S4, but it's quite easy
to add. You simply have to add the corresponding dependencies to your project and enable reporting
to these systems during the initialization of your application. See the [metrics](http://metrics.codahale.com)
documentation for more information.
+
+
+[^1]: csv-based metrics logging are usually specified as a node configuration, and must point
to a clean directory, different for each node. Otherwise the metrics logging system will try
to create multiple new files with the same name, and depending on your setup, that may not
be possible.
\ No newline at end of file


Mime
View raw message