incubator-s4-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mmo...@apache.org
Subject svn commit: r1488543 [1/5] - in /incubator/s4/site: ./ contrib/ doc/0.6.0/ doc/0.6.0/application_dependencies/ doc/0.6.0/configuration/ doc/0.6.0/dev_tips/ doc/0.6.0/event_dispatch/ doc/0.6.0/event_injection/ doc/0.6.0/fault_tolerance/ doc/0.6.0/metric...
Date Sat, 01 Jun 2013 16:37:45 GMT
Author: mmorel
Date: Sat Jun  1 16:37:44 2013
New Revision: 1488543

URL: http://svn.apache.org/r1488543
Log:
update for 0.6.0 release

Modified:
    incubator/s4/site/contrib/index.html
    incubator/s4/site/doc/0.6.0/application_dependencies/index.html
    incubator/s4/site/doc/0.6.0/configuration/index.html
    incubator/s4/site/doc/0.6.0/dev_tips/index.html
    incubator/s4/site/doc/0.6.0/event_dispatch/index.html
    incubator/s4/site/doc/0.6.0/event_injection/index.html
    incubator/s4/site/doc/0.6.0/fault_tolerance/index.html
    incubator/s4/site/doc/0.6.0/index.html
    incubator/s4/site/doc/0.6.0/metrics/index.html
    incubator/s4/site/doc/0.6.0/overview/index.html
    incubator/s4/site/doc/0.6.0/recommended_practices/index.html
    incubator/s4/site/doc/0.6.0/tools/index.html
    incubator/s4/site/doc/0.6.0/twitter_trending_example/index.html
    incubator/s4/site/doc/0.6.0/walkthrough/index.html
    incubator/s4/site/download/index.html
    incubator/s4/site/index.html
    incubator/s4/site/style/ie.css
    incubator/s4/site/style/nav.css
    incubator/s4/site/style/print.css
    incubator/s4/site/style/screen.css
    incubator/s4/site/style/style.css
    incubator/s4/site/team/index.html

Modified: incubator/s4/site/contrib/index.html
URL: http://svn.apache.org/viewvc/incubator/s4/site/contrib/index.html?rev=1488543&r1=1488542&r2=1488543&view=diff
==============================================================================
--- incubator/s4/site/contrib/index.html (original)
+++ incubator/s4/site/contrib/index.html Sat Jun  1 16:37:44 2013
@@ -1,16 +1,17 @@
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<!DOCTYPE html>
 <html>
-<head>
-<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
-<title>S4: Contribute</title>
-<meta content="A general-purpose distributed stream computing platform" name="description">
-<link href="../style/screen.css" media="screen" rel="stylesheet" type="text/css">
-<link href="../style/print.css" media="print" rel="stylesheet" type="text/css">
-<!--[if lt IE 9]>
+  <head>
+    <title>S4: Contribute</title>
+    <meta content="text/html; charset=utf-8" http-equiv="Content-Type">
+    <meta content="A general-purpose distributed stream computing platform" name="description">
+    <link href="../style/screen.css" media="screen" rel="stylesheet" type="text/css">
+    <link href="../style/print.css" media="print" rel="stylesheet" type="text/css">
+    <!--[if lt IE 9]>
       <link href="../style/ie.css" media="screen" rel="stylesheet" type="text/css">
-    <![endif]--><link href="../style/style.css" rel="stylesheet" type="text/css">
-<link href="../style/nav.css" rel="stylesheet" type="text/css">
-<script type="text/javascript">
+    <![endif]-->
+    <link href="../style/style.css" rel="stylesheet" type="text/css">
+    <link href="../style/nav.css" rel="stylesheet" type="text/css">
+    <script type="text/javascript">
         var _gaq = _gaq || [];
         _gaq.push(['_setAccount', 'UA-19490961-1']);
         _gaq.push(['_setDomainName', 'incubator.apache.org']);
@@ -20,111 +21,116 @@
           ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
           var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
         })();
-      </script><link href="../style/coderay.css" rel="stylesheet" type="text/css">
-<link href="../style/pygmentize.css" rel="stylesheet" type="text/css">
-</head>
-<body>
+      </script>
+    <link href="../style/coderay.css" rel="stylesheet" type="text/css">
+    <link href="../style/pygmentize.css" rel="stylesheet" type="text/css">
+  </head>
+  <body>
     <div id="header">
       <div class="container">
         <div id="logo">
           <a href="../">
-            <img src="../images/s4_test.png"></a>
+            <img src="../images/s4_test.png">
+          </a>
+        </div>
+        <div id="navbar">
+          <ul id="nav">
+            <li>
+              <a href="../">home</a>
+            </li>
+            <li>
+              <a href="../doc/0.6.0">doc</a>
+            </li>
+            <li>
+              <a href="http://git-wip-us.apache.org/repos/asf?p=incubator-s4.git" onclick="_gaq.push(['_trackEvent', 'External', 'Apache Git', 'http://github.com/s4']);">code</a>
+            </li>
+            <li>
+              <a href="http://people.apache.org/~mmorel/apache-s4-0.6.0-incubating-doc/javadoc/">API</a>
+            </li>
+            <li>
+              <a href=".">get involved</a>
+            </li>
+            <li>
+              <a href="../team">team</a>
+            </li>
+            <li>
+              <a href="../download">download</a>
+            </li>
+          </ul>
         </div>
-        <div id="navbar"><ul id="nav">
-<li>
-    <a href="../">home</a>
-  </li>
-  <li>
-    <a href="https://cwiki.apache.org/confluence/display/S4/S4+Wiki">doc [0.5]</a>
-  </li>
-  <li>
-    <a href="http://git-wip-us.apache.org/repos/asf?p=incubator-s4.git" onclick="_gaq.push(['_trackEvent', 'External', 'Apache Git', 'http://github.com/s4']);">code</a>
-  </li>
-  <li>
-    <a href="http://people.apache.org/~mmorel/apache-s4-0.5.0-incubating-doc/javadoc/">API</a>
-  </li>
-  <li>
-    <a href=".">get involved</a>
-  </li>
-  <li>
-    <a href="../team">team</a>
-  </li>
-  <li>
-    <a href="../download">download</a>
-  </li>
-</ul></div>
       </div>
     </div>
     <div id="wrapper">
       <div class="container" id="container">
-<div class="span-18 colborder" id="main">
-  <!-- collaboration -->
-  <h2 class="secheader">collaboration model</h2>
-  <p>The project is maintain by volunteers and is open to anybody interested in contributing to the project. To contribute, you can write code, review code, report bugs, suggest ideas, propose design changes, write documentation, answer questions in the mailing list, and much more. The more you contribute the more you can make a difference.</p>
-  <!-- version control -->
-  <h2 class="secheader">version control</h2>
-  <p>S4 sources are available through <b>git</b>. Most development is done on the <i>piper</i> branch</p>
-  <p>See a summary <a href="https://git-wip-us.apache.org/repos/asf?p=incubator-s4.git;a=summary">here.</a></p>
-  <!-- access to the repository -->
-  <h2 class="secheader">access to the repository</h2>
-  <ul>
-<li>as a committer: <pre>git clone https://user_name@git-wip-us.apache.org/repos/asf/incubator-s4.git</pre>
+        <div class="span-18 colborder" id="main">
+          <!-- collaboration -->
+          <h2 class="secheader">collaboration model</h2>
+          <p>The project is maintain by volunteers and is open to anybody interested in contributing to the project. To contribute, you can write code, review code, report bugs, suggest ideas, propose design changes, write documentation, answer questions in the mailing list, and much more. The more you contribute the more you can make a difference.</p>
+          <!-- version control -->
+          <h2 class="secheader">version control</h2>
+          <p>S4 sources are available through <b>git</b>. Most development is done on the <i>piper</i> branch</p>
+          <p>See a summary <a href="https://git-wip-us.apache.org/repos/asf?p=incubator-s4.git;a=summary">here.</a></p>
+          <!-- access to the repository -->
+          <h2 class="secheader">access to the repository</h2>
+          <ul>
+            <li>as a committer: <pre>git clone https://user_name@git-wip-us.apache.org/repos/asf/incubator-s4.git</pre>
 </li>
-    <li>as a user or contributor: <pre>git clone http://git-wip-us.apache.org/repos/asf/incubator-s4.git</pre> (though since June 2012, you may have to try with <b>https</b> :<pre>git clone https://git-wip-us.apache.org/repos/asf/incubator-s4.git</pre> , even though this is not reflected yet on the Apache git documentation).</li>
-  </ul>
-<ul></ul>
-<p>For more information, have a look there: <a href="https://git-wip-us.apache.org/">https://git-wip-us.apache.org/</a></p>
-  <!-- eclipse project -->
-  <h2 class="secheader">creating an eclipse project</h2>
-  <p>type: </p>
+            <li>as a user or contributor: <pre>git clone http://git-wip-us.apache.org/repos/asf/incubator-s4.git</pre> (though since June 2012, you may have to try with <b>https</b> :<pre>git clone https://git-wip-us.apache.org/repos/asf/incubator-s4.git</pre> , even though this is not reflected yet on the Apache git documentation).</li>
+          </ul>
+          <ul></ul>
+          <p>For more information, have a look there: <a href="https://git-wip-us.apache.org/">https://git-wip-us.apache.org/</a></p>
+          <!-- eclipse project -->
+          <h2 class="secheader">creating an eclipse project</h2>
+          <p>type: </p>
 <pre>./gradlew eclipse</pre> You can now import the project into Eclipse. To learn more, visit the <b><a href="http://www.gradle.org">Gradle website</a></b>.
-  <!-- submission guidelines -->
-  <h2 class="secheader">submission guidelines</h2>
-  <p>Before you start, send a message to the S4 developer <b><a href="https://cwiki.apache.org/confluence/display/S4/S4+Apache+mailing+lists">mailing list</a></b>, or file a bug report in <b><a href="https://issues.apache.org/jira/browse/S4">Jira</a></b>. Describe your proposed changes and check that they fit in with what others are doing and have planned for the project.</p>
-  <!-- modifying the source code -->
-  <h2 class="secheader">modifying the source code</h2>
-  <p>Here are a few things to keep in mind:</p>
-  <ol>
-<li>All public classes, interfaces, and methods should have informative <b><a href="http://www.oracle.com/technetwork/java/javase/documentation/index-137868.html">Javadoc comments</a></b>.</li>
-    </ol>
+          <!-- submission guidelines -->
+          <h2 class="secheader">submission guidelines</h2>
+          <p>Before you start, send a message to the S4 developer <b><a href="https://cwiki.apache.org/confluence/display/S4/S4+Apache+mailing+lists">mailing list</a></b>, or file a bug report in <b><a href="https://issues.apache.org/jira/browse/S4">Jira</a></b>. Describe your proposed changes and check that they fit in with what others are doing and have planned for the project.</p>
+          <!-- modifying the source code -->
+          <h2 class="secheader">modifying the source code</h2>
+          <p>Here are a few things to keep in mind:</p>
+          <ol>
+            <li>All public classes, interfaces, and methods should have informative <b><a href="http://www.oracle.com/technetwork/java/javase/documentation/index-137868.html">Javadoc comments</a></b>.</li>
+            </ol>
 <ul>
-<li>BAD: <span class="high">getCondensedLength()</span>: Gets condensed length.</li>
-      <li>BETTER: <span class="high">getCondensedLength()</span>: Number of characters in the string, excluding non alpha-numeric characters.</li>
-    </ul>
-<li>Do not use @author tags.</li>
-    <li>Code should be written according to <b><a href="http://www.oracle.com/technetwork/java/codeconv-138413.html">Java's conventions</a></b>.</li>
-    <li>Formatting: (from S4 0.5, see "automatic formatting" below)</li>
-    <ul>
-<li>Indent four spaces per level, not two or six or eight, etc... four.</li>
-      <li>No tabs for indentation, spaces only.</li>
-    </ul>
-<li>Contributions should pass existing unit tests.</li>
-    <li>New unit tests should be provided to demonstrate bugs and fixes. <b><a href="http://www.junit.org">Junit</a></b> is our test framework.</li>
-  
-  <!-- automatic source code formatting -->
-  <h2 class="secheader">automatic source code formatting</h2>
-  <p>You must configure your IDE to follow the formatting guidelines. This is needed to get clean diffs.</p>
-  <p>To automatically format code in Eclipse:</p>
-  <ol>
-<li>preferences --&gt; Java --&gt; Code style --&gt; Formatter , then import s4/config/eclipse/s4-eclipse-format.xml</li>
-    <li>preferences --&gt; Java --&gt; Editor --&gt; Save actions --&gt; format source code,  format all lines</li>
-  </ol>
-<p>If you don't use Eclipse, you may still use Eclipse's code formatter in a <a href="http://www.peterfriese.de/formatting-your-code-using-the-eclipse-code-formatter/">standalone mode.</a></p>
-</div>
-<div class="span-5 last" id="contrib_sidebar">
-  <div id="participate">
-    <h2 class="secheader">participate</h2>
-    <ul class="large greybar">
-<li>Join the <a href="https://cwiki.apache.org/confluence/display/S4/S4+Apache+mailing+lists" onclick="_gaq.push(['_trackEvent', 'External', 'Forum', 'Contrib 2 - https://cwiki.apache.org/confluence/display/S4/S4+Apache+mailing+lists']);">forum</a>
+              <li>BAD: <span class="high">getCondensedLength()</span>: Gets condensed length.</li>
+              <li>BETTER: <span class="high">getCondensedLength()</span>: Number of characters in the string, excluding non alpha-numeric characters.</li>
+            </ul>
+            <li>Do not use @author tags.</li>
+            <li>Code should be written according to <b><a href="http://www.oracle.com/technetwork/java/codeconv-138413.html">Java's conventions</a></b>.</li>
+            <li>Formatting: (from S4 0.5, see "automatic formatting" below)</li>
+            <ul>
+              <li>Indent four spaces per level, not two or six or eight, etc... four.</li>
+              <li>No tabs for indentation, spaces only.</li>
+            </ul>
+            <li>Contributions should pass existing unit tests.</li>
+            <li>New unit tests should be provided to demonstrate bugs and fixes. <b><a href="http://www.junit.org">Junit</a></b> is our test framework.</li>
+          
+          <!-- automatic source code formatting -->
+          <h2 class="secheader">automatic source code formatting</h2>
+          <p>You must configure your IDE to follow the formatting guidelines. This is needed to get clean diffs.</p>
+          <p>To automatically format code in Eclipse:</p>
+          <ol>
+            <li>preferences --&gt; Java --&gt; Code style --&gt; Formatter , then import s4/config/eclipse/s4-eclipse-format.xml</li>
+            <li>preferences --&gt; Java --&gt; Editor --&gt; Save actions --&gt; format source code,  format all lines</li>
+          </ol>
+          <p>If you don't use Eclipse, you may still use Eclipse's code formatter in a <a href="http://www.peterfriese.de/formatting-your-code-using-the-eclipse-code-formatter/">standalone mode.</a></p>
+        </div>
+        <div class="span-5 last" id="contrib_sidebar">
+          <div id="participate">
+            <h2 class="secheader">participate</h2>
+            <ul class="large greybar">
+              <li>Join the <a href="https://cwiki.apache.org/confluence/display/S4/S4+Apache+mailing+lists" onclick="_gaq.push(['_trackEvent', 'External', 'Forum', 'Contrib 2 - https://cwiki.apache.org/confluence/display/S4/S4+Apache+mailing+lists']);">forum</a>
 </li>
-    </ul>
-</div>
-  <div id="twitter_follow">
-    <a href="http://www.twitter.com/s4project" onclick="_gaq.push(['_trackEvent', 'External', 'Twitter', 'Contrib - http://www.twitter.com/s4project']);">
-      <img alt="Follow s4project on Twitter" src="http://twitter-badges.s3.amazonaws.com/follow_us-b.png"></a>
-  </div>
-</div>
-</div>
+            </ul>
+          </div>
+          <div id="twitter_follow">
+            <a href="http://www.twitter.com/s4project" onclick="_gaq.push(['_trackEvent', 'External', 'Twitter', 'Contrib - http://www.twitter.com/s4project']);">
+              <img alt="Follow s4project on Twitter" src="http://twitter-badges.s3.amazonaws.com/follow_us-b.png">
+            </a>
+          </div>
+        </div>
+      </div>
     </div>
     <div id="footer">
       <div class="container">

Modified: incubator/s4/site/doc/0.6.0/application_dependencies/index.html
URL: http://svn.apache.org/viewvc/incubator/s4/site/doc/0.6.0/application_dependencies/index.html?rev=1488543&r1=1488542&r2=1488543&view=diff
==============================================================================
--- incubator/s4/site/doc/0.6.0/application_dependencies/index.html (original)
+++ incubator/s4/site/doc/0.6.0/application_dependencies/index.html Sat Jun  1 16:37:44 2013
@@ -1,16 +1,17 @@
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<!DOCTYPE html>
 <html>
-<head>
-<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
-<title>S4: Adding application dependencies</title>
-<meta content="A general-purpose distributed stream computing platform" name="description">
-<link href="../../../style/screen.css" media="screen" rel="stylesheet" type="text/css">
-<link href="../../../style/print.css" media="print" rel="stylesheet" type="text/css">
-<!--[if lt IE 9]>
+  <head>
+    <title>S4: Adding application dependencies</title>
+    <meta content="text/html; charset=utf-8" http-equiv="Content-Type">
+    <meta content="A general-purpose distributed stream computing platform" name="description">
+    <link href="../../../style/screen.css" media="screen" rel="stylesheet" type="text/css">
+    <link href="../../../style/print.css" media="print" rel="stylesheet" type="text/css">
+    <!--[if lt IE 9]>
       <link href="../../../style/ie.css" media="screen" rel="stylesheet" type="text/css">
-    <![endif]--><link href="../../../style/style.css" rel="stylesheet" type="text/css">
-<link href="../../../style/nav.css" rel="stylesheet" type="text/css">
-<script type="text/javascript">
+    <![endif]-->
+    <link href="../../../style/style.css" rel="stylesheet" type="text/css">
+    <link href="../../../style/nav.css" rel="stylesheet" type="text/css">
+    <script type="text/javascript">
         var _gaq = _gaq || [];
         _gaq.push(['_setAccount', 'UA-19490961-1']);
         _gaq.push(['_setDomainName', 'incubator.apache.org']);
@@ -20,99 +21,105 @@
           ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
           var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
         })();
-      </script><link href="../../../style/coderay.css" rel="stylesheet" type="text/css">
-<link href="../../../style/pygmentize.css" rel="stylesheet" type="text/css">
-</head>
-<body>
+      </script>
+    <link href="../../../style/coderay.css" rel="stylesheet" type="text/css">
+    <link href="../../../style/pygmentize.css" rel="stylesheet" type="text/css">
+  </head>
+  <body>
     <div id="header">
       <div class="container">
         <div id="logo">
           <a href="../../../">
-            <img src="../../../images/s4_test.png"></a>
+            <img src="../../../images/s4_test.png">
+          </a>
+        </div>
+        <div id="navbar">
+          <ul id="nav">
+            <li>
+              <a href="../../../">home</a>
+            </li>
+            <li>
+              <a href="..">doc</a>
+            </li>
+            <li>
+              <a href="http://git-wip-us.apache.org/repos/asf?p=incubator-s4.git" onclick="_gaq.push(['_trackEvent', 'External', 'Apache Git', 'http://github.com/s4']);">code</a>
+            </li>
+            <li>
+              <a href="http://people.apache.org/~mmorel/apache-s4-0.6.0-incubating-doc/javadoc/">API</a>
+            </li>
+            <li>
+              <a href="../../../contrib">get involved</a>
+            </li>
+            <li>
+              <a href="../../../team">team</a>
+            </li>
+            <li>
+              <a href="../../../download">download</a>
+            </li>
+          </ul>
         </div>
-        <div id="navbar"><ul id="nav">
-<li>
-    <a href="../../../">home</a>
-  </li>
-  <li>
-    <a href="https://cwiki.apache.org/confluence/display/S4/S4+Wiki">doc [0.5]</a>
-  </li>
-  <li>
-    <a href="http://git-wip-us.apache.org/repos/asf?p=incubator-s4.git" onclick="_gaq.push(['_trackEvent', 'External', 'Apache Git', 'http://github.com/s4']);">code</a>
-  </li>
-  <li>
-    <a href="http://people.apache.org/~mmorel/apache-s4-0.5.0-incubating-doc/javadoc/">API</a>
-  </li>
-  <li>
-    <a href="../../../contrib">get involved</a>
-  </li>
-  <li>
-    <a href="../../../team">team</a>
-  </li>
-  <li>
-    <a href="../../../download">download</a>
-  </li>
-</ul></div>
       </div>
     </div>
     <div id="wrapper">
       <div class="container" id="container">
-<blockquote>
-  <p>Make sure you have already read the <a href="../walkthrough">walkthrough</a></p>
-</blockquote>
-
-<h1 id="how-to-add-dependencies-to-my-s4-application">How to add dependencies to my S4 application?</h1>
-
-<p>Your application typically depends on various external libraries. Here is how to configure those dependencies in an S4 project. We assume here that you are working with a sample project automatically generated through the <code>s4 newApp</code> script.</p>
-
-<h2 id="dependencies-on-public-artifacts">Dependencies on public artifacts</h2>
-
-<ul>
-<li>
-    <p>Add maven artifacts definitions to the gradle build file. For instance, add twitter4j_core and twitter4j_stream:</p>
-
-    <pre><code>  project.ext["libraries"] = [
-             twitter4j_core:     'org.twitter4j:twitter4j-core:2.2.5',
-             twitter4j_stream:   'org.twitter4j:twitter4j-stream:2.2.5',
-             s4_base:            'org.apache.s4:s4-base:0.6.0',
-             s4_comm:            'org.apache.s4:s4-comm:0.6.0',
-             s4_core:            'org.apache.s4:s4-core:0.6.0'
-         ]
-</code></pre>
-  </li>
-  <li>
-    <p>Add these dependencies as compile-time dependencies. For instance:</p>
-
-    <pre><code>  dependencies {
-     compile (libraries.s4_base)
-     compile (libraries.s4_comm)
-     compile (libraries.s4_core)
-     compile (libraries.twitter4j_core)
-     compile (libraries.twitter4j_stream)
-  }
-</code></pre>
-  </li>
-  <li>
-    <p>If you use an IDE such as eclipse, you may update your project’s classpath with: <code>./gradlew eclipse</code></p>
-  </li>
-</ul>
-<p>A good source for finding dependencies is for instance <a href="http://search.maven.org/">http://search.maven.org/</a> where you also get the syntax for gradle scripts (see grails syntax).</p>
-
-<blockquote>
-  <p>The application dependencies will be automatically included in the s4r archive that you create and publish.</p>
-</blockquote>
-
-<h2 id="dependencies-on-non-public-artifacts">Dependencies on non-public artifacts</h2>
-
-<p>You may have dependencies that are not published to maven repositories. In that case you should either:</p>
-
-<ul>
-<li>publish them to your local maven repository, see <a href="%5Bhttp://maven.apache.org/guides/mini/guide-3rd-party-jars-local.html">http://maven.apache.org/guides/mini/guide-3rd-party-jars-local.html</a>
-</li>
-  <li>add them to the <em>lib</em> directory</li>
-</ul>
-<p>In both cases you still have to declare them as compile-time dependencies.</p>
-</div>
+        <blockquote>
+          <p>Make sure you have already read the <a href="../walkthrough">walkthrough</a></p>
+        </blockquote>
+        
+        <h1 id="how-to-add-dependencies-to-my-s4-application">How to add dependencies to my S4 application?</h1>
+        
+        <p>Your application typically depends on various external libraries. Here is how to configure those dependencies in an S4 project. We assume here that you are working with a sample project automatically generated through the <code>s4 newApp</code> script.</p>
+        
+        <h2 id="dependencies-on-public-artifacts">Dependencies on public artifacts</h2>
+        
+        <ul>
+          <li>
+            <p>Add maven artifacts definitions to the gradle build file. For instance, add twitter4j_core and twitter4j_stream:</p>
+        
+            <pre><code>  project.ext["libraries"] = [
+                     twitter4j_core:     'org.twitter4j:twitter4j-core:2.2.5',
+                     twitter4j_stream:   'org.twitter4j:twitter4j-stream:2.2.5',
+                     s4_base:            'org.apache.s4:s4-base:0.6.0',
+                     s4_comm:            'org.apache.s4:s4-comm:0.6.0',
+                     s4_core:            'org.apache.s4:s4-core:0.6.0'
+                 ]
+        </code></pre>
+          </li>
+          <li>
+            <p>Add these dependencies as compile-time dependencies. For instance:</p>
+        
+            <pre><code>  dependencies {
+             compile (libraries.s4_base)
+             compile (libraries.s4_comm)
+             compile (libraries.s4_core)
+             compile (libraries.twitter4j_core)
+             compile (libraries.twitter4j_stream)
+          }
+        </code></pre>
+          </li>
+          <li>
+            <p>If you use an IDE such as eclipse, you may update your project’s classpath with: <code>./gradlew eclipse</code></p>
+          </li>
+        </ul>
+        
+        <p>A good source for finding dependencies is for instance <a href="http://search.maven.org/">http://search.maven.org/</a> where you also get the syntax for gradle scripts (see grails syntax).</p>
+        
+        <blockquote>
+          <p>The application dependencies will be automatically included in the s4r archive that you create and publish.</p>
+        </blockquote>
+        
+        <h2 id="dependencies-on-non-public-artifacts">Dependencies on non-public artifacts</h2>
+        
+        <p>You may have dependencies that are not published to maven repositories. In that case you should either:</p>
+        
+        <ul>
+          <li>publish them to your local maven repository, see <a href="%5Bhttp://maven.apache.org/guides/mini/guide-3rd-party-jars-local.html">http://maven.apache.org/guides/mini/guide-3rd-party-jars-local.html</a>
+        </li>
+          <li>add them to the <em>lib</em> directory</li>
+        </ul>
+        
+        <p>In both cases you still have to declare them as compile-time dependencies.</p>
+      </div>
     </div>
     <div id="footer">
       <div class="container">

Modified: incubator/s4/site/doc/0.6.0/configuration/index.html
URL: http://svn.apache.org/viewvc/incubator/s4/site/doc/0.6.0/configuration/index.html?rev=1488543&r1=1488542&r2=1488543&view=diff
==============================================================================
--- incubator/s4/site/doc/0.6.0/configuration/index.html (original)
+++ incubator/s4/site/doc/0.6.0/configuration/index.html Sat Jun  1 16:37:44 2013
@@ -1,16 +1,17 @@
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<!DOCTYPE html>
 <html>
-<head>
-<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
-<title>S4: Configuration</title>
-<meta content="A general-purpose distributed stream computing platform" name="description">
-<link href="../../../style/screen.css" media="screen" rel="stylesheet" type="text/css">
-<link href="../../../style/print.css" media="print" rel="stylesheet" type="text/css">
-<!--[if lt IE 9]>
+  <head>
+    <title>S4: Configuration</title>
+    <meta content="text/html; charset=utf-8" http-equiv="Content-Type">
+    <meta content="A general-purpose distributed stream computing platform" name="description">
+    <link href="../../../style/screen.css" media="screen" rel="stylesheet" type="text/css">
+    <link href="../../../style/print.css" media="print" rel="stylesheet" type="text/css">
+    <!--[if lt IE 9]>
       <link href="../../../style/ie.css" media="screen" rel="stylesheet" type="text/css">
-    <![endif]--><link href="../../../style/style.css" rel="stylesheet" type="text/css">
-<link href="../../../style/nav.css" rel="stylesheet" type="text/css">
-<script type="text/javascript">
+    <![endif]-->
+    <link href="../../../style/style.css" rel="stylesheet" type="text/css">
+    <link href="../../../style/nav.css" rel="stylesheet" type="text/css">
+    <script type="text/javascript">
         var _gaq = _gaq || [];
         _gaq.push(['_setAccount', 'UA-19490961-1']);
         _gaq.push(['_setDomainName', 'incubator.apache.org']);
@@ -20,222 +21,233 @@
           ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
           var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
         })();
-      </script><link href="../../../style/coderay.css" rel="stylesheet" type="text/css">
-<link href="../../../style/pygmentize.css" rel="stylesheet" type="text/css">
-</head>
-<body>
+      </script>
+    <link href="../../../style/coderay.css" rel="stylesheet" type="text/css">
+    <link href="../../../style/pygmentize.css" rel="stylesheet" type="text/css">
+  </head>
+  <body>
     <div id="header">
       <div class="container">
         <div id="logo">
           <a href="../../../">
-            <img src="../../../images/s4_test.png"></a>
+            <img src="../../../images/s4_test.png">
+          </a>
+        </div>
+        <div id="navbar">
+          <ul id="nav">
+            <li>
+              <a href="../../../">home</a>
+            </li>
+            <li>
+              <a href="..">doc</a>
+            </li>
+            <li>
+              <a href="http://git-wip-us.apache.org/repos/asf?p=incubator-s4.git" onclick="_gaq.push(['_trackEvent', 'External', 'Apache Git', 'http://github.com/s4']);">code</a>
+            </li>
+            <li>
+              <a href="http://people.apache.org/~mmorel/apache-s4-0.6.0-incubating-doc/javadoc/">API</a>
+            </li>
+            <li>
+              <a href="../../../contrib">get involved</a>
+            </li>
+            <li>
+              <a href="../../../team">team</a>
+            </li>
+            <li>
+              <a href="../../../download">download</a>
+            </li>
+          </ul>
         </div>
-        <div id="navbar"><ul id="nav">
-<li>
-    <a href="../../../">home</a>
-  </li>
-  <li>
-    <a href="https://cwiki.apache.org/confluence/display/S4/S4+Wiki">doc [0.5]</a>
-  </li>
-  <li>
-    <a href="http://git-wip-us.apache.org/repos/asf?p=incubator-s4.git" onclick="_gaq.push(['_trackEvent', 'External', 'Apache Git', 'http://github.com/s4']);">code</a>
-  </li>
-  <li>
-    <a href="http://people.apache.org/~mmorel/apache-s4-0.5.0-incubating-doc/javadoc/">API</a>
-  </li>
-  <li>
-    <a href="../../../contrib">get involved</a>
-  </li>
-  <li>
-    <a href="../../../team">team</a>
-  </li>
-  <li>
-    <a href="../../../download">download</a>
-  </li>
-</ul></div>
       </div>
     </div>
     <div id="wrapper">
       <div class="container" id="container">
-<blockquote>
-  <p>How to configure S4 clusters and applications</p>
-</blockquote>
-
-<h1 id="toolset">Toolset</h1>
-
-<p>S4 provides a set of tools to:</p>
-
-<ul>
-<li>define S4 clusters: <code>s4 newCluster</code>
-</li>
-  <li>start S4 nodes: <code>s4 node</code>
-</li>
-  <li>package applications: <code>s4 s4r</code>
-</li>
-  <li>deploy applications: <code>s4 deploy</code>
-</li>
-  <li>start a Zookeeper server for easy testing: <code>s4 zkServer</code>
-    <ul>
-<li>
-<code>s4 zkServer -t</code> will start a Zookeeper server and automatically configure 2 clusters</li>
-    </ul>
-</li>
-  <li>view the status of S4 clusters coordinated by a given Zookeeper ensemble: <code>s4 status</code>
-</li>
-  <li>
-<code>s4</code> will  give you a list of available commands.</li>
-  <li>
-<code>./s4 &lt;command&gt; -help</code> will provide detailed documentation for each of these commands.</li>
-</ul>
-<h1 id="cluster-configuration">Cluster configuration</h1>
-
-<p>Before starting S4 nodes, you must define a logical cluster by specifying:</p>
-
-<ul>
-<li>a name for the cluster</li>
-  <li>a number of partitions (~ tasks)</li>
-  <li>an initial port number for listener sockets
-    <ul>
-<li>you must specify a free port, considering that each of the nodes of the cluster will open a different port, with a number monotonically increasing from the initial number. For instance, for a cluster of 10 nodes and an initial port 12000, ports 12000 to 12009 will be used among the nodes.</li>
-      <li>those ports are used for inter node communication.</li>
-    </ul>
-</li>
-</ul>
-<p>The cluster configuration is maintained in Zookeeper, and can be set using S4 tools:</p>
-
-<pre><code>./s4 newCluster -c=cluster1 -nbTasks=2 -flp=12000
-</code></pre>
-
-<p>See tool documentation by typing:</p>
-
-<pre><code>./s4 newCluster -help
-</code></pre>
-
-<h1 id="node-configuration">Node configuration</h1>
-
-<p><strong>Platform code and application code are fully configurable, at deployment time.</strong></p>
-
-<p>S4 nodes start as simple <strong>bootstrap</strong> processes whose initial role is merely to connect the cluster manager:</p>
-
-<ol>
-<li>the bootstrap code connects to the cluster manager</li>
-  <li>when an application is available on the cluster, the node gets notified</li>
-  <li>it downloads the platform configuration and code, as specified in the configuration of the deployed application.</li>
-  <li>the communication and core components are loaded, bound and initialized</li>
-  <li>the application configuration and code, as specified in the configuration of the deployed applciation, is downloaded</li>
-  <li>the application is initialized and started</li>
-</ol>
-<p>This figure illustrates the separation between the bootstrap code, the S4 platform code, and application code in an S4 node:</p>
-
-<p><img src="../../../images/doc/0.6.0/s4_node_layers.png" alt="image"></p>
-
-<p>Therefore, for starting an S4 node on a given host, you only need to specify:</p>
-
-<ul>
-<li>the connection string to the cluster management system (Zookeeper) <code>localhost:2181</code> by default</li>
-  <li>the name of the logical cluster to which this node will belong</li>
-</ul>
-<p>Example:
-<code>./s4 node -c=cluster1 -zk=host.domain.com</code></p>
-
-<h2 id="overriding-node-configuration">Overriding node configuration</h2>
-<p>By default, nodes read configuration from configuration files in the classpath <code>default.s4.base.properties</code>, <code>default.s4.comm.properties</code> and <code>default.s4.core.properties</code>.</p>
-
-<p>It is possible to override these parameters by using the <code>-p</code> option when starting a node.</p>
-
-<h1 id="application-configuration">Application configuration</h1>
-
-<p>Deploying applications is easier when we can define both the parameters of the application <strong>and</strong> the target environment.</p>
-
-<p>In S4, we achieve this by specifying <strong>both</strong> application parameters and S4 platform parameters in the deployment phase :</p>
-
-<ul>
-<li>which application class to use</li>
-  <li>where to fetch application code</li>
-  <li>which specific modules to use</li>
-  <li>where to fetch these modules</li>
-  <li>string configuration parameters - that can be used by the application and the modules</li>
-</ul>
-<h2 id="modules-configuration">Modules configuration</h2>
-
-<p>S4 follows a modular design and uses <a href="http://code.google.com/p/google-guice/">Guice</a> for defining modules and injecting dependencies.</p>
-
-<p>As illustrated above, an S4 node is composed of:
-* a base module that specifies how to connect to the cluster manager and how to download code
-* a communication module that specifies communication protocols, event listeners and senders
-* a core module that specifies the deployment mechanism, serialization mechanism
-* an application</p>
-
-<h3 id="default-parameters">default parameters</h3>
-
-<p>For the <a href="https://github.com/apache/incubator-s4/blob/dev/subprojects/s4-comm/src/main/resources/default.s4.comm.properties">comm module</a>: communication protocols, tuning parameters for sending events</p>
-
-<p>For the core module, there is no default parameters.</p>
-
-<h3 id="overriding-modules">overriding modules</h3>
-
-<p>We provide default modules, but you may directly specify others through the command line, and it is also possible to override them with new modules and even specify new ones (custom modules classes must provide an empty no-args constructor).</p>
-
-<p>Custom overriding modules can be specified when deploying the application, through the<code>deploy</code> command, through the <code>emc</code> or <code>modulesClasses</code> option.</p>
-
-<p>For instance, in order to enable file system based checkpointing, pass the corresponding checkpointing module class :</p>
-
-<pre><code>./s4 deploy -s4r=uri/to/app.s4r -c=cluster1 -appName=myApp \
--emc=org.apache.s4.core.ft.FileSystemBackendCheckpointingModule 
-</code></pre>
-
-<p>You can also write your own custom modules. In that case, just package them into a jar file, and specify how to fetch that file when deploying the application, with the <code>mu</code> or <code>modulesURIs</code>  option.</p>
-
-<p>For instance, if you checkpoint through a specific key value store, you can write you own checkpointing implementation and module, package that into fancyKeyValueStoreCheckpointingModule.jar , and then:</p>
-
-<pre><code>./s4 deploy -c=cluster1 -emc=my.project.FancyKeyValueStoreBackendCheckpointingModule \
--mu=uri/to/fancyKeyValueStoreCheckpointingModule.jar
-</code></pre>
-
-<h3 id="overriding-parameters">overriding parameters</h3>
-
-<p>A simple way to pass parameters to your application code is by:</p>
-
-<ul>
-<li>injecting them in the application class (primitive types, enums and class literals are automatically converted), for instance:</li>
-</ul>
-<pre><code class="language-java"><span class="nd">@Inject</span>
-<span class="nd">@Named</span><span class="o">(</span><span class="s">"thePortNumber"</span><span class="o">)</span>
-<span class="kt">int</span> <span class="n">port</span></code></pre>
-
-<ul>
-<li>specifying the parameter value when deploying the application (using <code>-p</code> inline with the node command, or with the ‘<code>@</code>’ syntax)</li>
-</ul>
-<p>S4 uses an internal Guice module that automatically injects configuration parameters passed through the deploy command to matching <code>@Named</code> parameters.</p>
-
-<p>Both application and platform parameters can be overriden. For instance, specifying a custom storage path for the file system based checkpointing mechanism would be passing the <code>s4.checkpointing.filesystem.storageRootPath</code> parameter:</p>
-
-<pre><code>./s4 deploy -s4r=uri/to/app.s4r -c=cluster1 -appName=myApp \
--emc=org.apache.s4.core.ft.FileSystemBackendCheckpointingModule \ 
--p=s4.checkpointing.filesystem.storageRootPath=/custom/path 
-</code></pre>
-
-<blockquote>
-  <p>Make sure you pass overriding <strong>platform</strong> parameters through the <code>-p</code> option in the <code>node</code> command, and pass <strong>application</strong> parameters through the <code>-p</code> option in the <code>deploy</code> command.</p>
-</blockquote>
-
-<h2 id="file-based-configuration">File-based configuration</h2>
-
-<p>Instead of specifying node parameters inline, you may refer to a file with the ‘@’ notation:
-<code>./s4 deploy @/path/to/config/file</code>
-With contents of the referenced file like:</p>
-
-<pre><code>-s4r=uri/to/app.s4r
--c=cluster1
--appName=myApp
--emc=org.apache.s4.core.ft.FileSystemBackendCheckpointingModule
--p=param1=value1,param2=value2
-</code></pre>
-
-<h2 id="logging">Logging</h2>
-
-<p>S4 uses <a href="http://logback.qos.ch/">logback</a>, and <a href="https://git-wip-us.apache.org/repos/asf?p=incubator-s4.git;a=blob_plain;f=subprojects/s4-core/src/main/resources/logback.xml;h=ea8c85a104b475f1b9dea641656e76eb3b6a9d4c;hb=piper">here</a> is the default configuration file. You may tweak this configuration by adding your own logback.xml file in the <code>lib/</code> directory (for a binary release) or in the <code>subprojects/s4-tools/build/install/s4-tools/lib/</code> directory (for a source release or checkout from git).</p>
-</div>
+        <blockquote>
+          <p>How to configure S4 clusters and applications</p>
+        </blockquote>
+        
+        <h1 id="toolset">Toolset</h1>
+        
+        <p>S4 provides a set of tools to:</p>
+        
+        <ul>
+          <li>define S4 clusters: <code>s4 newCluster</code>
+        </li>
+          <li>start S4 nodes: <code>s4 node</code>
+        </li>
+          <li>package applications: <code>s4 s4r</code>
+        </li>
+          <li>deploy applications: <code>s4 deploy</code>
+        </li>
+          <li>start a Zookeeper server for easy testing: <code>s4 zkServer</code>
+            <ul>
+              <li>
+        <code>s4 zkServer -t</code> will start a Zookeeper server and automatically configure 2 clusters</li>
+            </ul>
+          </li>
+          <li>view the status of S4 clusters coordinated by a given Zookeeper ensemble: <code>s4 status</code>
+        </li>
+          <li>
+        <code>s4</code> will  give you a list of available commands.</li>
+          <li>
+        <code>./s4 &lt;command&gt; -help</code> will provide detailed documentation for each of these commands.</li>
+        </ul>
+        
+        <h1 id="cluster-configuration">Cluster configuration</h1>
+        
+        <p>Before starting S4 nodes, you must define a logical cluster by specifying:</p>
+        
+        <ul>
+          <li>a name for the cluster</li>
+          <li>a number of partitions (~ tasks)</li>
+          <li>an initial port number for listener sockets
+            <ul>
+              <li>you must specify a free port, considering that each of the nodes of the cluster will open a different port, with a number monotonically increasing from the initial number. For instance, for a cluster of 10 nodes and an initial port 12000, ports 12000 to 12009 will be used among the nodes.</li>
+              <li>those ports are used for inter node communication.</li>
+            </ul>
+          </li>
+        </ul>
+        
+        <p>The cluster configuration is maintained in Zookeeper, and can be set using S4 tools:</p>
+        
+        <pre><code>./s4 newCluster -c=cluster1 -nbTasks=2 -flp=12000
+        </code></pre>
+        
+        <p>See tool documentation by typing:</p>
+        
+        <pre><code>./s4 newCluster -help
+        </code></pre>
+        
+        <h1 id="node-configuration">Node configuration</h1>
+        
+        <p><strong>Platform code and application code are fully configurable, at deployment time.</strong></p>
+        
+        <p>S4 nodes start as simple <strong>bootstrap</strong> processes whose initial role is merely to connect the cluster manager:</p>
+        
+        <ol>
+          <li>the bootstrap code connects to the cluster manager</li>
+          <li>when an application is available on the cluster, the node gets notified</li>
+          <li>it downloads the platform configuration and code, as specified in the configuration of the deployed application.</li>
+          <li>the communication and core components are loaded, bound and initialized</li>
+          <li>the application configuration and code, as specified in the configuration of the deployed applciation, is downloaded</li>
+          <li>the application is initialized and started</li>
+        </ol>
+        
+        <p>This figure illustrates the separation between the bootstrap code, the S4 platform code, and application code in an S4 node:</p>
+        
+        <p><img src="../../../images/doc/0.6.0/s4_node_layers.png" alt="image"></p>
+        
+        <p>Therefore, for starting an S4 node on a given host, you only need to specify:</p>
+        
+        <ul>
+          <li>the connection string to the cluster management system (Zookeeper) <code>localhost:2181</code> by default</li>
+          <li>the name of the logical cluster to which this node will belong</li>
+        </ul>
+        
+        <p>Example:
+        <code>./s4 node -c=cluster1 -zk=host.domain.com</code></p>
+        
+        <h2 id="overriding-node-configuration">Overriding node configuration</h2>
+        <p>By default, nodes read configuration from configuration files in the classpath <code>default.s4.base.properties</code>, <code>default.s4.comm.properties</code> and <code>default.s4.core.properties</code>.</p>
+        
+        <p>It is possible to override these parameters by using the <code>-p</code> option when starting a node.</p>
+        
+        <h1 id="application-configuration">Application configuration</h1>
+        
+        <p>Deploying applications is easier when we can define both the parameters of the application <strong>and</strong> the target environment.</p>
+        
+        <p>In S4, we achieve this by specifying <strong>both</strong> application parameters and S4 platform parameters in the deployment phase :</p>
+        
+        <ul>
+          <li>which application class to use</li>
+          <li>where to fetch application code</li>
+          <li>which specific modules to use</li>
+          <li>where to fetch these modules</li>
+          <li>string configuration parameters - that can be used by the application and the modules</li>
+        </ul>
+        
+        <h2 id="modules-configuration">Modules configuration</h2>
+        
+        <p>S4 follows a modular design and uses <a href="http://code.google.com/p/google-guice/">Guice</a> for defining modules and injecting dependencies.</p>
+        
+        <p>As illustrated above, an S4 node is composed of:
+        * a base module that specifies how to connect to the cluster manager and how to download code
+        * a communication module that specifies communication protocols, event listeners and senders
+        * a core module that specifies the deployment mechanism, serialization mechanism
+        * an application</p>
+        
+        <h3 id="default-parameters">default parameters</h3>
+        
+        <p>For the <a href="https://github.com/apache/incubator-s4/blob/dev/subprojects/s4-comm/src/main/resources/default.s4.comm.properties">comm module</a>: communication protocols, tuning parameters for sending events</p>
+        
+        <p>For the core module, there is no default parameters.</p>
+        
+        <h3 id="overriding-modules">overriding modules</h3>
+        
+        <p>We provide default modules, but you may directly specify others through the command line, and it is also possible to override them with new modules and even specify new ones (custom modules classes must provide an empty no-args constructor).</p>
+        
+        <p>Custom overriding modules can be specified when deploying the application, through the<code>deploy</code> command, through the <code>emc</code> or <code>modulesClasses</code> option.</p>
+        
+        <p>For instance, in order to enable file system based checkpointing, pass the corresponding checkpointing module class :</p>
+        
+        <pre><code>./s4 deploy -s4r=uri/to/app.s4r -c=cluster1 -appName=myApp \
+        -emc=org.apache.s4.core.ft.FileSystemBackendCheckpointingModule 
+        </code></pre>
+        
+        <p>You can also write your own custom modules. In that case, just package them into a jar file, and specify how to fetch that file when deploying the application, with the <code>mu</code> or <code>modulesURIs</code>  option.</p>
+        
+        <p>For instance, if you checkpoint through a specific key value store, you can write you own checkpointing implementation and module, package that into fancyKeyValueStoreCheckpointingModule.jar , and then:</p>
+        
+        <pre><code>./s4 deploy -c=cluster1 -emc=my.project.FancyKeyValueStoreBackendCheckpointingModule \
+        -mu=uri/to/fancyKeyValueStoreCheckpointingModule.jar
+        </code></pre>
+        
+        <h3 id="overriding-parameters">overriding parameters</h3>
+        
+        <p>A simple way to pass parameters to your application code is by:</p>
+        
+        <ul>
+          <li>injecting them in the application class (primitive types, enums and class literals are automatically converted), for instance:</li>
+        </ul>
+        
+        <pre><code class="language-java"><span class="nd">@Inject</span>
+        <span class="nd">@Named</span><span class="o">(</span><span class="s">"thePortNumber"</span><span class="o">)</span>
+        <span class="kt">int</span> <span class="n">port</span></code></pre>
+        
+        <ul>
+          <li>specifying the parameter value when deploying the application (using <code>-p</code> inline with the node command, or with the ‘<code>@</code>’ syntax)</li>
+        </ul>
+        
+        <p>S4 uses an internal Guice module that automatically injects configuration parameters passed through the deploy command to matching <code>@Named</code> parameters.</p>
+        
+        <p>Both application and platform parameters can be overriden. For instance, specifying a custom storage path for the file system based checkpointing mechanism would be passing the <code>s4.checkpointing.filesystem.storageRootPath</code> parameter:</p>
+        
+        <pre><code>./s4 deploy -s4r=uri/to/app.s4r -c=cluster1 -appName=myApp \
+        -emc=org.apache.s4.core.ft.FileSystemBackendCheckpointingModule \ 
+        -p=s4.checkpointing.filesystem.storageRootPath=/custom/path 
+        </code></pre>
+        
+        <blockquote>
+          <p>Make sure you pass overriding <strong>platform</strong> parameters through the <code>-p</code> option in the <code>node</code> command, and pass <strong>application</strong> parameters through the <code>-p</code> option in the <code>deploy</code> command.</p>
+        </blockquote>
+        
+        <h2 id="file-based-configuration">File-based configuration</h2>
+        
+        <p>Instead of specifying node parameters inline, you may refer to a file with the ‘@’ notation:
+        <code>./s4 deploy @/path/to/config/file</code>
+        With contents of the referenced file like:</p>
+        
+        <pre><code>-s4r=uri/to/app.s4r
+        -c=cluster1
+        -appName=myApp
+        -emc=org.apache.s4.core.ft.FileSystemBackendCheckpointingModule
+        -p=param1=value1,param2=value2
+        </code></pre>
+        
+        <h2 id="logging">Logging</h2>
+        
+        <p>S4 uses <a href="http://logback.qos.ch/">logback</a>, and <a href="https://git-wip-us.apache.org/repos/asf?p=incubator-s4.git;a=blob_plain;f=subprojects/s4-core/src/main/resources/logback.xml;h=ea8c85a104b475f1b9dea641656e76eb3b6a9d4c;hb=piper">here</a> is the default configuration file. You may tweak this configuration by adding your own logback.xml file in the <code>lib/</code> directory (for a binary release) or in the <code>subprojects/s4-tools/build/install/s4-tools/lib/</code> directory (for a source release or checkout from git).</p>
+      </div>
     </div>
     <div id="footer">
       <div class="container">

Modified: incubator/s4/site/doc/0.6.0/dev_tips/index.html
URL: http://svn.apache.org/viewvc/incubator/s4/site/doc/0.6.0/dev_tips/index.html?rev=1488543&r1=1488542&r2=1488543&view=diff
==============================================================================
--- incubator/s4/site/doc/0.6.0/dev_tips/index.html (original)
+++ incubator/s4/site/doc/0.6.0/dev_tips/index.html Sat Jun  1 16:37:44 2013
@@ -1,16 +1,17 @@
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<!DOCTYPE html>
 <html>
-<head>
-<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
-<title>S4: Development tips</title>
-<meta content="A general-purpose distributed stream computing platform" name="description">
-<link href="../../../style/screen.css" media="screen" rel="stylesheet" type="text/css">
-<link href="../../../style/print.css" media="print" rel="stylesheet" type="text/css">
-<!--[if lt IE 9]>
+  <head>
+    <title>S4: Development tips</title>
+    <meta content="text/html; charset=utf-8" http-equiv="Content-Type">
+    <meta content="A general-purpose distributed stream computing platform" name="description">
+    <link href="../../../style/screen.css" media="screen" rel="stylesheet" type="text/css">
+    <link href="../../../style/print.css" media="print" rel="stylesheet" type="text/css">
+    <!--[if lt IE 9]>
       <link href="../../../style/ie.css" media="screen" rel="stylesheet" type="text/css">
-    <![endif]--><link href="../../../style/style.css" rel="stylesheet" type="text/css">
-<link href="../../../style/nav.css" rel="stylesheet" type="text/css">
-<script type="text/javascript">
+    <![endif]-->
+    <link href="../../../style/style.css" rel="stylesheet" type="text/css">
+    <link href="../../../style/nav.css" rel="stylesheet" type="text/css">
+    <script type="text/javascript">
         var _gaq = _gaq || [];
         _gaq.push(['_setAccount', 'UA-19490961-1']);
         _gaq.push(['_setDomainName', 'incubator.apache.org']);
@@ -20,101 +21,111 @@
           ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
           var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
         })();
-      </script><link href="../../../style/coderay.css" rel="stylesheet" type="text/css">
-<link href="../../../style/pygmentize.css" rel="stylesheet" type="text/css">
-</head>
-<body>
+      </script>
+    <link href="../../../style/coderay.css" rel="stylesheet" type="text/css">
+    <link href="../../../style/pygmentize.css" rel="stylesheet" type="text/css">
+  </head>
+  <body>
     <div id="header">
       <div class="container">
         <div id="logo">
           <a href="../../../">
-            <img src="../../../images/s4_test.png"></a>
+            <img src="../../../images/s4_test.png">
+          </a>
+        </div>
+        <div id="navbar">
+          <ul id="nav">
+            <li>
+              <a href="../../../">home</a>
+            </li>
+            <li>
+              <a href="..">doc</a>
+            </li>
+            <li>
+              <a href="http://git-wip-us.apache.org/repos/asf?p=incubator-s4.git" onclick="_gaq.push(['_trackEvent', 'External', 'Apache Git', 'http://github.com/s4']);">code</a>
+            </li>
+            <li>
+              <a href="http://people.apache.org/~mmorel/apache-s4-0.6.0-incubating-doc/javadoc/">API</a>
+            </li>
+            <li>
+              <a href="../../../contrib">get involved</a>
+            </li>
+            <li>
+              <a href="../../../team">team</a>
+            </li>
+            <li>
+              <a href="../../../download">download</a>
+            </li>
+          </ul>
         </div>
-        <div id="navbar"><ul id="nav">
-<li>
-    <a href="../../../">home</a>
-  </li>
-  <li>
-    <a href="https://cwiki.apache.org/confluence/display/S4/S4+Wiki">doc [0.5]</a>
-  </li>
-  <li>
-    <a href="http://git-wip-us.apache.org/repos/asf?p=incubator-s4.git" onclick="_gaq.push(['_trackEvent', 'External', 'Apache Git', 'http://github.com/s4']);">code</a>
-  </li>
-  <li>
-    <a href="http://people.apache.org/~mmorel/apache-s4-0.5.0-incubating-doc/javadoc/">API</a>
-  </li>
-  <li>
-    <a href="../../../contrib">get involved</a>
-  </li>
-  <li>
-    <a href="../../../team">team</a>
-  </li>
-  <li>
-    <a href="../../../download">download</a>
-  </li>
-</ul></div>
       </div>
     </div>
     <div id="wrapper">
       <div class="container" id="container">
-<blockquote>
-  <p>Here are a few tips to ease the development of S4 applications.</p>
-</blockquote>
-
-<h1 id="import-an-s4-project-into-your-ide">Import an S4 project into your IDE</h1>
-
-<p>You can run <code>gradlew eclipse</code> or <code>gradlew idea</code> at the root of your S4 application directory. Then simply import the project into eclipse or intellij. You’ll have both your application classes <em>and</em> S4 libraries imported to the classpath of the project.</p>
-
-<p>In order to get the transitive dependencies of the platform included as well, you should:</p>
-
-<ul>
-<li>Download a source distribution</li>
-  <li>
-    <p>Install S4 and its dependencies in your local maven repository</p>
-
-    <pre><code>  // from s4 source distribution root directory
-  ./gradlew install -DskipTests
-</code></pre>
-  </li>
-  <li>Then run <code>gradlew eclipse</code> or <code>gradlew idea</code>
-</li>
-</ul>
-<hr>
-<h1 id="start-a-local-zookeeper-instance">Start a local Zookeeper instance</h1>
-
-<ul>
-<li>
-    <p>Use the default test configuration (2 clusters with following configs: <code>-c=testCluster1:flp=12000:nbTasks=1</code> and <code>-c=testCluster2:flp=13000:nbTasks=1</code>)</p>
-
-    <pre><code>  s4 zkServer -t
-</code></pre>
-  </li>
-  <li>
-    <p>Start a Zookeeper instance with your custom configuration, e.g. with 1 partition:</p>
-
-    <pre><code>  s4 zkServer -clusters=c=testCluster1:flp=12000:nbTasks=1
-</code></pre>
-  </li>
-</ul>
-<hr>
-<h1 id="load-an-application-in-a-new-node-directly-from-an-ide">Load an application in a new node directly from an IDE</h1>
-
-<p>This allows to <em>skip the packaging phase!</em></p>
-
-<p>Requirements:</p>
-
-<ul>
-<li>application classes <strong>and</strong> S4 classes are in your classpath. See above.</li>
-  <li>application already configured in cluster (with the <code>-appClass</code> option, no need to package the app)</li>
-</ul>
-<p>Then just run the <code>org.apache.s4.core.S4Node</code> class and pass:</p>
-
-<ul>
-<li>the cluster name: <code>-c=testCluster1</code>
-</li>
-</ul>
-<p>If you use a local Zookeeper instance on the default port (2181), there is no need to specify the <code>-zk</code> option.</p>
-</div>
+        <blockquote>
+          <p>Here are a few tips to ease the development of S4 applications.</p>
+        </blockquote>
+        
+        <h1 id="import-an-s4-project-into-your-ide">Import an S4 project into your IDE</h1>
+        
+        <p>You can run <code>gradlew eclipse</code> or <code>gradlew idea</code> at the root of your S4 application directory. Then simply import the project into eclipse or intellij. You’ll have both your application classes <em>and</em> S4 libraries imported to the classpath of the project.</p>
+        
+        <p>In order to get the transitive dependencies of the platform included as well, you should:</p>
+        
+        <ul>
+          <li>Download a source distribution</li>
+          <li>
+            <p>Install S4 and its dependencies in your local maven repository</p>
+        
+            <pre><code>  // from s4 source distribution root directory
+          ./gradlew install -DskipTests
+        </code></pre>
+          </li>
+          <li>Then run <code>gradlew eclipse</code> or <code>gradlew idea</code>
+        </li>
+        </ul>
+        
+        <hr>
+        
+        <h1 id="start-a-local-zookeeper-instance">Start a local Zookeeper instance</h1>
+        
+        <ul>
+          <li>
+            <p>Use the default test configuration (2 clusters with following configs: <code>-c=testCluster1:flp=12000:nbTasks=1</code> and <code>-c=testCluster2:flp=13000:nbTasks=1</code>)</p>
+        
+            <pre><code>  s4 zkServer -t
+        </code></pre>
+          </li>
+          <li>
+            <p>Start a Zookeeper instance with your custom configuration, e.g. with 1 partition:</p>
+        
+            <pre><code>  s4 zkServer -clusters=c=testCluster1:flp=12000:nbTasks=1
+        </code></pre>
+          </li>
+        </ul>
+        
+        <hr>
+        
+        <h1 id="load-an-application-in-a-new-node-directly-from-an-ide">Load an application in a new node directly from an IDE</h1>
+        
+        <p>This allows to <em>skip the packaging phase!</em></p>
+        
+        <p>Requirements:</p>
+        
+        <ul>
+          <li>application classes <strong>and</strong> S4 classes are in your classpath. See above.</li>
+          <li>application already configured in cluster (with the <code>-appClass</code> option, no need to package the app)</li>
+        </ul>
+        
+        <p>Then just run the <code>org.apache.s4.core.S4Node</code> class and pass:</p>
+        
+        <ul>
+          <li>the cluster name: <code>-c=testCluster1</code>
+        </li>
+        </ul>
+        
+        <p>If you use a local Zookeeper instance on the default port (2181), there is no need to specify the <code>-zk</code> option.</p>
+      </div>
     </div>
     <div id="footer">
       <div class="container">

Modified: incubator/s4/site/doc/0.6.0/event_dispatch/index.html
URL: http://svn.apache.org/viewvc/incubator/s4/site/doc/0.6.0/event_dispatch/index.html?rev=1488543&r1=1488542&r2=1488543&view=diff
==============================================================================
--- incubator/s4/site/doc/0.6.0/event_dispatch/index.html (original)
+++ incubator/s4/site/doc/0.6.0/event_dispatch/index.html Sat Jun  1 16:37:44 2013
@@ -1,16 +1,17 @@
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<!DOCTYPE html>
 <html>
-<head>
-<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
-<title>S4: Event dispatch</title>
-<meta content="A general-purpose distributed stream computing platform" name="description">
-<link href="../../../style/screen.css" media="screen" rel="stylesheet" type="text/css">
-<link href="../../../style/print.css" media="print" rel="stylesheet" type="text/css">
-<!--[if lt IE 9]>
+  <head>
+    <title>S4: Event dispatch</title>
+    <meta content="text/html; charset=utf-8" http-equiv="Content-Type">
+    <meta content="A general-purpose distributed stream computing platform" name="description">
+    <link href="../../../style/screen.css" media="screen" rel="stylesheet" type="text/css">
+    <link href="../../../style/print.css" media="print" rel="stylesheet" type="text/css">
+    <!--[if lt IE 9]>
       <link href="../../../style/ie.css" media="screen" rel="stylesheet" type="text/css">
-    <![endif]--><link href="../../../style/style.css" rel="stylesheet" type="text/css">
-<link href="../../../style/nav.css" rel="stylesheet" type="text/css">
-<script type="text/javascript">
+    <![endif]-->
+    <link href="../../../style/style.css" rel="stylesheet" type="text/css">
+    <link href="../../../style/nav.css" rel="stylesheet" type="text/css">
+    <script type="text/javascript">
         var _gaq = _gaq || [];
         _gaq.push(['_setAccount', 'UA-19490961-1']);
         _gaq.push(['_setDomainName', 'incubator.apache.org']);
@@ -20,187 +21,195 @@
           ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
           var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
         })();
-      </script><link href="../../../style/coderay.css" rel="stylesheet" type="text/css">
-<link href="../../../style/pygmentize.css" rel="stylesheet" type="text/css">
-</head>
-<body>
+      </script>
+    <link href="../../../style/coderay.css" rel="stylesheet" type="text/css">
+    <link href="../../../style/pygmentize.css" rel="stylesheet" type="text/css">
+  </head>
+  <body>
     <div id="header">
       <div class="container">
         <div id="logo">
           <a href="../../../">
-            <img src="../../../images/s4_test.png"></a>
+            <img src="../../../images/s4_test.png">
+          </a>
+        </div>
+        <div id="navbar">
+          <ul id="nav">
+            <li>
+              <a href="../../../">home</a>
+            </li>
+            <li>
+              <a href="..">doc</a>
+            </li>
+            <li>
+              <a href="http://git-wip-us.apache.org/repos/asf?p=incubator-s4.git" onclick="_gaq.push(['_trackEvent', 'External', 'Apache Git', 'http://github.com/s4']);">code</a>
+            </li>
+            <li>
+              <a href="http://people.apache.org/~mmorel/apache-s4-0.6.0-incubating-doc/javadoc/">API</a>
+            </li>
+            <li>
+              <a href="../../../contrib">get involved</a>
+            </li>
+            <li>
+              <a href="../../../team">team</a>
+            </li>
+            <li>
+              <a href="../../../download">download</a>
+            </li>
+          </ul>
         </div>
-        <div id="navbar"><ul id="nav">
-<li>
-    <a href="../../../">home</a>
-  </li>
-  <li>
-    <a href="https://cwiki.apache.org/confluence/display/S4/S4+Wiki">doc [0.5]</a>
-  </li>
-  <li>
-    <a href="http://git-wip-us.apache.org/repos/asf?p=incubator-s4.git" onclick="_gaq.push(['_trackEvent', 'External', 'Apache Git', 'http://github.com/s4']);">code</a>
-  </li>
-  <li>
-    <a href="http://people.apache.org/~mmorel/apache-s4-0.5.0-incubating-doc/javadoc/">API</a>
-  </li>
-  <li>
-    <a href="../../../contrib">get involved</a>
-  </li>
-  <li>
-    <a href="../../../team">team</a>
-  </li>
-  <li>
-    <a href="../../../download">download</a>
-  </li>
-</ul></div>
       </div>
     </div>
     <div id="wrapper">
       <div class="container" id="container">
-<blockquote>
-  <p>Exploring how events are dispatched to, from and within S4 nodes</p>
-</blockquote>
-
-<p>Events are dispatched according to their key.</p>
-
-<p>The key is identified in an <code>Event</code> through a <code>KeyFinder</code>.</p>
-
-<p>Dispatch can be configured for:</p>
-
-<ul>
-<li>dispatching events to partitions (<em>outgoing dispatch</em>)</li>
-  <li>dispatching external events within a partition  (<em>incoming dispatch</em>)</li>
-</ul>
-<h1 id="outgoing-dispatch">Outgoing dispatch</h1>
-
-<p>A stream can be defined with a KeyFinder, as :</p>
-
-<pre><code class="language-java"><span class="n">Stream</span><span class="o">&lt;</span><span class="n">TopicEvent</span><span class="o">&gt;</span> <span class="n">topicSeenStream</span> <span class="o">=</span> <span class="n">createStream</span><span class="o">(</span><span class="s">"TopicSeen"</span><span class="o">,</span> <span class="k">new</span> <span class="n">KeyFinder</span><span class="o">&lt;</span><span class="n">TopicEvent</span><span class="o">&gt;()</span> <span class="o">{</span>
-	    <span class="nd">@Override</span>
-	    <span class="kd">public</span> <span class="n">List</span><span class="o">&lt;</span><span class="n">String</span><span class="o">&gt;</span> <span class="nf">get</span><span class="o">(</span><span class="kd">final</span> <span class="n">TopicEvent</span> <span class="n">arg0</span><span class="o">)</span> <span class="o">{</span>
-	        <span class="k">return</span> <span class="n">ImmutableList</span><span class="o">.</span><span class="na">of</span><span class="o">(</span><span class="n">arg0</span><span class="o">.</span><span class="na">getTopic</span><span class="o">());</span>
-	    <span class="o">}</span>
-	<span class="o">},</span> <span class="n">topicCountAndReportPE</span><span class="o">);</span></code></pre>
-
-<p>When an event is sent to the “TopicSeen” stream, its key will be identified through the KeyFinder implementation, hashed and dispatched to the matching partition.</p>
-
-<p>The same logic applies when defining <em>output streams</em>.</p>
-
-<p>If we use an AdapterApp subclass, the <code>remoteStreamKeyFinder</code> should be defined in the <code>onInit()</code> method, <em>before</em> calling <code>super.onInit()</code>:</p>
-
-<pre><code class="language-java">    <span class="nd">@Override</span>
-    <span class="kd">protected</span> <span class="kt">void</span> <span class="nf">onInit</span><span class="o">()</span> <span class="o">{</span>
-    <span class="o">...</span> 
-    <span class="n">remoteStreamKeyFinder</span> <span class="o">=</span> <span class="k">new</span> <span class="n">KeyFinder</span><span class="o">&lt;</span><span class="n">Event</span><span class="o">&gt;()</span> <span class="o">{</span>
-        <span class="nd">@Override</span>
-        <span class="kd">public</span> <span class="n">List</span><span class="o">&lt;</span><span class="n">String</span><span class="o">&gt;</span> <span class="nf">get</span><span class="o">(</span><span class="n">Event</span> <span class="n">event</span><span class="o">)</span> <span class="o">{</span>
-            <span class="k">return</span> <span class="n">ImmutableList</span><span class="o">.</span><span class="na">of</span><span class="o">(</span><span class="n">event</span><span class="o">.</span><span class="na">get</span><span class="o">(</span><span class="s">"theKeyField"</span><span class="o">));</span>
-        <span class="o">}</span>
-    <span class="o">};</span>
-    <span class="kd">super</span><span class="o">.</span><span class="na">onInit</span><span class="o">()</span>
-    <span class="o">...</span></code></pre>
-
-<p>If we use a standard App, we use the <code>createOutputStream(String name, KeyFinder&lt;Event&gt; keyFinder)</code> method.</p>
-
-<blockquote>
-  <p>If the KeyFinder is not defined for the output streams, events are sent to partitions of the connected cluster in a round robin fashion.</p>
-</blockquote>
-
-<h1 id="incoming-dispatch-from-external-events">Incoming dispatch from external events</h1>
-
-<p>When receiving events from a remote application, we <em>must</em> define how external events are dispatched internally, to which PEs and based on which keys. For that purpose, we simply define and <em>input stream</em> with the corresponding KeyFinder:</p>
-
-<pre><code class="language-java"><span class="n">createInputStream</span><span class="o">(</span><span class="s">"names"</span><span class="o">,</span> <span class="k">new</span> <span class="n">KeyFinder</span><span class="o">&lt;</span><span class="n">Event</span><span class="o">&gt;()</span> <span class="o">{</span>
-
-    <span class="nd">@Override</span>
-    <span class="kd">public</span> <span class="n">List</span><span class="o">&lt;</span><span class="n">String</span><span class="o">&gt;</span> <span class="nf">get</span><span class="o">(</span><span class="n">Event</span> <span class="n">event</span><span class="o">)</span> <span class="o">{</span>
-        <span class="k">return</span> <span class="n">Arrays</span><span class="o">.</span><span class="na">asList</span><span class="o">(</span><span class="k">new</span> <span class="n">String</span><span class="o">[]</span> <span class="o">{</span> <span class="n">event</span><span class="o">.</span><span class="na">get</span><span class="o">(</span><span class="s">"name"</span><span class="o">)</span> <span class="o">});</span>
-       <span class="o">}</span>
-    <span class="o">},</span> <span class="n">helloPE</span><span class="o">);</span></code></pre>
-
-<p>In this case, a name is extracted from each event, the PE instance with this key is retrieved or created, and the event sent to that instance.</p>
-
-<p>Alternatively, we can use a unique PE instance for processing events in a given node. For that we simply define the input stream without a KeyFinder, <em>and</em> use a singleton PE:</p>
-
-<pre><code class="language-java"><span class="n">HelloPE</span> <span class="n">helloPE</span> <span class="o">=</span> <span class="n">createPE</span><span class="o">(</span><span class="n">HelloPE</span><span class="o">.</span><span class="na">class</span><span class="o">);</span>
-<span class="n">helloPE</span><span class="o">.</span><span class="na">setSingleton</span><span class="o">(</span><span class="kc">true</span><span class="o">);</span>
-<span class="n">createInputStream</span><span class="o">(</span><span class="s">"names"</span><span class="o">,</span> <span class="n">helloPE</span><span class="o">);</span></code></pre>
-
-<p>In this case, all events will be dispatched to the only HelloPE instance in this partition, regardless of the content of the event.</p>
-
-<h1 id="internals-and-tuning">Internals and tuning</h1>
-
-<p>S4 follows a staged event driven architecture and uses a pipeline of executors to process messages. </p>
-
-<h2 id="executors">executors</h2>
-<p>An executor is an object that executes tasks. It usually keeps a bounded queue of task items and schedules their execution through a pool of threads.</p>
-
-<p>When processing queues are full, executors may adopt various possible behaviours, in particular, in S4:</p>
-
-<ul>
-<li>
-<strong>blocking</strong>: the current thread simply waits until the queue is not full</li>
-  <li>
-<strong>shedding</strong>: the current event is dropped</li>
-  <li>
-<strong>throttling</strong>, i.e. placing an upper bound on the processing rate, is a convenient way to avoid sending too many messages too fast.</li>
-</ul>
-<p>S4 provides various default implementations of these behaviours and you can also define your own custom executors as appropriate.</p>
-
-<h2 id="workflow">workflow</h2>
-
-<p>The following picture illustrates the pipeline of executors.</p>
-
-<p><img src="../../../images/doc/0.6.0/executors.png" alt="image"></p>
-
-<h3 id="when-a-node-receives-a-message">When a node receives a message:</h3>
-
-<ol>
-<li>data is received on a socket and chunked into a message, in the form of an array of bytes</li>
-  <li>the message is passed to a deserializer executor
-    <ul>
-<li>this executor is loaded with the application, and therefore has access to application classes, so that application specific messages can be deserialized</li>
-      <li>by default it uses 1 thread and <strong>blocks</strong> if the processing queue is full</li>
-    </ul>
-</li>
-  <li>the event (the deserialized message) is dispatched to a stream executor 
-    <ul>
-<li>the stream executor is selected according to the stream information contained in the event</li>
-      <li>by default it <strong>blocks</strong> if the processing queue is full</li>
-    </ul>
-</li>
-  <li>the event is processed in the PE instance that matches the key of the event</li>
-</ol>
-<h3 id="when-a-pe-emits-a-message">When a PE emits a message:</h3>
-
-<ol>
-<li>an event is passed to a referenced stream</li>
-  <li>if the target cluster is remote, the event is passed to a remote sender executor</li>
-  <li>otherwise, 
-    <ul>
-<li>if the target partition is the current one, the event is directly passed to the corresponding stream executor (see step 3 above)</li>
-      <li>otherwise, the event is passed to a local sender executor</li>
-    </ul>
-</li>
-  <li>remote or local sender executors serialize the event into an array of bytes
-    <ul>
-<li>remote sender executors are <strong>blocking</strong> by default, if their processing queue is full</li>
-      <li>local sender executors are <strong>throttling</strong> by default, with a configurable maximum rate. If events arrive at a higher rate, they are <strong>dropped</strong>.</li>
-    </ul>
-</li>
-</ol>
-<h2 id="configuration-parameters">configuration parameters</h2>
-
-<ul>
-<li>blocking executors can lead to deadlocks, depending on the application graph</li>
-  <li>back pressure can be taken into account when using TCP: if downstream systems saturate, messages cannot be sent downstream, and sending queues fill up. With knowledge of the application, is is possible to add some mechanisms to react appropriately</li>
-  <li>executors can be replaced by other implementations in custom modules, by overriding the appropriate bindings (see <code>DefaultCommModule</code> and <code>DefaultCodeModule</code>)</li>
-  <li>the maximum number of threads to use to process a given stream can be specified <strong>in the application</strong>, using the <code>setParallelism()</code> method of <code>Stream</code>
-</li>
-  <li>default parameters are specified in <code>default.s4.comm.properties</code>
-</li>
-</ul>
-</div>
+        <blockquote>
+          <p>Exploring how events are dispatched to, from and within S4 nodes</p>
+        </blockquote>
+        
+        <p>Events are dispatched according to their key.</p>
+        
+        <p>The key is identified in an <code>Event</code> through a <code>KeyFinder</code>.</p>
+        
+        <p>Dispatch can be configured for:</p>
+        
+        <ul>
+          <li>dispatching events to partitions (<em>outgoing dispatch</em>)</li>
+          <li>dispatching external events within a partition  (<em>incoming dispatch</em>)</li>
+        </ul>
+        
+        <h1 id="outgoing-dispatch">Outgoing dispatch</h1>
+        
+        <p>A stream can be defined with a KeyFinder, as :</p>
+        
+        <pre><code class="language-java"><span class="n">Stream</span><span class="o">&lt;</span><span class="n">TopicEvent</span><span class="o">&gt;</span> <span class="n">topicSeenStream</span> <span class="o">=</span> <span class="n">createStream</span><span class="o">(</span><span class="s">"TopicSeen"</span><span class="o">,</span> <span class="k">new</span> <span class="n">KeyFinder</span><span class="o">&lt;</span><span class="n">TopicEvent</span><span class="o">&gt;()</span> <span class="o">{</span>
+        	    <span class="nd">@Override</span>
+        	    <span class="kd">public</span> <span class="n">List</span><span class="o">&lt;</span><span class="n">String</span><span class="o">&gt;</span> <span class="nf">get</span><span class="o">(</span><span class="kd">final</span> <span class="n">TopicEvent</span> <span class="n">arg0</span><span class="o">)</span> <span class="o">{</span>
+        	        <span class="k">return</span> <span class="n">ImmutableList</span><span class="o">.</span><span class="na">of</span><span class="o">(</span><span class="n">arg0</span><span class="o">.</span><span class="na">getTopic</span><span class="o">());</span>
+        	    <span class="o">}</span>
+        	<span class="o">},</span> <span class="n">topicCountAndReportPE</span><span class="o">);</span></code></pre>
+        
+        <p>When an event is sent to the “TopicSeen” stream, its key will be identified through the KeyFinder implementation, hashed and dispatched to the matching partition.</p>
+        
+        <p>The same logic applies when defining <em>output streams</em>.</p>
+        
+        <p>If we use an AdapterApp subclass, the <code>remoteStreamKeyFinder</code> should be defined in the <code>onInit()</code> method, <em>before</em> calling <code>super.onInit()</code>:</p>
+        
+        <pre><code class="language-java">    <span class="nd">@Override</span>
+            <span class="kd">protected</span> <span class="kt">void</span> <span class="nf">onInit</span><span class="o">()</span> <span class="o">{</span>
+            <span class="o">...</span> 
+            <span class="n">remoteStreamKeyFinder</span> <span class="o">=</span> <span class="k">new</span> <span class="n">KeyFinder</span><span class="o">&lt;</span><span class="n">Event</span><span class="o">&gt;()</span> <span class="o">{</span>
+                <span class="nd">@Override</span>
+                <span class="kd">public</span> <span class="n">List</span><span class="o">&lt;</span><span class="n">String</span><span class="o">&gt;</span> <span class="nf">get</span><span class="o">(</span><span class="n">Event</span> <span class="n">event</span><span class="o">)</span> <span class="o">{</span>
+                    <span class="k">return</span> <span class="n">ImmutableList</span><span class="o">.</span><span class="na">of</span><span class="o">(</span><span class="n">event</span><span class="o">.</span><span class="na">get</span><span class="o">(</span><span class="s">"theKeyField"</span><span class="o">));</span>
+                <span class="o">}</span>
+            <span class="o">};</span>
+            <span class="kd">super</span><span class="o">.</span><span class="na">onInit</span><span class="o">()</span>
+            <span class="o">...</span></code></pre>
+        
+        <p>If we use a standard App, we use the <code>createOutputStream(String name, KeyFinder&lt;Event&gt; keyFinder)</code> method.</p>
+        
+        <blockquote>
+          <p>If the KeyFinder is not defined for the output streams, events are sent to partitions of the connected cluster in a round robin fashion.</p>
+        </blockquote>
+        
+        <h1 id="incoming-dispatch-from-external-events">Incoming dispatch from external events</h1>
+        
+        <p>When receiving events from a remote application, we <em>must</em> define how external events are dispatched internally, to which PEs and based on which keys. For that purpose, we simply define and <em>input stream</em> with the corresponding KeyFinder:</p>
+        
+        <pre><code class="language-java"><span class="n">createInputStream</span><span class="o">(</span><span class="s">"names"</span><span class="o">,</span> <span class="k">new</span> <span class="n">KeyFinder</span><span class="o">&lt;</span><span class="n">Event</span><span class="o">&gt;()</span> <span class="o">{</span>
+        
+            <span class="nd">@Override</span>
+            <span class="kd">public</span> <span class="n">List</span><span class="o">&lt;</span><span class="n">String</span><span class="o">&gt;</span> <span class="nf">get</span><span class="o">(</span><span class="n">Event</span> <span class="n">event</span><span class="o">)</span> <span class="o">{</span>
+                <span class="k">return</span> <span class="n">Arrays</span><span class="o">.</span><span class="na">asList</span><span class="o">(</span><span class="k">new</span> <span class="n">String</span><span class="o">[]</span> <span class="o">{</span> <span class="n">event</span><span class="o">.</span><span class="na">get</span><span class="o">(</span><span class="s">"name"</span><span class="o">)</span> <span class="o">});</span>
+               <span class="o">}</span>
+            <span class="o">},</span> <span class="n">helloPE</span><span class="o">);</span></code></pre>
+        
+        <p>In this case, a name is extracted from each event, the PE instance with this key is retrieved or created, and the event sent to that instance.</p>
+        
+        <p>Alternatively, we can use a unique PE instance for processing events in a given node. For that we simply define the input stream without a KeyFinder, <em>and</em> use a singleton PE:</p>
+        
+        <pre><code class="language-java"><span class="n">HelloPE</span> <span class="n">helloPE</span> <span class="o">=</span> <span class="n">createPE</span><span class="o">(</span><span class="n">HelloPE</span><span class="o">.</span><span class="na">class</span><span class="o">);</span>
+        <span class="n">helloPE</span><span class="o">.</span><span class="na">setSingleton</span><span class="o">(</span><span class="kc">true</span><span class="o">);</span>
+        <span class="n">createInputStream</span><span class="o">(</span><span class="s">"names"</span><span class="o">,</span> <span class="n">helloPE</span><span class="o">);</span></code></pre>
+        
+        <p>In this case, all events will be dispatched to the only HelloPE instance in this partition, regardless of the content of the event.</p>
+        
+        <h1 id="internals-and-tuning">Internals and tuning</h1>
+        
+        <p>S4 follows a staged event driven architecture and uses a pipeline of executors to process messages. </p>
+        
+        <h2 id="executors">executors</h2>
+        <p>An executor is an object that executes tasks. It usually keeps a bounded queue of task items and schedules their execution through a pool of threads.</p>
+        
+        <p>When processing queues are full, executors may adopt various possible behaviours, in particular, in S4:</p>
+        
+        <ul>
+          <li>
+        <strong>blocking</strong>: the current thread simply waits until the queue is not full</li>
+          <li>
+        <strong>shedding</strong>: the current event is dropped</li>
+          <li>
+        <strong>throttling</strong>, i.e. placing an upper bound on the processing rate, is a convenient way to avoid sending too many messages too fast.</li>
+        </ul>
+        
+        <p>S4 provides various default implementations of these behaviours and you can also define your own custom executors as appropriate.</p>
+        
+        <h2 id="workflow">workflow</h2>
+        
+        <p>The following picture illustrates the pipeline of executors.</p>
+        
+        <p><img src="../../../images/doc/0.6.0/executors.png" alt="image"></p>
+        
+        <h3 id="when-a-node-receives-a-message">When a node receives a message:</h3>
+        
+        <ol>
+          <li>data is received on a socket and chunked into a message, in the form of an array of bytes</li>
+          <li>the message is passed to a deserializer executor
+            <ul>
+              <li>this executor is loaded with the application, and therefore has access to application classes, so that application specific messages can be deserialized</li>
+              <li>by default it uses 1 thread and <strong>blocks</strong> if the processing queue is full</li>
+            </ul>
+          </li>
+          <li>the event (the deserialized message) is dispatched to a stream executor 
+            <ul>
+              <li>the stream executor is selected according to the stream information contained in the event</li>
+              <li>by default it <strong>blocks</strong> if the processing queue is full</li>
+            </ul>
+          </li>
+          <li>the event is processed in the PE instance that matches the key of the event</li>
+        </ol>
+        
+        <h3 id="when-a-pe-emits-a-message">When a PE emits a message:</h3>
+        
+        <ol>
+          <li>an event is passed to a referenced stream</li>
+          <li>if the target cluster is remote, the event is passed to a remote sender executor</li>
+          <li>otherwise, 
+            <ul>
+              <li>if the target partition is the current one, the event is directly passed to the corresponding stream executor (see step 3 above)</li>
+              <li>otherwise, the event is passed to a local sender executor</li>
+            </ul>
+          </li>
+          <li>remote or local sender executors serialize the event into an array of bytes
+            <ul>
+              <li>remote sender executors are <strong>blocking</strong> by default, if their processing queue is full</li>
+              <li>local sender executors are <strong>throttling</strong> by default, with a configurable maximum rate. If events arrive at a higher rate, they are <strong>dropped</strong>.</li>
+            </ul>
+          </li>
+        </ol>
+        
+        <h2 id="configuration-parameters">configuration parameters</h2>
+        
+        <ul>
+          <li>blocking executors can lead to deadlocks, depending on the application graph</li>
+          <li>back pressure can be taken into account when using TCP: if downstream systems saturate, messages cannot be sent downstream, and sending queues fill up. With knowledge of the application, is is possible to add some mechanisms to react appropriately</li>
+          <li>executors can be replaced by other implementations in custom modules, by overriding the appropriate bindings (see <code>DefaultCommModule</code> and <code>DefaultCodeModule</code>)</li>
+          <li>the maximum number of threads to use to process a given stream can be specified <strong>in the application</strong>, using the <code>setParallelism()</code> method of <code>Stream</code>
+        </li>
+          <li>default parameters are specified in <code>default.s4.comm.properties</code>
+        </li>
+        </ul>
+      </div>
     </div>
     <div id="footer">
       <div class="container">



Mime
View raw message