incubator-s4-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mmo...@apache.org
Subject [37/50] [abbrv] git commit: added classloading of generated classes
Date Tue, 03 Jan 2012 14:03:28 GMT
added classloading of generated classes

- also: generic loading mechanism for applications
- added regression test (still to be improved)
- temporarily ignore hanging UDPTest


Project: http://git-wip-us.apache.org/repos/asf/incubator-s4/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s4/commit/29b00ef2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s4/tree/29b00ef2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s4/diff/29b00ef2

Branch: refs/heads/piper
Commit: 29b00ef2bc3c8427ea7c8e22e3dcdedbaa6244de
Parents: a6b382d
Author: Matthieu Morel <mmorel@apache.org>
Authored: Wed Oct 26 13:23:19 2011 +0200
Committer: Matthieu Morel <mmorel@apache.org>
Committed: Wed Oct 26 16:01:17 2011 +0200

----------------------------------------------------------------------
 .../java/org/apache/s4/base/util/JarLoader.java    |   65 ------
 .../org/apache/s4/base/util/MultiClassLoader.java  |   14 +-
 .../java/org/apache/s4/base/util/S4RLoader.java    |   84 ++++++++
 .../src/test/java/org/apache/s4/comm/UDPTest.java  |    6 +-
 .../src/main/java/org/apache/s4/core/Server.java   |  113 ++++++-----
 .../s4/core/gen/OverloadDispatcherGenerator.java   |   60 ++++--
 .../test/s4/core/apploading/AppLoadingTest.java    |  164 +++++++++++++++
 .../java/test/s4/core/apploading/SimpleApp.java    |   47 ++++
 .../java/test/s4/core/apploading/SimpleModule.java |    7 +
 .../java/test/s4/core/apploading/SimplePE.java     |   66 ++++++
 .../src/test/java/test/s4/fixtures/TestUtils.java  |   49 +++--
 11 files changed, 518 insertions(+), 157 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/29b00ef2/subprojects/s4-base/src/main/java/org/apache/s4/base/util/JarLoader.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-base/src/main/java/org/apache/s4/base/util/JarLoader.java b/subprojects/s4-base/src/main/java/org/apache/s4/base/util/JarLoader.java
deleted file mode 100644
index 7fc9c06..0000000
--- a/subprojects/s4-base/src/main/java/org/apache/s4/base/util/JarLoader.java
+++ /dev/null
@@ -1,65 +0,0 @@
-package org.apache.s4.base.util;
-
-import java.util.ArrayList;
-import java.util.Enumeration;
-import java.util.List;
-import java.util.jar.JarEntry;
-import java.util.jar.JarFile;
-
-/**
- * 
- * CREDITS
- * 
- * <p>
- * The source code for this class was derived from <a href=
- * "http://code.google.com/p/db4o-om/source/browse/trunk/objectmanager-api/src/com/db4o/objectmanager/configuration/MultiClassLoader.java"
- * >this project</a>.
- * 
- * 
- */
-public class JarLoader extends MultiClassLoader {
-
-    private JarResources jarResource;
-
-    public JarLoader(String jarPath) {
-        jarResource = new JarResources(jarPath);
-    }
-
-    @Override
-    protected byte[] loadClassBytes(String className) {
-        className = formatClassName(className);
-        return jarResource.getResource(className);
-    }
-
-    public List<Class<?>> getClasses(String path) {
-        List<Class<?>> classes = new ArrayList<Class<?>>();
-        try {
-            JarLoader jarLoader = new JarLoader(path);
-            JarFile jarFile = new JarFile(path);
-            for (Enumeration<JarEntry> e = jarFile.entries(); e
-                    .hasMoreElements();) {
-                try {
-                    JarEntry entry = e.nextElement();
-                    if (entry.getName().endsWith(".class")) {
-                        String className = entry
-                                .getName()
-                                .substring(0, entry.getName().indexOf(".class"))
-                                .replace("/", ".").replace("\\", ".");
-
-                        Class<?> clazz = jarLoader.loadClass(className);
-                        // clazz.asSubclass(IoAdapter.class);
-                        if (clazz != null) {
-                            classes.add(clazz);
-                        }
-                    }
-                } catch (Exception ex) {
-                }
-            }
-
-        } catch (Exception e) {
-            e.printStackTrace();
-        }
-        return classes;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/29b00ef2/subprojects/s4-base/src/main/java/org/apache/s4/base/util/MultiClassLoader.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-base/src/main/java/org/apache/s4/base/util/MultiClassLoader.java
b/subprojects/s4-base/src/main/java/org/apache/s4/base/util/MultiClassLoader.java
index ab1dabf..8c702df 100644
--- a/subprojects/s4-base/src/main/java/org/apache/s4/base/util/MultiClassLoader.java
+++ b/subprojects/s4-base/src/main/java/org/apache/s4/base/util/MultiClassLoader.java
@@ -28,7 +28,7 @@ abstract public class MultiClassLoader extends ClassLoader {
     private static final Logger logger = LoggerFactory
             .getLogger(MultiClassLoader.class);
 
-    private Map<String, Class<?>> classes;
+    private final Map<String, Class<?>> classes;
     private char classNameReplacementChar;
 
     public MultiClassLoader() {
@@ -40,10 +40,12 @@ abstract public class MultiClassLoader extends ClassLoader {
      * This is a simple version for external clients since they will always want
      * the class resolved before it is returned to them.
      */
+    @Override
     public Class<?> loadClass(String className) throws ClassNotFoundException {
         return (loadClass(className, true));
     }
 
+    @Override
     public synchronized Class<?> loadClass(String className, boolean resolveIt)
             throws ClassNotFoundException {
 
@@ -53,19 +55,19 @@ abstract public class MultiClassLoader extends ClassLoader {
                 + ", resolveIt: " + resolveIt);
 
         /* Check our local cache of classes. */
-        result = (Class<?>) classes.get(className);
+        result = classes.get(className);
         if (result != null) {
-            logger.debug("Returning cached result.");
+            logger.debug("Returning cached result for class [{}]", className);
             return result;
         }
 
         /* Check with the primordial class loader. */
         try {
             result = super.findSystemClass(className);
-            logger.debug("Returning system class (in CLASSPATH).");
+            logger.debug("Returning system class (in CLASSPATH) [{}]", className);
             return result;
         } catch (ClassNotFoundException e) {
-            logger.debug("Not a system class.");
+            logger.debug("Not a system class [{}]", className);
         }
 
         classBytes = loadClassBytes(className);
@@ -92,7 +94,7 @@ abstract public class MultiClassLoader extends ClassLoader {
         if (result == null)
             return null;
         classes.put(className, result);
-        logger.debug("Returning newly loaded class.");
+        logger.debug("Returning newly loaded class [{}]", className);
         return result;
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/29b00ef2/subprojects/s4-base/src/main/java/org/apache/s4/base/util/S4RLoader.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-base/src/main/java/org/apache/s4/base/util/S4RLoader.java b/subprojects/s4-base/src/main/java/org/apache/s4/base/util/S4RLoader.java
new file mode 100644
index 0000000..0d54454
--- /dev/null
+++ b/subprojects/s4-base/src/main/java/org/apache/s4/base/util/S4RLoader.java
@@ -0,0 +1,84 @@
+package org.apache.s4.base.util;
+
+import java.util.ArrayList;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
+
+import com.google.common.collect.MapMaker;
+
+/**
+ * 
+ * CREDITS
+ * 
+ * <p>
+ * The source code for this class was derived from <a href=
+ * "http://code.google.com/p/db4o-om/source/browse/trunk/objectmanager-api/src/com/db4o/objectmanager/configuration/MultiClassLoader.java"
+ * >this project</a>.
+ * 
+ * 
+ */
+public class S4RLoader extends MultiClassLoader {
+
+    private final JarResources jarResource;
+    private final Map<String, byte[]> generatedClassBytes = new HashMap<String,
byte[]>();
+    
+    public S4RLoader(String jarPath) {
+        jarResource = new JarResources(jarPath);
+    }
+
+    /**
+     * In order to load dynamically generated classes with the same classloader than 
+     * the one used for loading application classes from an s4r archive, we register these
+     * generated classes and bytecode in this classloader. They can be picked later.
+     * 
+     */
+    public void addGeneratedClassBytes(String className, byte[] classBytes) {
+        generatedClassBytes.put(className, classBytes);
+    }
+    
+    @Override
+    protected byte[] loadClassBytes(String className) {
+        if (generatedClassBytes.containsKey(className)) {
+            // note: no need to keep that data any longer
+            return generatedClassBytes.remove(className);
+        }
+        className = formatClassName(className);
+        return jarResource.getResource(className);
+    }
+
+    public List<Class<?>> getClasses(String path) {
+        List<Class<?>> classes = new ArrayList<Class<?>>();
+        try {
+            S4RLoader jarLoader = new S4RLoader(path);
+            JarFile jarFile = new JarFile(path);
+            for (Enumeration<JarEntry> e = jarFile.entries(); e
+                    .hasMoreElements();) {
+                try {
+                    JarEntry entry = e.nextElement();
+                    if (entry.getName().endsWith(".class")) {
+                        String className = entry
+                                .getName()
+                                .substring(0, entry.getName().indexOf(".class"))
+                                .replace("/", ".").replace("\\", ".");
+
+                        Class<?> clazz = jarLoader.loadClass(className);
+                        // clazz.asSubclass(IoAdapter.class);
+                        if (clazz != null) {
+                            classes.add(clazz);
+                        }
+                    }
+                } catch (Exception ex) {
+                }
+            }
+
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+        return classes;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/29b00ef2/subprojects/s4-comm/src/test/java/org/apache/s4/comm/UDPTest.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-comm/src/test/java/org/apache/s4/comm/UDPTest.java b/subprojects/s4-comm/src/test/java/org/apache/s4/comm/UDPTest.java
index 720dc28..289790b 100644
--- a/subprojects/s4-comm/src/test/java/org/apache/s4/comm/UDPTest.java
+++ b/subprojects/s4-comm/src/test/java/org/apache/s4/comm/UDPTest.java
@@ -18,6 +18,7 @@ import org.apache.s4.comm.topology.Topology;
 import org.apache.s4.comm.topology.TopologyFromFile;
 import org.apache.s4.comm.udp.UDPEmitter;
 import org.apache.s4.comm.udp.UDPListener;
+import org.junit.Ignore;
 
 import com.google.inject.AbstractModule;
 import com.google.inject.Binder;
@@ -25,7 +26,10 @@ import com.google.inject.Guice;
 import com.google.inject.Injector;
 import com.google.inject.name.Names;
 
-public class UDPTest extends SimpleDeliveryTest {
+// TODO: this test class excluded and set as abstract because:
+// 1. test hangs and blocks the whole test suite
+// 2. it does not use junit 4 and therefore there is no simple way to time out
+public abstract class UDPTest extends SimpleDeliveryTest {
 
 	@Override
 	protected void setUp() {

http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/29b00ef2/subprojects/s4-core/src/main/java/org/apache/s4/core/Server.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-core/src/main/java/org/apache/s4/core/Server.java b/subprojects/s4-core/src/main/java/org/apache/s4/core/Server.java
index ac4b296..20a2e52 100644
--- a/subprojects/s4-core/src/main/java/org/apache/s4/core/Server.java
+++ b/subprojects/s4-core/src/main/java/org/apache/s4/core/Server.java
@@ -1,16 +1,19 @@
 package org.apache.s4.core;
 
-import java.net.URL;
-import java.util.jar.Attributes;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.jar.Attributes.Name;
 import java.util.jar.JarFile;
-import java.util.jar.Manifest;
 
-import org.apache.s4.base.util.JarLoader;
+import org.apache.s4.base.util.S4RLoader;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import ch.qos.logback.classic.Level;
 
+import com.google.common.io.PatternFilenameFilter;
 import com.google.inject.AbstractModule;
 import com.google.inject.Guice;
 import com.google.inject.Inject;
@@ -24,21 +27,26 @@ import com.google.inject.name.Named;
  */
 public class Server {
 
-    final private String moduleName;
+    private static final Logger logger = LoggerFactory.getLogger(Server.class);
+
+    final private String commModuleName;
     final private String logLevel;
+    public static final String MANIFEST_S4_APP_CLASS = "S4-App-Class";
+    // NOTE: currently we use a directory, but this will be changed by a URL (ref to zookeeper?),

+    // so that applications can be downloaded from a remote repository
+    final private static String S4_APPS_PATH = System.getProperty("s4.apps.path", System.getProperty("user.dir")
+            + "/bin/apps");
+    List<App> apps = new ArrayList<App>();
 
     /**
      * 
      */
     @Inject
-    public Server(@Named("comm.module") String moduleName,
-            @Named("s4.logger_level") String logLevel) {
-        this.moduleName = moduleName;
+    public Server(@Named("comm.module") String commModuleName, @Named("s4.logger_level")
String logLevel) {
+        this.commModuleName = commModuleName;
         this.logLevel = logLevel;
     }
 
-    private static final Logger logger = LoggerFactory.getLogger(Server.class);
-
     public void start() throws Exception {
 
         /* Set up logger basic configuration. */
@@ -51,7 +59,7 @@ public class Server {
 
         /* Initialize communication layer module. */
         try {
-            module = (AbstractModule) Class.forName(moduleName).newInstance();
+            module = (AbstractModule) Class.forName(commModuleName).newInstance();
         } catch (Exception e) {
             logger.error("Unable to instantiate communication layer module.", e);
         }
@@ -59,55 +67,54 @@ public class Server {
         /* After some indirection we get the injector. */
         injector = Guice.createInjector(module);
 
-        // HERE WE SHOULD LOOP TO CHECK IF WE NEED TO LOAD OR UNLOAD APPS.
-
-        // MAKE SURE YOU COPY THE RESOURCE TO THE CLASSPATH
-        // example: subprojects/s4-core/bin/apps/MY_RESOURCE (in Eclipse)
-        // String resource = "/apps/HelloApp.jar";
-        String resource = "/apps/CounterExample.s4r";
-        // Read the jar as a resource into a URL.
-        URL url = this.getClass().getResource(resource);
-        if (url == null) {
-            logger.error("Couldn't read resource.");
-            System.exit(-1);
+        File[] s4rFiles = new File(S4_APPS_PATH).listFiles(new PatternFilenameFilter("\\w+\\.s4r"));
+        for (File s4rFile : s4rFiles) {
+            loadApp(injector, s4rFile);
         }
-        logger.trace("Read: {}", url.toString());
 
-        /* Convert the URL to a File and load the jar. */
-        JarLoader cl = new JarLoader(url.getFile());
+        // now init + start apps
+        for (App app : apps) {
+            logger.info("Starting app " + app.getClass().getName());
+            app.init();
+            app.start();
+        }
+
+        logger.info("Completed applications startup");
+
+    }
 
-        /* Read MANIFEST main attributes. We need the name of the App class. */
-        String appClassName="";
+    private void loadApp(Injector injector, File s4r) {
+
+        S4RLoader cl = new S4RLoader(s4r.getAbsolutePath());
         try {
-            JarFile jar = new JarFile(url.getFile());
-            Manifest manifest = jar.getManifest();
-            Attributes attributes = manifest.getMainAttributes();
-            for (Object name : attributes.keySet()) {
-                logger.debug(name + ": " + attributes.getValue((Attributes.Name)name));
+            JarFile s4rFile = new JarFile(s4r);
+            if (s4rFile.getManifest() == null) {
+                logger.warn("Cannot load s4r archive [{}] : missing manifest file");
+                return;
+            }
+            if (!s4rFile.getManifest().getMainAttributes().containsKey(new Name(MANIFEST_S4_APP_CLASS)))
{
+                logger.warn("Cannot load s4r archive [{}] : missing attribute [{}] in manifest",
s4r.getAbsolutePath(),
+                        MANIFEST_S4_APP_CLASS);
+                return;
+            }
+            String appClassName = s4rFile.getManifest().getMainAttributes().getValue(MANIFEST_S4_APP_CLASS);
+            App app = null;
+
+            try {
+                Object o = (cl.loadClass(appClassName)).newInstance();
+                app = (App) o;
+            } catch (Exception e) {
+                logger.error("Could not load s4 application form s4r file [{" + s4r.getAbsolutePath()
+ "}]", e);
+                return;
             }
-            appClassName = attributes.getValue("S4-App-Class");
-        } catch (Exception e) {
-           logger.error(e.getMessage(), e);
-        }
-        
-        logger.info("Loading application class: " + appClassName);
-        App myApp = null;
 
-        /* Create app. App must have a zero-arg constructor. */
-        try {
-            Object o = (cl.loadClass(appClassName)).newInstance();
-            myApp = (App) o;
-        } catch (Exception e) {
-            System.out.println("Caught exception : " + e);
-            e.printStackTrace();
+            Sender sender = injector.getInstance(Sender.class);
+            Receiver receiver = injector.getInstance(Receiver.class);
+            app.setCommLayer(sender, receiver);
+            apps.add(app);
+        } catch (IOException e) {
+            logger.error("Could not load s4 application form s4r file [{" + s4r.getAbsolutePath()
+ "}]", e);
         }
 
-        /* Set up app and call life-cycle methods. */
-        Sender sender = injector.getInstance(Sender.class);
-        Receiver receiver = injector.getInstance(Receiver.class);
-        myApp.setCommLayer(sender, receiver);
-        myApp.init();
-        myApp.start();
-        myApp.close();
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/29b00ef2/subprojects/s4-core/src/main/java/org/apache/s4/core/gen/OverloadDispatcherGenerator.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-core/src/main/java/org/apache/s4/core/gen/OverloadDispatcherGenerator.java
b/subprojects/s4-core/src/main/java/org/apache/s4/core/gen/OverloadDispatcherGenerator.java
index 2847690..5e2fdec 100644
--- a/subprojects/s4-core/src/main/java/org/apache/s4/core/gen/OverloadDispatcherGenerator.java
+++ b/subprojects/s4-core/src/main/java/org/apache/s4/core/gen/OverloadDispatcherGenerator.java
@@ -20,14 +20,14 @@ import static org.objectweb.asm.Opcodes.V1_6;
 import java.io.File;
 import java.io.IOException;
 import java.lang.reflect.Method;
-import java.net.URL;
-import java.net.URLClassLoader;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Random;
 
 import org.apache.s4.base.Event;
+import org.apache.s4.base.util.MultiClassLoader;
+import org.apache.s4.base.util.S4RLoader;
 import org.apache.s4.core.ProcessingElement;
 import org.objectweb.asm.ClassWriter;
 import org.objectweb.asm.Label;
@@ -44,8 +44,8 @@ import com.google.common.io.Files;
  * 
  * <p>
  * When an event is transferred to a processing element, the generated proxy
- * finds the corresponding <code>onEvent</code> method with the event
- * type argument matching the current parameter and calls this method.
+ * finds the corresponding <code>onEvent</code> method with the event type
+ * argument matching the current parameter and calls this method.
  * </p>
  * <p>
  * If there is no exact match, the closest type in the hierarchy of events is
@@ -61,7 +61,7 @@ public class OverloadDispatcherGenerator {
     private final List<Hierarchy> inputEventHierarchies = new ArrayList<Hierarchy>();
     private final List<Hierarchy> outputEventHierarchies = new ArrayList<Hierarchy>();
     private Class<?> targetClass;
-    private static final boolean DUMP = false;
+    private static final boolean DUMP = true;
 
     public OverloadDispatcherGenerator() {
     }
@@ -112,20 +112,21 @@ public class OverloadDispatcherGenerator {
         generateEventDispatchMethod(cw, "dispatchEvent", inputEventHierarchies, "onEvent");
         // dispatch output events method
         generateEventDispatchMethod(cw, "dispatchTrigger", outputEventHierarchies, "onTrigger");
-        
+
         cw.visitEnd();
 
         if (DUMP) {
             try {
                 LoggerFactory.getLogger(getClass()).debug(
                         "Dumping generated overload dispatcher class for PE of class [" +
targetClass + "]");
-                Files.write(cw.toByteArray(), new File(System.getProperty("user.dir") + "/"
+ dispatcherClassName
+                Files.write(cw.toByteArray(), new File(System.getProperty("java.io.tmpdir")
+ "/" + dispatcherClassName
                         + ".class"));
             } catch (IOException e) {
                 e.printStackTrace();
             }
         }
-        return new OverloadDispatcherClassLoader().loadClassFromBytes(dispatcherClassName,
cw.toByteArray());
+        return new OverloadDispatcherClassLoader(targetClass.getClassLoader()).loadClassFromBytes(dispatcherClassName,
+                cw.toByteArray());
 
     }
 
@@ -219,8 +220,7 @@ public class OverloadDispatcherGenerator {
         private final List<Class<?>> classes = new ArrayList<Class<?>>();
 
         public Hierarchy(Class<?> clazz) {
-            for (Class<?> currentClass = clazz; currentClass != null; currentClass
= currentClass
-                    .getSuperclass()) {
+            for (Class<?> currentClass = clazz; currentClass != null; currentClass
= currentClass.getSuperclass()) {
                 classes.add(currentClass);
             }
         }
@@ -267,18 +267,42 @@ public class OverloadDispatcherGenerator {
         }
     }
 
-    public static class OverloadDispatcherClassLoader extends URLClassLoader {
-        public OverloadDispatcherClassLoader() {
-            super(new URL[] {});
+    /**
+     * 
+     * Delegates to S4Classloader if it was used to load the PE prototype class, 
+     * by passing to S4Classloader the generated bytecode.
+     * 
+     * Falls back to parent classloader otherwise.
+     *
+     */
+    private static class OverloadDispatcherClassLoader extends ClassLoader {
+
+        ClassLoader s4AppLoader;
+
+        public OverloadDispatcherClassLoader(ClassLoader s4AppLoader) {
+            this.s4AppLoader = s4AppLoader;
         }
 
         public Class<?> loadClassFromBytes(String name, byte[] bytes) {
-            try {
-                return this.loadClass(name);
-            } catch (ClassNotFoundException cnfe) {
-                // expected
+
+            if (s4AppLoader instanceof S4RLoader) {
+                ((S4RLoader)s4AppLoader).addGeneratedClassBytes(name, bytes);
+                try {
+                    return s4AppLoader.loadClass(name);
+                } catch (ClassNotFoundException e) {
+                    // TODO throw a runtime exception?
+                    e.printStackTrace();
+                    return null;
+                }
+            } else {
+                try {
+                    return this.loadClass(name);
+                } catch (ClassNotFoundException cnfe) {
+                    // expected
+                }
+                return this.defineClass(name, bytes, 0, bytes.length);
             }
-            return this.defineClass(name, bytes, 0, bytes.length);
         }
+
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/29b00ef2/subprojects/s4-core/src/test/java/test/s4/core/apploading/AppLoadingTest.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-core/src/test/java/test/s4/core/apploading/AppLoadingTest.java
b/subprojects/s4-core/src/test/java/test/s4/core/apploading/AppLoadingTest.java
new file mode 100644
index 0000000..d0cf3a8
--- /dev/null
+++ b/subprojects/s4-core/src/test/java/test/s4/core/apploading/AppLoadingTest.java
@@ -0,0 +1,164 @@
+package test.s4.core.apploading;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.jar.JarEntry;
+import java.util.jar.JarOutputStream;
+import java.util.jar.Manifest;
+
+import org.apache.s4.core.Server;
+import org.apache.zookeeper.ZooKeeper;
+import org.apache.zookeeper.server.NIOServerCnxn.Factory;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import test.s4.fixtures.TestUtils;
+
+import com.google.common.io.ByteStreams;
+import com.google.common.io.Files;
+
+/**
+ * 
+ * Tests packaging and deployment of an S4 app
+ * 
+ */
+public class AppLoadingTest {
+
+    public static long ZOOKEEPER_PORT = 21810;
+    private static Factory zookeeperServerConnectionFactory = null;
+    private Process forkedApp;
+
+    @Before
+    public void prepare() throws Exception {
+        TestUtils.cleanupTmpDirs();
+        zookeeperServerConnectionFactory = TestUtils.startZookeeperServer();
+        final ZooKeeper zk = TestUtils.createZkClient();
+        try {
+            zk.delete("/simpleAppCreated", -1);
+        } catch (Exception ignored) {
+        }
+
+        zk.close();
+    }
+
+    @After
+    public void cleanup() throws Exception {
+        TestUtils.stopZookeeperServer(zookeeperServerConnectionFactory);
+        TestUtils.killS4App(forkedApp);
+    }
+
+    @Ignore("fix paths")
+    @Test
+    public void testA() throws Exception, InterruptedException {
+
+        // add all classes from counter app
+        File rootAppDir = new File ( new File(System.getProperty("user.dir")).getParentFile().getAbsolutePath()+"/s4-example/bin");
+        File appFilesDir = new File(rootAppDir, "org/apache/s4/example/counter");
+        generateS4RFromDirectoryContents(rootAppDir, appFilesDir, "counterExample",
+                "org.apache.s4.example.counter.MyApp");
+
+        forkedApp = TestUtils.forkS4Node();
+        Thread.sleep(15000);
+    }
+
+    private void generateS4RFromDirectoryContents(File rootAppDir, File appFilesDir, String
s4rName, String appClassName)
+            throws IOException, FileNotFoundException {
+        Collection<File> s4rFiles = listFilesRecursively(appFilesDir);
+        File jarFile = new File(System.getProperty("user.dir") + "/bin/apps/" + s4rName +
".s4r");
+        Files.createParentDirs(jarFile);
+        FileOutputStream fos = new FileOutputStream(jarFile);
+        JarOutputStream jos = new JarOutputStream(fos);
+        System.out.println(System.getProperty("java.class.path"));
+        for (File file : s4rFiles) {
+            JarEntry jarEntry = new JarEntry(file.getAbsolutePath().substring(rootAppDir.getAbsolutePath().length()));
+            jos.putNextEntry(jarEntry);
+            ByteStreams.copy(Files.newInputStreamSupplier(file), jos);
+        }
+        // add manifest
+        File manifest = File.createTempFile("s4app", "manifest");
+        String manifestContents = "Manifest-Version: 1.0\n" + Server.MANIFEST_S4_APP_CLASS
+ ": " + appClassName + "\n";
+        Files.write(manifestContents, manifest, Charset.forName("UTF-8"));
+        JarEntry jarEntry = new JarEntry("META-INF/MANIFEST.MF");
+        jos.putNextEntry(jarEntry);
+        ByteStreams.copy(Files.newInputStreamSupplier(manifest), jos);
+
+        jos.close();
+    }
+
+    private Collection<File> listFilesRecursively(File dir) {
+        if (dir.isDirectory()) {
+            File[] listFiles = dir.listFiles();
+            List<File> filesToAdd = new ArrayList<File>();
+            if (listFiles.length != 0) {
+                for (File file : listFiles) {
+                    if (file.isFile()) {
+                        filesToAdd.add(file);
+                    } else if (file.isDirectory()) {
+                        filesToAdd.addAll(listFilesRecursively(file));
+                    }
+                }
+            }
+            return filesToAdd;
+        } else {
+            // TODO throw exception
+            return null;
+        }
+    }
+
+    /**
+     * 
+     * 1. generates an s4r package from classes in the apploading package (TODO process still
to be improved), 
+     * 2. deploys it to bin/apps 
+     * 3. starts a forked S4 node, which loads apps from bin/apps
+     * 4. verifies app is working (s4 app started, event correctly processed)
+     * 
+     * NOTE: we'll need to add an automatic test for which we make sure code cannot be in
the classpath
+     */
+    @Test
+    public void testAppLoading() throws Exception {
+        
+        // TODO fix paths
+
+        final ZooKeeper zk = TestUtils.createZkClient();
+
+        File rootAppDir = new File(System.getProperty("user.dir") + "/build/classes/test/");
+        File appFilesDir = new File(rootAppDir, "test/s4/core/apploading");
+        // 1. create app jar and place it in tmp/s4-apps
+        generateS4RFromDirectoryContents(rootAppDir, appFilesDir, "appLoadingTest", SimpleApp.class.getName());
+
+        CountDownLatch signalAppStarted = new CountDownLatch(1);
+        // 2. start s4 node and check results
+        forkedApp = TestUtils.forkS4Node();
+
+        // TODO wait for ready state (zk node available)
+        Thread.sleep(5000);
+
+        // note: make sure we don't delete existing znode if it was already created
+        TestUtils.watchAndSignalCreation("/simpleAppCreated", signalAppStarted, zk, false);
+
+        Assert.assertTrue(signalAppStarted.await(20, TimeUnit.SECONDS));
+
+        String time1 = String.valueOf(System.currentTimeMillis());
+
+        CountDownLatch signalEvent1Processed = new CountDownLatch(1);
+        TestUtils.watchAndSignalCreation("/onEvent@" + time1, signalEvent1Processed, zk);
+
+        TestUtils.injectIntoStringSocketAdapter(time1);
+
+        // check event processed
+        Assert.assertTrue(signalEvent1Processed.await(5, TimeUnit.SECONDS));
+
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/29b00ef2/subprojects/s4-core/src/test/java/test/s4/core/apploading/SimpleApp.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-core/src/test/java/test/s4/core/apploading/SimpleApp.java b/subprojects/s4-core/src/test/java/test/s4/core/apploading/SimpleApp.java
new file mode 100644
index 0000000..c49d6d3
--- /dev/null
+++ b/subprojects/s4-core/src/test/java/test/s4/core/apploading/SimpleApp.java
@@ -0,0 +1,47 @@
+package test.s4.core.apploading;
+
+import java.io.IOException;
+
+import org.apache.s4.core.App;
+import org.apache.s4.core.Stream;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.ZooDefs.Ids;
+import org.apache.zookeeper.ZooKeeper;
+
+import test.s4.fixtures.SocketAdapter;
+import test.s4.fixtures.TestUtils;
+import test.s4.wordcount.SentenceKeyFinder;
+import test.s4.wordcount.StringEvent;
+
+public class SimpleApp extends App {
+    private SocketAdapter<StringEvent> socketAdapter;
+    
+    public SimpleApp () {}
+
+    @Override
+    protected void start() {
+        try {
+            final ZooKeeper zk = TestUtils.createZkClient();
+            zk.create("/simpleAppCreated", new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+            zk.close();
+        } catch (Exception e) {
+            System.exit(-1);
+        }
+    }
+
+    @Override
+    protected void init() {
+        SimplePE prototype = createPE(SimplePE.class);
+        Stream<StringEvent> stream = createStream("stream", new SentenceKeyFinder(),
prototype);
+        try {
+            socketAdapter = new SocketAdapter<StringEvent>(stream, new SocketAdapter.SentenceEventFactory());
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    protected void close() {
+        // TODO Auto-generated method stub
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/29b00ef2/subprojects/s4-core/src/test/java/test/s4/core/apploading/SimpleModule.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-core/src/test/java/test/s4/core/apploading/SimpleModule.java b/subprojects/s4-core/src/test/java/test/s4/core/apploading/SimpleModule.java
new file mode 100644
index 0000000..cfd6862
--- /dev/null
+++ b/subprojects/s4-core/src/test/java/test/s4/core/apploading/SimpleModule.java
@@ -0,0 +1,7 @@
+package test.s4.core.apploading;
+
+import test.s4.fixtures.GenericTestModule;
+
+public class SimpleModule extends GenericTestModule<SimpleApp> {
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/29b00ef2/subprojects/s4-core/src/test/java/test/s4/core/apploading/SimplePE.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-core/src/test/java/test/s4/core/apploading/SimplePE.java b/subprojects/s4-core/src/test/java/test/s4/core/apploading/SimplePE.java
new file mode 100644
index 0000000..3325dc0
--- /dev/null
+++ b/subprojects/s4-core/src/test/java/test/s4/core/apploading/SimplePE.java
@@ -0,0 +1,66 @@
+package test.s4.core.apploading;
+
+import java.io.IOException;
+
+import org.apache.s4.core.App;
+import org.apache.s4.core.ProcessingElement;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.ZooDefs.Ids;
+import org.apache.zookeeper.ZooKeeper;
+import org.slf4j.LoggerFactory;
+
+import test.s4.fixtures.TestUtils;
+import test.s4.wordcount.StringEvent;
+
+public class SimplePE  extends ProcessingElement implements Watcher {
+
+    private ZooKeeper zk;
+
+    public SimplePE() {}
+    
+    public SimplePE(App app) {
+        super(app);
+    }
+    
+    public void onEvent(StringEvent event) {
+        try {
+            LoggerFactory.getLogger(getClass()).debug("processing envent {}", event.getString());
+            zk.create("/onEvent@"+event.getString(), new byte[0], Ids.OPEN_ACL_UNSAFE,
+                    CreateMode.PERSISTENT);
+            zk.close();
+        } catch (KeeperException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        } catch (InterruptedException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        }
+    }
+
+    @Override
+    protected void onCreate() {
+        if (zk == null) {
+            try {
+                zk = new ZooKeeper("localhost:" + TestUtils.ZK_PORT, 4000, this);
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        }
+        
+    }
+
+    @Override
+    protected void onRemove() {
+        // TODO Auto-generated method stub
+        
+    }
+
+    @Override
+    public void process(WatchedEvent event) {
+        // TODO Auto-generated method stub
+        
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/29b00ef2/subprojects/s4-core/src/test/java/test/s4/fixtures/TestUtils.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-core/src/test/java/test/s4/fixtures/TestUtils.java b/subprojects/s4-core/src/test/java/test/s4/fixtures/TestUtils.java
index 7efec95..f01b587 100644
--- a/subprojects/s4-core/src/test/java/test/s4/fixtures/TestUtils.java
+++ b/subprojects/s4-core/src/test/java/test/s4/fixtures/TestUtils.java
@@ -1,6 +1,5 @@
 package test.s4.fixtures;
 
-
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileInputStream;
@@ -14,12 +13,14 @@ import java.net.InetSocketAddress;
 import java.net.ServerSocket;
 import java.net.Socket;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.concurrent.CountDownLatch;
 
 import junit.framework.Assert;
 
 import org.apache.s4.core.App;
+import org.apache.s4.core.Main;
 import org.apache.s4.core.ProcessingElement;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.WatchedEvent;
@@ -51,23 +52,31 @@ public class TestUtils {
     static {
         logger.info("Storage dir: " + DEFAULT_STORAGE_DIR);
     }
+
     public static Process forkS4App(Class<?> moduleClass, Class<?> appClass)
throws IOException, InterruptedException {
+        return forkProcess(App.class.getName(), moduleClass.getName(), appClass.getName());
+    }
 
+    public static Process forkS4Node() throws IOException, InterruptedException {
+        return forkProcess(Main.class.getName(), new String[] {});
+    }
+
+    private static Process forkProcess(String mainClass, String... args) throws IOException,
InterruptedException {
         List<String> cmdList = new ArrayList<String>();
         cmdList.add("java");
         cmdList.add("-cp");
         cmdList.add(System.getProperty("java.class.path"));
-        // cmdList.add("-Xdebug");
-        // cmdList.add("-Xnoagent");
-        //
-        // cmdList.add("-Xrunjdwp:transport=dt_socket,address=8788,server=y,suspend=n");
-        cmdList.add(App.class.getName());
-        cmdList.add(moduleClass.getName());
-        cmdList.add(appClass.getName());
-
-        // System.out.println(Arrays.toString(cmdList.toArray(new
-        // String[]{})).replace(",", ""));
-
+//      cmdList.add("-Xdebug");
+//      cmdList.add("-Xnoagent");
+//     
+//      cmdList.add("-Xrunjdwp:transport=dt_socket,address=8788,server=y,suspend=n");
+
+        cmdList.add(mainClass);
+        for (String arg : args) {
+            cmdList.add(arg);
+        }
+        
+        System.out.println(Arrays.toString(cmdList.toArray(new String[] {})).replace(",",
""));
         ProcessBuilder pb = new ProcessBuilder(cmdList);
 
         pb.directory(new File(System.getProperty("user.dir")));
@@ -166,7 +175,7 @@ public class TestUtils {
             KeeperException {
 
         List<String> cmdList = new ArrayList<String>();
-        final File zkDataDir = new File(System.getProperty("user.dir") + File.separator +
"tmp" + File.separator
+        final File zkDataDir = new File(System.getProperty("java.io.tmpdir") + File.separator
+ "tmp" + File.separator
                 + "zookeeper" + File.separator + "data");
         if (zkDataDir.exists()) {
             TestUtils.deleteDirectoryContents(zkDataDir);
@@ -269,8 +278,20 @@ public class TestUtils {
     public static void watchAndSignalCreation(String path, final CountDownLatch latch, final
ZooKeeper zk)
             throws KeeperException, InterruptedException {
 
+        // by default delete existing nodes with same path
+        watchAndSignalCreation(path, latch, zk, false);
+    }
+
+    
+    public static void watchAndSignalCreation(String path, final CountDownLatch latch, final
ZooKeeper zk, boolean deleteIfExists)
+            throws KeeperException, InterruptedException {
+
         if (zk.exists(path, false) != null) {
-            zk.delete(path, -1);
+            if (deleteIfExists) {
+                zk.delete(path, -1);
+            } else {
+                latch.countDown();
+            }
         }
         zk.exists(path, new Watcher() {
             @Override


Mime
View raw message