avro-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From scottca...@apache.org
Subject svn commit: r1059139 - in /avro/trunk: ./ lang/java/mapred/src/main/java/org/apache/avro/mapred/tether/ lang/java/tools/ lang/java/tools/src/main/java/org/apache/avro/tool/ lang/java/tools/src/test/java/org/apache/avro/tool/
Date Fri, 14 Jan 2011 19:41:04 GMT
Author: scottcarey
Date: Fri Jan 14 19:41:03 2011
New Revision: 1059139

URL: http://svn.apache.org/viewvc?rev=1059139&view=rev
Log:
AVRO-663 Java build: Change build to create an avro-tools.jar that conforms to maven standards
and contains dependencies within

Added:
    avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/TetherTool.java
Modified:
    avro/trunk/CHANGES.txt
    avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/tether/TetherJob.java
    avro/trunk/lang/java/tools/pom.xml
    avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/Main.java
    avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestDataFileTools.java

Modified: avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1059139&r1=1059138&r2=1059139&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Fri Jan 14 19:41:03 2011
@@ -26,7 +26,8 @@ Avro 1.5.0 (unreleased)
       API for Hadoop MapReduce with Avro data.
       Dependencies: avro-ipc.jar, hadoop-core, and jopt-simple.
     * avro-tools.jar
-      Avro command-line tools.
+      Avro command-line tools stand-alone jar.
+      Dependencies are contained within the jar.
       Dependencies: all of the above.
 
     (scottcarey)

Modified: avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/tether/TetherJob.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/tether/TetherJob.java?rev=1059139&r1=1059138&r2=1059139&view=diff
==============================================================================
--- avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/tether/TetherJob.java
(original)
+++ avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/tether/TetherJob.java
Fri Jan 14 19:41:03 2011
@@ -19,10 +19,6 @@
 package org.apache.avro.mapred.tether;
 
 import java.io.IOException;
-import java.io.InputStream;
-import java.io.PrintStream;
-import java.io.File;
-import java.util.List;
 import java.util.Collection;
 import java.net.URI;
 import java.net.URISyntaxException;
@@ -30,24 +26,14 @@ import java.net.URISyntaxException;
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.io.NullWritable;
 import org.apache.hadoop.filecache.DistributedCache;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.FileInputFormat;
-import org.apache.hadoop.mapred.FileOutputFormat;
 import org.apache.hadoop.mapred.JobClient;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.RunningJob;
 
-import joptsimple.OptionParser;
-import joptsimple.OptionSet;
-import joptsimple.OptionSpec;
-
-import org.apache.avro.Schema;
-import org.apache.avro.tool.Tool;
-import org.apache.avro.mapred.AvroJob;
-
-/** Constructs and submits tether jobs. This may either be used as a
- * commandline-based or API-based method to launch tether jobs. */
-public class TetherJob extends Configured implements Tool {
+/** Constructs and submits tether jobs. This may be used as an API-based
+ *  method to launch tether jobs. */
+@SuppressWarnings("deprecation")
+public class TetherJob extends Configured {
 
   /** Get the URI of the application's executable. */
   public static URI getExecutable(JobConf job) {
@@ -102,55 +88,4 @@ public class TetherJob extends Configure
     DistributedCache.addCacheFile(getExecutable(job), job);
   }
 
-  // Tool methods
-
-  @Override
-  public String getName() { return "tether"; }
-
-  @Override
-  public String getShortDescription() {return "Run a tethered mapreduce job.";}
-
-  @Override
-  public int run(InputStream ins, PrintStream outs, PrintStream err,
-                 List<String> args) throws Exception {
-
-    OptionParser p = new OptionParser();
-    OptionSpec<URI> exec =
-      p.accepts("program", "executable program, usually in HDFS")
-      .withRequiredArg().ofType(URI.class);
-    OptionSpec<String> in = p.accepts("in", "comma-separated input paths")
-      .withRequiredArg().ofType(String.class);
-    OptionSpec<Path> out = p.accepts("out", "output directory")
-      .withRequiredArg().ofType(Path.class);
-    OptionSpec<File> outSchema = p.accepts("outschema", "output schema file")
-      .withRequiredArg().ofType(File.class);
-    OptionSpec<File> mapOutSchema =
-      p.accepts("outschemamap", "map output schema file, if different")
-      .withOptionalArg().ofType(File.class);
-    OptionSpec<Integer> reduces = p.accepts("reduces", "number of reduces")
-      .withOptionalArg().ofType(Integer.class);
-
-    JobConf job = new JobConf();
-      
-    try {
-      OptionSet opts = p.parse(args.toArray(new String[0]));
-      FileInputFormat.addInputPaths(job, in.value(opts));
-      FileOutputFormat.setOutputPath(job, out.value(opts));
-      TetherJob.setExecutable(job, exec.value(opts));
-      job.set(AvroJob.OUTPUT_SCHEMA,
-              Schema.parse(outSchema.value(opts)).toString());
-      if (opts.hasArgument(mapOutSchema))
-        job.set(AvroJob.MAP_OUTPUT_SCHEMA,
-                Schema.parse(mapOutSchema.value(opts)).toString());
-      if (opts.hasArgument(reduces))
-        job.setNumReduceTasks(reduces.value(opts));
-    } catch (Exception e) {
-      p.printHelpOn(err);
-      return -1;
-    }
-
-    runJob(job);
-    return 0;
-  }
-
 }

Modified: avro/trunk/lang/java/tools/pom.xml
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/tools/pom.xml?rev=1059139&r1=1059138&r2=1059139&view=diff
==============================================================================
--- avro/trunk/lang/java/tools/pom.xml (original)
+++ avro/trunk/lang/java/tools/pom.xml Fri Jan 14 19:41:03 2011
@@ -31,17 +31,40 @@
   <description>Avro command line tools and utilities</description>
 
   <build>
+    <resources>
+      <resource>
+        <includes>
+          <include>VERSION.txt</include>
+        </includes>
+        <directory>../../../share/</directory>
+      </resource>
+      <resource>
+        <includes>
+          <include>NOTICE.txt</include>
+        </includes>
+        <directory>../../../</directory>
+      </resource>
+    </resources>
     <plugins>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-jar-plugin</artifactId>
-        <configuration>
-          <archive>
-            <manifest>
-              <mainClass>org.apache.avro.tool.Main</mainClass>
-            </manifest>
-          </archive>
-        </configuration>
+        <artifactId>maven-shade-plugin</artifactId>
+        <version>1.4</version>
+        <executions>
+          <execution>
+            <goals>
+              <goal>shade</goal>
+            </goals>
+            <configuration>
+              <transformers>
+                <transformer
+                  implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                  <mainClass>org.apache.avro.tool.Main</mainClass>
+                </transformer>
+              </transformers>
+            </configuration>
+          </execution>
+        </executions>
       </plugin>
     </plugins>
   </build>
@@ -70,6 +93,81 @@
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-core</artifactId>
+      <optional>true</optional>
+      <exclusions>
+        <exclusion>
+          <groupId>org.mortbay.jetty</groupId>
+          <artifactId>jetty</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.mortbay.jetty</groupId>
+          <artifactId>jetty-util</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.mortbay.jetty</groupId>
+          <artifactId>servlet-api</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.mortbay.jetty</groupId>
+          <artifactId>jsp-api-2.1</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.mortbay.jetty</groupId>
+          <artifactId>jsp-2.1</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>tomcat</groupId>
+          <artifactId>jasper-compiler</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>tomcat</groupId>
+          <artifactId>jasper-runtime</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>net.java.dev.jets3t</groupId>
+          <artifactId>jets3t</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>commons-net</groupId>
+          <artifactId>commons-net</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>commons-httpclient</groupId>
+          <artifactId>commons-httpclient</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>commons-el</groupId>
+          <artifactId>commons-el</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>xmlenc</groupId>
+          <artifactId>xmlenc</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.mortbay.jetty</groupId>
+          <artifactId>servlet-api-2.5</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.eclipse.jdt</groupId>
+          <artifactId>core</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>commons-cli</groupId>
+          <artifactId>commons-cli</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>commons-codec</groupId>
+          <artifactId>commons-codec</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>hsqldb</groupId>
+          <artifactId>hsqldb</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>oro</groupId>
+          <artifactId>oro</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>net.sf.jopt-simple</groupId>

Modified: avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/Main.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/Main.java?rev=1059139&r1=1059138&r2=1059139&view=diff
==============================================================================
--- avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/Main.java (original)
+++ avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/Main.java Fri Jan 14 19:41:03
2011
@@ -23,7 +23,6 @@ import java.util.TreeMap;
 
 import java.io.InputStream;
 
-import org.apache.avro.mapred.tether.TetherJob;
 import org.apache.avro.reflect.InduceSchemaTool;
 import org.apache.avro.specific.SpecificCompiler.SpecificCompilerTool;
 
@@ -52,7 +51,7 @@ public class Main {
         new RpcSendTool(),
         new FromTextTool(),
         new ToTextTool(),
-        new TetherJob()
+        new TetherTool()
         }) {
       Tool prev = tools.put(tool.getName(), tool);
       if (prev != null) {

Added: avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/TetherTool.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/TetherTool.java?rev=1059139&view=auto
==============================================================================
--- avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/TetherTool.java (added)
+++ avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/TetherTool.java Fri Jan
14 19:41:03 2011
@@ -0,0 +1,78 @@
+package org.apache.avro.tool;
+
+import java.io.File;
+import java.io.InputStream;
+import java.io.PrintStream;
+import java.net.URI;
+import java.util.List;
+
+import joptsimple.OptionParser;
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+
+import org.apache.avro.Schema;
+import org.apache.avro.mapred.AvroJob;
+import org.apache.avro.mapred.tether.TetherJob;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.FileOutputFormat;
+import org.apache.hadoop.mapred.JobConf;
+
+@SuppressWarnings("deprecation")
+public class TetherTool implements Tool {
+  public TetherJob job;
+
+  @Override
+  public String getName() {
+    return "tether";
+  }
+
+  @Override
+  public String getShortDescription() {
+    return "Run a tethered mapreduce job.";
+  }
+
+  @Override
+  public int run(InputStream ins, PrintStream outs, PrintStream err,
+      List<String> args) throws Exception {
+
+    OptionParser p = new OptionParser();
+    OptionSpec<URI> exec = p
+        .accepts("program", "executable program, usually in HDFS")
+        .withRequiredArg().ofType(URI.class);
+    OptionSpec<String> in = p.accepts("in", "comma-separated input paths")
+        .withRequiredArg().ofType(String.class);
+    OptionSpec<Path> out = p.accepts("out", "output directory")
+        .withRequiredArg().ofType(Path.class);
+    OptionSpec<File> outSchema = p.accepts("outschema", "output schema file")
+        .withRequiredArg().ofType(File.class);
+    OptionSpec<File> mapOutSchema = p
+        .accepts("outschemamap", "map output schema file, if different")
+        .withOptionalArg().ofType(File.class);
+    OptionSpec<Integer> reduces = p.accepts("reduces", "number of reduces")
+        .withOptionalArg().ofType(Integer.class);
+
+    JobConf job = new JobConf();
+
+    try {
+      OptionSet opts = p.parse(args.toArray(new String[0]));
+      FileInputFormat.addInputPaths(job, in.value(opts));
+      FileOutputFormat.setOutputPath(job, out.value(opts));
+      TetherJob.setExecutable(job, exec.value(opts));
+      job.set(AvroJob.OUTPUT_SCHEMA, Schema.parse(outSchema.value(opts))
+          .toString());
+      if (opts.hasArgument(mapOutSchema))
+        job.set(AvroJob.MAP_OUTPUT_SCHEMA,
+            Schema.parse(mapOutSchema.value(opts)).toString());
+      if (opts.hasArgument(reduces))
+        job.setNumReduceTasks(reduces.value(opts));
+    } catch (Exception e) {
+      p.printHelpOn(err);
+      return -1;
+    }
+
+    TetherJob.runJob(job);
+    return 0;
+  }
+
+}

Modified: avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestDataFileTools.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestDataFileTools.java?rev=1059139&r1=1059138&r2=1059139&view=diff
==============================================================================
--- avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestDataFileTools.java (original)
+++ avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestDataFileTools.java Fri
Jan 14 19:41:03 2011
@@ -23,9 +23,10 @@ import static org.junit.Assert.fail;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileOutputStream;
+import java.io.FileWriter;
 import java.io.IOException;
 import java.io.PrintStream;
-import java.io.FileWriter;
+import java.io.StringBufferInputStream;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -38,10 +39,10 @@ import org.apache.avro.file.DataFileRead
 import org.apache.avro.file.DataFileWriter;
 import org.apache.avro.generic.GenericDatumReader;
 import org.apache.avro.generic.GenericDatumWriter;
-import org.apache.tools.ant.filters.StringInputStream;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
+@SuppressWarnings("deprecation")
 public class TestDataFileTools {
   static final int COUNT = 10;
   static File sampleFile;
@@ -130,7 +131,7 @@ public class TestDataFileTools {
     args.add("-");
     args.addAll(extra);
     new DataFileWriteTool().run(
-        new StringInputStream(jsonData),
+        new StringBufferInputStream(jsonData),
         new PrintStream(out), // stdout
         null, // stderr
         args);
@@ -160,7 +161,7 @@ public class TestDataFileTools {
     PrintStream out = new PrintStream(baos);
     try {
       new DataFileWriteTool().run(
-          new StringInputStream("{"),
+          new StringBufferInputStream("{"),
           new PrintStream(out), // stdout
           null, // stderr          
           Arrays.asList("-schema", "{ \"type\":\"record\", \"fields\":" +
@@ -206,7 +207,7 @@ public class TestDataFileTools {
     FileOutputStream fout = new FileOutputStream(outFile);
     PrintStream out = new PrintStream(fout);
     new DataFileWriteTool().run(
-        new StringInputStream(json),
+        new StringBufferInputStream(json),
         new PrintStream(out), // stdout
         null, // stderr
         Arrays.asList("-schema", schema, "-"));



Mime
View raw message