Author: cutting
Date: Wed Apr 6 18:00:36 2011
New Revision: 1089551
URL: http://svn.apache.org/viewvc?rev=1089551&view=rev
Log:
Merge -c 1089550 from trunk to 1.5 branch. Fixes: AVRO-788.
Added:
avro/branches/branch-1.5/lang/java/avro/src/main/java/org/apache/avro/file/SnappyCodec.java
- copied unchanged from r1089550, avro/trunk/lang/java/avro/src/main/java/org/apache/avro/file/SnappyCodec.java
Modified:
avro/branches/branch-1.5/ (props changed)
avro/branches/branch-1.5/CHANGES.txt
avro/branches/branch-1.5/doc/src/content/xdocs/spec.xml
avro/branches/branch-1.5/lang/java/avro/pom.xml
avro/branches/branch-1.5/lang/java/avro/src/main/java/org/apache/avro/file/CodecFactory.java
avro/branches/branch-1.5/lang/java/avro/src/main/java/org/apache/avro/file/DataFileConstants.java
avro/branches/branch-1.5/lang/java/avro/src/test/java/org/apache/avro/TestDataFile.java
avro/branches/branch-1.5/lang/java/mapred/src/main/java/org/apache/avro/mapred/AvroJob.java
avro/branches/branch-1.5/lang/java/mapred/src/main/java/org/apache/avro/mapred/AvroOutputFormat.java
avro/branches/branch-1.5/lang/java/mapred/src/test/java/org/apache/avro/mapred/TestWeather.java
avro/branches/branch-1.5/lang/java/pom.xml
Propchange: avro/branches/branch-1.5/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Apr 6 18:00:36 2011
@@ -1 +1 @@
-/avro/trunk:1075938,1075993,1078917,1079055,1079060,1079063,1083246,1085921,1086727,1086730,1086866,1087076,1087129,1087136,1087439-1087440,1087463,1087472,1087792,1089128,1089131
+/avro/trunk:1075938,1075993,1078917,1079055,1079060,1079063,1083246,1085921,1086727,1086730,1086866,1087076,1087129,1087136,1087439-1087440,1087463,1087472,1087792,1089128,1089131,1089550
Modified: avro/branches/branch-1.5/CHANGES.txt
URL: http://svn.apache.org/viewvc/avro/branches/branch-1.5/CHANGES.txt?rev=1089551&r1=1089550&r2=1089551&view=diff
==============================================================================
--- avro/branches/branch-1.5/CHANGES.txt (original)
+++ avro/branches/branch-1.5/CHANGES.txt Wed Apr 6 18:00:36 2011
@@ -7,6 +7,9 @@ Avro 1.5.1 (unreleased)
AVRO-533. Add a C# implementation.
(Jeremy Custenborder, Dona Alvarez and thiru)
+ AVRO-788. Java: Add Snappy compression for data files, including
+ MapReduce API support. (cutting)
+
IMPROVEMENTS
AVRO-785. Java: Squash a Velocity warning by upgrading to Velocity 1.7.
Modified: avro/branches/branch-1.5/doc/src/content/xdocs/spec.xml
URL: http://svn.apache.org/viewvc/avro/branches/branch-1.5/doc/src/content/xdocs/spec.xml?rev=1089551&r1=1089550&r2=1089551&view=diff
==============================================================================
--- avro/branches/branch-1.5/doc/src/content/xdocs/spec.xml (original)
+++ avro/branches/branch-1.5/doc/src/content/xdocs/spec.xml Wed Apr 6 18:00:36 2011
@@ -695,7 +695,15 @@
</p>
</section>
</section>
-
+ <section>
+ <title>Optional Codecs</title>
+ <section>
+ <title>snappy</title>
+ <p>The "snappy" codec uses
+ Google's <a href="http://code.google.com/p/snappy/">Snappy</a>
+ compression library.</p>
+ </section>
+ </section>
</section>
<section>
Modified: avro/branches/branch-1.5/lang/java/avro/pom.xml
URL: http://svn.apache.org/viewvc/avro/branches/branch-1.5/lang/java/avro/pom.xml?rev=1089551&r1=1089550&r2=1089551&view=diff
==============================================================================
--- avro/branches/branch-1.5/lang/java/avro/pom.xml (original)
+++ avro/branches/branch-1.5/lang/java/avro/pom.xml Wed Apr 6 18:00:36 2011
@@ -39,6 +39,10 @@
<groupId>com.thoughtworks.paranamer</groupId>
<artifactId>paranamer</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ </dependency>
</dependencies>
<build>
Modified: avro/branches/branch-1.5/lang/java/avro/src/main/java/org/apache/avro/file/CodecFactory.java
URL: http://svn.apache.org/viewvc/avro/branches/branch-1.5/lang/java/avro/src/main/java/org/apache/avro/file/CodecFactory.java?rev=1089551&r1=1089550&r2=1089551&view=diff
==============================================================================
--- avro/branches/branch-1.5/lang/java/avro/src/main/java/org/apache/avro/file/CodecFactory.java
(original)
+++ avro/branches/branch-1.5/lang/java/avro/src/main/java/org/apache/avro/file/CodecFactory.java
Wed Apr 6 18:00:36 2011
@@ -36,6 +36,11 @@ public abstract class CodecFactory {
return new DeflateCodec.Option(compressionLevel);
};
+ /** Snappy codec.*/
+ public static CodecFactory snappyCodec() {
+ return new SnappyCodec.Option();
+ };
+
/** Creates internal Codec. */
protected abstract Codec createInstance();
@@ -50,6 +55,7 @@ public abstract class CodecFactory {
static {
addCodec("null", nullCodec());
addCodec("deflate", deflateCodec(DEFAULT_DEFLATE_LEVEL));
+ addCodec("snappy", snappyCodec());
}
/** Maps a codec name into a CodecOption. */
Modified: avro/branches/branch-1.5/lang/java/avro/src/main/java/org/apache/avro/file/DataFileConstants.java
URL: http://svn.apache.org/viewvc/avro/branches/branch-1.5/lang/java/avro/src/main/java/org/apache/avro/file/DataFileConstants.java?rev=1089551&r1=1089550&r2=1089551&view=diff
==============================================================================
--- avro/branches/branch-1.5/lang/java/avro/src/main/java/org/apache/avro/file/DataFileConstants.java
(original)
+++ avro/branches/branch-1.5/lang/java/avro/src/main/java/org/apache/avro/file/DataFileConstants.java
Wed Apr 6 18:00:36 2011
@@ -36,5 +36,6 @@ public class DataFileConstants {
public static final String CODEC = "avro.codec";
public static final String NULL_CODEC = "null";
public static final String DEFLATE_CODEC = "deflate";
+ public static final String SNAPPY_CODEC = "snappy";
}
Modified: avro/branches/branch-1.5/lang/java/avro/src/test/java/org/apache/avro/TestDataFile.java
URL: http://svn.apache.org/viewvc/avro/branches/branch-1.5/lang/java/avro/src/test/java/org/apache/avro/TestDataFile.java?rev=1089551&r1=1089550&r2=1089551&view=diff
==============================================================================
--- avro/branches/branch-1.5/lang/java/avro/src/test/java/org/apache/avro/TestDataFile.java
(original)
+++ avro/branches/branch-1.5/lang/java/avro/src/test/java/org/apache/avro/TestDataFile.java
Wed Apr 6 18:00:36 2011
@@ -61,6 +61,7 @@ public class TestDataFile {
r.add(new Object[] { CodecFactory.deflateCodec(1) });
r.add(new Object[] { CodecFactory.deflateCodec(9) });
r.add(new Object[] { CodecFactory.nullCodec() });
+ r.add(new Object[] { CodecFactory.snappyCodec() });
return r;
}
Modified: avro/branches/branch-1.5/lang/java/mapred/src/main/java/org/apache/avro/mapred/AvroJob.java
URL: http://svn.apache.org/viewvc/avro/branches/branch-1.5/lang/java/mapred/src/main/java/org/apache/avro/mapred/AvroJob.java?rev=1089551&r1=1089550&r2=1089551&view=diff
==============================================================================
--- avro/branches/branch-1.5/lang/java/mapred/src/main/java/org/apache/avro/mapred/AvroJob.java
(original)
+++ avro/branches/branch-1.5/lang/java/mapred/src/main/java/org/apache/avro/mapred/AvroJob.java
Wed Apr 6 18:00:36 2011
@@ -42,6 +42,8 @@ public class AvroJob {
public static final String MAP_OUTPUT_SCHEMA = "avro.map.output.schema";
/** The configuration key for a job's output schema. */
public static final String OUTPUT_SCHEMA = "avro.output.schema";
+ /** The configuration key for a job's output compression codec. */
+ public static final String OUTPUT_CODEC = "avro.output.codec";
/** The configuration key prefix for a text output metadata. */
public static final String TEXT_PREFIX = "avro.meta.text.";
/** The configuration key prefix for a binary output metadata. */
@@ -82,6 +84,11 @@ public class AvroJob {
configureAvroOutput(job);
}
+ /** Configure a job's output compression codec. */
+ public static void setOutputCodec(JobConf job, String codec) {
+ job.set(OUTPUT_CODEC, codec);
+ }
+
/** Add metadata to job output files.*/
public static void setOutputMeta(JobConf job, String key, String value) {
job.set(TEXT_PREFIX+key, value);
Modified: avro/branches/branch-1.5/lang/java/mapred/src/main/java/org/apache/avro/mapred/AvroOutputFormat.java
URL: http://svn.apache.org/viewvc/avro/branches/branch-1.5/lang/java/mapred/src/main/java/org/apache/avro/mapred/AvroOutputFormat.java?rev=1089551&r1=1089550&r2=1089551&view=diff
==============================================================================
--- avro/branches/branch-1.5/lang/java/mapred/src/main/java/org/apache/avro/mapred/AvroOutputFormat.java
(original)
+++ avro/branches/branch-1.5/lang/java/mapred/src/main/java/org/apache/avro/mapred/AvroOutputFormat.java
Wed Apr 6 18:00:36 2011
@@ -36,6 +36,7 @@ import org.apache.avro.reflect.ReflectDa
import org.apache.avro.file.DataFileWriter;
import org.apache.avro.file.CodecFactory;
import static org.apache.avro.file.DataFileConstants.DEFAULT_SYNC_INTERVAL;
+import static org.apache.avro.file.DataFileConstants.DEFLATE_CODEC;
/** An {@link org.apache.hadoop.mapred.OutputFormat} for Avro data files. */
public class AvroOutputFormat <T>
@@ -80,7 +81,11 @@ public class AvroOutputFormat <T>
if (FileOutputFormat.getCompressOutput(job)) {
int level = job.getInt(DEFLATE_LEVEL_KEY, DEFAULT_DEFLATE_LEVEL);
- writer.setCodec(CodecFactory.deflateCodec(level));
+ String codecName = job.get(AvroJob.OUTPUT_CODEC, DEFLATE_CODEC);
+ CodecFactory factory = codecName.equals(DEFLATE_CODEC)
+ ? CodecFactory.deflateCodec(level)
+ : CodecFactory.fromString(codecName);
+ writer.setCodec(factory);
}
writer.setSyncInterval(job.getInt(SYNC_INTERVAL_KEY, DEFAULT_SYNC_INTERVAL));
Modified: avro/branches/branch-1.5/lang/java/mapred/src/test/java/org/apache/avro/mapred/TestWeather.java
URL: http://svn.apache.org/viewvc/avro/branches/branch-1.5/lang/java/mapred/src/test/java/org/apache/avro/mapred/TestWeather.java?rev=1089551&r1=1089550&r2=1089551&view=diff
==============================================================================
--- avro/branches/branch-1.5/lang/java/mapred/src/test/java/org/apache/avro/mapred/TestWeather.java
(original)
+++ avro/branches/branch-1.5/lang/java/mapred/src/test/java/org/apache/avro/mapred/TestWeather.java
Wed Apr 6 18:00:36 2011
@@ -33,6 +33,7 @@ import org.apache.avro.Schema.Type;
import org.apache.avro.io.DatumReader;
import org.apache.avro.specific.SpecificDatumReader;
import org.apache.avro.file.DataFileReader;
+import static org.apache.avro.file.DataFileConstants.SNAPPY_CODEC;
import org.junit.Test;
import static org.junit.Assert.*;
@@ -123,6 +124,7 @@ public class TestWeather {
FileInputFormat.setInputPaths(job, input);
FileOutputFormat.setOutputPath(job, output);
FileOutputFormat.setCompressOutput(job, true);
+ AvroJob.setOutputCodec(job, SNAPPY_CODEC);
JobClient.runJob(job);
Modified: avro/branches/branch-1.5/lang/java/pom.xml
URL: http://svn.apache.org/viewvc/avro/branches/branch-1.5/lang/java/pom.xml?rev=1089551&r1=1089550&r2=1089551&view=diff
==============================================================================
--- avro/branches/branch-1.5/lang/java/pom.xml (original)
+++ avro/branches/branch-1.5/lang/java/pom.xml Wed Apr 6 18:00:36 2011
@@ -45,6 +45,7 @@
<jetty-version>6.1.26</jetty-version>
<netty-version>3.2.4.Final</netty-version>
<jopt-simple-version>3.2</jopt-simple-version>
+ <snappy-version>1.0.1-rc3</snappy-version>
</properties>
<issueManagement>
@@ -339,6 +340,13 @@
<version>${hadoop-version}</version>
<scope>compile</scope>
</dependency>
+ <dependency>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ <version>${snappy-version}</version>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
</dependencies>
</dependencyManagement>
|