avro-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r1096800 - in /avro/branches/branch-1.5: ./ lang/java/mapred/src/main/java/org/apache/avro/mapred/ lang/java/mapred/src/test/java/org/apache/avro/mapred/
Date Tue, 26 Apr 2011 16:34:50 GMT
Author: cutting
Date: Tue Apr 26 16:34:50 2011
New Revision: 1096800

URL: http://svn.apache.org/viewvc?rev=1096800&view=rev
Log:
Merge -c 1096798 from trunk to 1.5 branch.  Fixes AVRO-808.

Added:
    avro/branches/branch-1.5/lang/java/mapred/src/main/java/org/apache/avro/mapred/AvroAsTextInputFormat.java
      - copied unchanged from r1096798, avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/AvroAsTextInputFormat.java
    avro/branches/branch-1.5/lang/java/mapred/src/main/java/org/apache/avro/mapred/AvroAsTextRecordReader.java
      - copied unchanged from r1096798, avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/AvroAsTextRecordReader.java
    avro/branches/branch-1.5/lang/java/mapred/src/test/java/org/apache/avro/mapred/TestAvroAsTextInputFormat.java
      - copied unchanged from r1096798, avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/mapred/TestAvroAsTextInputFormat.java
Modified:
    avro/branches/branch-1.5/   (props changed)
    avro/branches/branch-1.5/CHANGES.txt
    avro/branches/branch-1.5/lang/java/mapred/src/test/java/org/apache/avro/mapred/WordCountUtil.java

Propchange: avro/branches/branch-1.5/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr 26 16:34:50 2011
@@ -1 +1 @@
-/avro/trunk:1075938,1075993,1078917,1079055,1079060,1079063,1083246,1085921,1086727,1086730,1086866,1087076,1087129,1087136,1087439-1087440,1087463,1087472,1087792,1089128,1089131,1089550,1094812,1095206-1095208,1095493,1095529,1095548,1095550
+/avro/trunk:1075938,1075993,1078917,1079055,1079060,1079063,1083246,1085921,1086727,1086730,1086866,1087076,1087129,1087136,1087439-1087440,1087463,1087472,1087792,1089128,1089131,1089550,1094812,1095206-1095208,1095493,1095529,1095548,1095550,1096798

Modified: avro/branches/branch-1.5/CHANGES.txt
URL: http://svn.apache.org/viewvc/avro/branches/branch-1.5/CHANGES.txt?rev=1096800&r1=1096799&r2=1096800&view=diff
==============================================================================
--- avro/branches/branch-1.5/CHANGES.txt (original)
+++ avro/branches/branch-1.5/CHANGES.txt Tue Apr 26 16:34:50 2011
@@ -10,6 +10,9 @@ Avro 1.5.1 (unreleased)
     AVRO-788. Java: Add Snappy compression for data files, including
     MapReduce API support. (cutting)
 
+    AVRO-808. Java: Add AvroAsTextInputFormat for use with streaming.
+    (Tom White via cutting)
+
   IMPROVEMENTS
 
     AVRO-785. Java: Squash a Velocity warning by upgrading to Velocity 1.7.

Modified: avro/branches/branch-1.5/lang/java/mapred/src/test/java/org/apache/avro/mapred/WordCountUtil.java
URL: http://svn.apache.org/viewvc/avro/branches/branch-1.5/lang/java/mapred/src/test/java/org/apache/avro/mapred/WordCountUtil.java?rev=1096800&r1=1096799&r2=1096800&view=diff
==============================================================================
--- avro/branches/branch-1.5/lang/java/mapred/src/test/java/org/apache/avro/mapred/WordCountUtil.java
(original)
+++ avro/branches/branch-1.5/lang/java/mapred/src/test/java/org/apache/avro/mapred/WordCountUtil.java
Tue Apr 26 16:34:50 2011
@@ -20,13 +20,19 @@ package org.apache.avro.mapred;
 
 import static org.junit.Assert.*;
 
+import java.io.BufferedReader;
+import java.io.FileReader;
 import java.io.IOException;
 import java.io.File;
 import java.io.InputStream;
 import java.io.FileInputStream;
 import java.io.BufferedInputStream;
 import java.io.PrintStream;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
 import java.util.StringTokenizer;
 import java.util.Map;
 import java.util.TreeMap;
@@ -53,6 +59,8 @@ class WordCountUtil {
     = new File(new File(DIR, "in"), "lines.txt");
   private static final File COUNTS_FILE
     = new File(new File(DIR, "out"), "part-00000.avro");
+  private static final File SORTED_FILE
+    = new File(new File(DIR, "out"), "part-00000");
 
   public static final String[] LINES = new String[] {
     "the quick brown fox jumps over the lazy dog",
@@ -84,6 +92,17 @@ class WordCountUtil {
       out.append(new Utf8(line));
     out.close();
   }
+
+  public static void writeLinesBytesFile() throws IOException {
+    FileUtil.fullyDelete(DIR);
+    DatumWriter<ByteBuffer> writer = new GenericDatumWriter<ByteBuffer>();
+    DataFileWriter<ByteBuffer> out = new DataFileWriter<ByteBuffer>(writer);
+    LINES_FILE.getParentFile().mkdirs();
+    out.create(Schema.create(Schema.Type.BYTES), LINES_FILE);
+    for (String line : LINES)
+      out.append(ByteBuffer.wrap(line.getBytes("UTF-8")));
+    out.close();
+  }
   
   public static void writeLinesTextFile() throws IOException {
     FileUtil.fullyDelete(DIR);
@@ -110,6 +129,19 @@ class WordCountUtil {
     in.close();
     assertEquals(COUNTS.size(), numWords);
   }
+  
+  public static void validateSortedFile() throws Exception {
+    BufferedReader reader = new BufferedReader(new FileReader(SORTED_FILE));
+    List<String> sortedLines = new ArrayList<String>();
+    for (String line : LINES) {
+      sortedLines.add(line);
+    }
+    Collections.sort(sortedLines);
+    for (String expectedLine : sortedLines) {
+      assertEquals(expectedLine, reader.readLine().trim());
+    }
+    assertNull(reader.readLine());
+  }
 
   // metadata tests
   private static final String STRING_KEY = "string-key";



Mime
View raw message