avro-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From th...@apache.org
Subject svn commit: r899182 - in /hadoop/avro/trunk: CHANGES.txt lang/java/src/java/org/apache/avro/io/BinaryDecoder.java lang/java/src/test/java/org/apache/avro/io/Perf.java
Date Thu, 14 Jan 2010 12:47:30 GMT
Author: thiru
Date: Thu Jan 14 12:47:30 2010
New Revision: 899182

URL: http://svn.apache.org/viewvc?rev=899182&view=rev
Log:
AVRO-315. Performance improvements to BinaryDecoder

Added:
    hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/Perf.java
Modified:
    hadoop/avro/trunk/CHANGES.txt
    hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/BinaryDecoder.java

Modified: hadoop/avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/CHANGES.txt?rev=899182&r1=899181&r2=899182&view=diff
==============================================================================
--- hadoop/avro/trunk/CHANGES.txt (original)
+++ hadoop/avro/trunk/CHANGES.txt Thu Jan 14 12:47:30 2010
@@ -222,6 +222,8 @@
     AVRO-291. Set NODELAY in Java's SocketTransceiver.
     (Eric Evans via cutting)
 
+    AVRO-315. Performance improvements to BinaryDecoder (thiru)
+
   BUG FIXES
  
     AVRO-176. Safeguard against bad istreams before reading. (sbanacho)

Modified: hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/BinaryDecoder.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/BinaryDecoder.java?rev=899182&r1=899181&r2=899182&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/BinaryDecoder.java (original)
+++ hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/BinaryDecoder.java Thu Jan 14
12:47:30 2010
@@ -129,31 +129,29 @@
     return (n >>> 1) ^ -(n & 1); // back to two's-complement
   }
 
+  private final byte[] buf = new byte[8];
+
   @Override
   public float readFloat() throws IOException {
-    int n = 0;
-    for (int i = 0, shift = 0; i < 4; i++, shift += 8) {
-      int k = in.read();
-      if (k >= 0) {
-        n |= (k & 0xff) << shift;
-      } else {
-        throw new EOFException();
-      }
-    }
+    doReadBytes(buf, 0, 4);
+    int n = (((int) buf[0]) & 0xff)
+      |  ((((int) buf[1]) & 0xff) << 8)
+      |  ((((int) buf[2]) & 0xff) << 16)
+      |  ((((int) buf[3]) & 0xff) << 24);
     return Float.intBitsToFloat(n);
   }
 
   @Override
   public double readDouble() throws IOException {
-    long n = 0;
-    for (int i = 0, shift = 0; i < 8; i++, shift += 8) {
-      long k = in.read();
-      if (k >= 0) {
-        n |= (k & 0xff) << shift;
-      } else {
-        throw new EOFException();
-      }
-    }
+    doReadBytes(buf, 0, 8);
+    long n = (((long) buf[0]) & 0xff)
+      |  ((((long) buf[1]) & 0xff) << 8)
+      |  ((((long) buf[2]) & 0xff) << 16)
+      |  ((((long) buf[3]) & 0xff) << 24)
+      |  ((((long) buf[4]) & 0xff) << 32)
+      |  ((((long) buf[5]) & 0xff) << 40)
+      |  ((((long) buf[6]) & 0xff) << 48)
+      |  ((((long) buf[7]) & 0xff) << 56);
     return Double.longBitsToDouble(n);
   }
     
@@ -217,9 +215,13 @@
    */
   private void doReadBytes(byte[] bytes, int start, int length)
     throws IOException {
-    while (length > 0) {
+    for (; ;) {
       int n = in.read(bytes, start, length);
-      if (n < 0) throw new EOFException();
+      if (n == length || length == 0) {
+        return;
+      } else if (n < 0) {
+        throw new EOFException();
+      }
       start += n;
       length -= n;
     }

Added: hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/Perf.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/Perf.java?rev=899182&view=auto
==============================================================================
--- hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/Perf.java (added)
+++ hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/Perf.java Thu Jan 14 12:47:30
2010
@@ -0,0 +1,189 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.avro.io;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.Random;
+
+import org.apache.avro.Schema;
+
+/**
+ * Performance tests for various low level operations of
+ * Avro encoding and decoding.
+ */
+public class Perf {
+  private static final int COUNT = 200000; // needs to be a multiple of 4
+  private static final int CYCLES = 150;
+  
+  
+  public static void main(String[] args) throws IOException {
+    Test[] tests = null;
+    if (args.length == 0) {
+      tests = new Test[] { new ReadInt(),
+          new ReadLong(), new ReadFloat(), new ReadDouble() };
+    } else if (args.length == 1) {
+      if (args[0].equals("-i")) {
+        tests = new Test[] { new ReadInt() };
+      } else if (args[0].equals("-f")) {
+        tests = new Test[] { new ReadFloat() };
+      } else if (args[0].equals("-d")) {
+        tests = new Test[] { new ReadDouble() };
+      } else if (args[0].equals("-l")) {
+        tests = new Test[] { new ReadLong() };
+      }
+    } else {
+      usage();
+      System.exit(1);
+    }
+    
+    for (Test t : tests) {
+      // warmup JVM 
+      for (int i = 0; i < CYCLES; i++) {
+        t.read();
+      }
+      // test
+      long s = 0;
+      for (int i = 0; i < CYCLES; i++) {
+        long l = t.read();
+        // System.out.println("** " + l);
+        s += l;
+      }
+       s /= 1000;
+      System.out.println(t.name + "(" + t.schema + "): " + s/1000 + " ms, " +  (CYCLES *
(double)COUNT)/s + " million numbers decoded /sec" );
+    }
+  }
+  
+  private static abstract class Test {
+    public final String name;
+    public final Schema schema;
+    protected byte[] data;
+    public Test(String name, String json) throws IOException {
+      this.name = name;
+      this.schema = Schema.parse(json);
+      ByteArrayOutputStream bao = new ByteArrayOutputStream();
+      Encoder e = new BinaryEncoder(bao);
+      genData(e);
+      data = bao.toByteArray();
+    }
+    public final long read() throws IOException {
+      Decoder d = new BinaryDecoder(new ByteArrayInputStream(data));
+      long t = System.nanoTime();
+      for (long l = d.readArrayStart(); l > 0; l = d.arrayNext()) {
+        for (int j = 0; j < l; j++) {
+          readInternal(d);
+        }
+      }
+      return (System.nanoTime() - t);
+    }
+    abstract void genData(Encoder e) throws IOException;
+    abstract void readInternal(Decoder d) throws IOException;
+  }
+  
+  private static class ReadInt extends Test {
+    public ReadInt() throws IOException {
+      super("ReadInt", "{ \"type\": \"array\", \"items\": \"int\"} ");
+    }
+    @Override void genData(Encoder e) throws IOException {
+      e.writeArrayStart();
+      e.setItemCount((COUNT/4) * 4); //next lowest multiple of 4  
+      Random r = new Random();
+      for (int i = 0; i < COUNT/4; i++) {
+        e.writeInt(r.nextInt(50)); // fits in 1 byte
+        e.writeInt(r.nextInt(5000)); // fits in 2 bytes
+        e.writeInt(r.nextInt(500000)); // fits in 3 bytes
+        e.writeInt(r.nextInt(150000000)); // most in 4, some in 5
+      }
+      e.writeArrayEnd();
+    }
+
+    @Override
+    void readInternal(Decoder d) throws IOException {
+       d.readInt();
+    }
+  }
+
+  private static class ReadLong extends Test {
+    public ReadLong() throws IOException {
+      super("ReadLong", "{ \"type\": \"array\", \"items\": \"long\"} ");
+    }
+    @Override void genData(Encoder e) throws IOException {
+      e.writeArrayStart();
+      e.setItemCount((COUNT / 4) *4);
+      Random r = new Random();
+      for (int i = 0; i < COUNT /4; i++) {
+        e.writeLong(r.nextLong() % 0x7FL); // half fit in 1, half in 2 
+        e.writeLong(r.nextLong() % 0x1FFFFFL); // half fit in <=3, half in 4
+        e.writeLong(r.nextLong() % 0x3FFFFFFFFL); // half in <=5, half in 6
+        e.writeLong(r.nextLong() % 0x1FFFFFFFFFFFFL); // half in <=8, half in 9 
+      }
+      e.writeArrayEnd();
+    }
+
+    @Override
+    void readInternal(Decoder d) throws IOException {
+       d.readLong();
+    }
+  }
+  
+  private static class ReadFloat extends Test {
+    public ReadFloat() throws IOException {
+      super("ReadFloat", "{ \"type\": \"array\", \"items\": \"float\"} ");
+    }
+    @Override void genData(Encoder e) throws IOException {
+       e.writeArrayStart();
+      e.setItemCount(COUNT);
+      Random r = new Random();
+      for (int i = 0; i < COUNT; i++) {
+        e.writeFloat(r.nextFloat());
+      }
+      e.writeArrayEnd();
+    }
+    @Override
+    void readInternal(Decoder d) throws IOException {
+      d.readFloat();
+    }
+  }
+
+  private static class ReadDouble extends Test {
+    public ReadDouble() throws IOException {
+      super("ReadDouble", "{ \"type\": \"array\", \"items\": \"double\"} ");
+    }
+    @Override void genData(Encoder e) throws IOException {
+       e.writeArrayStart();
+      e.setItemCount(COUNT);
+      Random r = new Random();
+      for (int i = 0; i < COUNT; i++) {
+        e.writeDouble(r.nextFloat());
+      }
+      e.writeArrayEnd();
+    }
+    @Override
+    void readInternal(Decoder d) throws IOException {
+      d.readDouble();
+    }
+  }
+  private static void usage() {
+    System.out.println("Usage: Perf { -i | -l | -f | -d }");
+    System.out.println("    -i      measures readInt() performance");
+    System.out.println("    -l      measures readLong() performance");
+    System.out.println("    -f      measures readFloat() performance");
+    System.out.println("    -d      measures readDouble() performance");
+  }
+}



Mime
View raw message