avro-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dcrea...@apache.org
Subject svn commit: r1507309 - in /avro/trunk: CHANGES.txt lang/c/CMakeLists.txt lang/c/FindSnappy.cmake lang/c/src/codec.c lang/c/src/codec.h
Date Fri, 26 Jul 2013 13:40:47 GMT
Author: dcreager
Date: Fri Jul 26 13:40:47 2013
New Revision: 1507309

URL: http://svn.apache.org/r1507309
Log:
AVRO-896. C: Snappy compression codec.

Like all other codecs, this is optional.  It will only be activated if the
snappy C library is installed at build time.  Contributed by Grisha Trubetskoy.

Added:
    avro/trunk/lang/c/FindSnappy.cmake
Modified:
    avro/trunk/CHANGES.txt
    avro/trunk/lang/c/CMakeLists.txt
    avro/trunk/lang/c/src/codec.c
    avro/trunk/lang/c/src/codec.h

Modified: avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1507309&r1=1507308&r2=1507309&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Fri Jul 26 13:40:47 2013
@@ -14,6 +14,8 @@ Trunk (not yet released)
 
     AVRO-823: C#: Add data file support. (David McIntosh via cutting)
 
+    AVRO-896. C: Snappy compression codec. (Grisha Trubetskoy via dcreager)
+
   IMPROVEMENTS
 
     AVRO-1260. Ruby: Improve read performance. (Martin Kleppmann via cutting)

Modified: avro/trunk/lang/c/CMakeLists.txt
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c/CMakeLists.txt?rev=1507309&r1=1507308&r2=1507309&view=diff
==============================================================================
--- avro/trunk/lang/c/CMakeLists.txt (original)
+++ avro/trunk/lang/c/CMakeLists.txt Fri Jul 26 13:40:47 2013
@@ -20,6 +20,8 @@ cmake_minimum_required(VERSION 2.4)
 project(AvroC)
 enable_testing()
 
+set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_SOURCE_DIR})
+
 # Eliminates warning about linker paths when linking both zlib and
 # liblzma.
 cmake_policy(SET CMP0003 NEW)
@@ -125,6 +127,7 @@ include_directories(${AvroC_SOURCE_DIR}/
 
 
 # Enable codecs
+
 find_package(ZLIB)
 if (ZLIB_FOUND)
     set(ZLIB_PKG zlib)
@@ -136,6 +139,16 @@ else (ZLIB_FOUND)
     message("Disabled deflate codec. zlib not found.")
 endif (ZLIB_FOUND)
 
+find_package(Snappy)
+if (SNAPPY_FOUND AND ZLIB_FOUND)  # Snappy borrows crc32 from zlib
+   set(SNAPPY_PKG libsnappy)
+   add_definitions(-DSNAPPY_CODEC)
+   message("Enabled snappy codec")
+else (SNAPPY_FOUND AND ZLIB_FOUND)
+   set (SNAPPY_PKG "")
+   message("Disabled snappy codec. libsnappy not found or zlib not found.")
+endif (SNAPPY_FOUND AND ZLIB_FOUND)
+
 find_package(PkgConfig)
 pkg_check_modules(LZMA liblzma)
 if (LZMA_FOUND)
@@ -149,8 +162,8 @@ else (LZMA_FOUND)
     message("Disabled lzma codec. liblzma not found.")
 endif (LZMA_FOUND)
 
-set(CODEC_LIBRARIES ${ZLIB_LIBRARIES} ${LZMA_LIBRARIES})
-set(CODEC_PKG "@ZLIB_PKG@ @LZMA_PKG@")
+set(CODEC_LIBRARIES ${ZLIB_LIBRARIES} ${LZMA_LIBRARIES} ${SNAPPY_LIBRARIES})
+set(CODEC_PKG "@ZLIB_PKG@ @LZMA_PKG@ @SNAPPY_PKG@")
 
 
 add_subdirectory(src)

Added: avro/trunk/lang/c/FindSnappy.cmake
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c/FindSnappy.cmake?rev=1507309&view=auto
==============================================================================
--- avro/trunk/lang/c/FindSnappy.cmake (added)
+++ avro/trunk/lang/c/FindSnappy.cmake Fri Jul 26 13:40:47 2013
@@ -0,0 +1,36 @@
+# Tries to find Snappy headers and libraries.
+#
+# Usage of this module as follows:
+#
+#  find_package(Snappy)
+#
+# Variables used by this module, they can change the default behaviour and need
+# to be set before calling find_package:
+#
+#  SNAPPY_ROOT_DIR  Set this variable to the root installation of
+#                    Snappy if the module has problems finding
+#                    the proper installation path.
+#
+# Variables defined by this module:
+#
+#  SNAPPY_FOUND              System has Snappy libs/headers
+#  SNAPPY_LIBRARIES          The Snappy libraries
+#  SNAPPY_INCLUDE_DIR        The location of Snappy headers
+
+find_path(SNAPPY_INCLUDE_DIR
+    NAMES snappy.h
+    HINTS ${SNAPPY_ROOT_DIR}/include)
+
+find_library(SNAPPY_LIBRARIES
+    NAMES snappy
+    HINTS ${SNAPPY_ROOT_DIR}/lib)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(Snappy DEFAULT_MSG
+    SNAPPY_LIBRARIES
+    SNAPPY_INCLUDE_DIR)
+
+mark_as_advanced(
+    SNAPPY_ROOT_DIR
+    SNAPPY_LIBRARIES
+    SNAPPY_INCLUDE_DIR)

Modified: avro/trunk/lang/c/src/codec.c
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c/src/codec.c?rev=1507309&r1=1507308&r2=1507309&view=diff
==============================================================================
--- avro/trunk/lang/c/src/codec.c (original)
+++ avro/trunk/lang/c/src/codec.c Fri Jul 26 13:40:47 2013
@@ -16,6 +16,10 @@
  */
 
 #include <string.h>
+#ifdef SNAPPY_CODEC
+#include <snappy-c.h>
+#include <byteswap.h>
+#endif
 #ifdef DEFLATE_CODEC
 #include <zlib.h>
 #endif
@@ -71,6 +75,112 @@ static int reset_null(avro_codec_t c)
 	return 0;
 }
 
+/* Snappy codec */
+
+#ifdef SNAPPY_CODEC
+
+static int
+codec_snappy(avro_codec_t codec)
+{
+	codec->name = "snappy";
+	codec->type = AVRO_CODEC_SNAPPY;
+	codec->block_size = 0;
+	codec->used_size = 0;
+	codec->block_data = NULL;
+	codec->codec_data = NULL;
+
+	return 0;
+}
+
+static int encode_snappy(avro_codec_t c, void * data, int64_t len)
+{
+        uint32_t crc;
+        size_t outlen = snappy_max_compressed_length(len);
+
+	if (!c->block_data) {
+		c->block_data = avro_malloc(outlen+4);
+		c->block_size = outlen+4;
+	} else if (c->block_size < (int64_t) (outlen+4)) {
+            c->block_data = avro_realloc(c->block_data, c->block_size, (outlen+4));
+		c->block_size = outlen+4;
+	}
+
+	if (!c->block_data) {
+		avro_set_error("Cannot allocate memory for snappy");
+		return 1;
+	}
+
+        if (snappy_compress(data, len, c->block_data, &outlen) != SNAPPY_OK)
+        {
+                avro_set_error("Error compressing block with Snappy");
+		return 1;
+	}
+
+        crc = __bswap_32(crc32(0, data, len));
+        memcpy(c->block_data+outlen, &crc, 4);
+        c->used_size = outlen+4;
+
+	return 0;
+}
+
+static int decode_snappy(avro_codec_t c, void * data, int64_t len)
+{
+        uint32_t crc;
+        size_t outlen;
+
+        if (snappy_uncompressed_length(data, len-4, &outlen) != SNAPPY_OK) {
+		avro_set_error("Uncompressed length error in snappy");
+		return 1;
+        }
+
+	if (!c->block_data) {
+		c->block_data = avro_malloc(outlen);
+		c->block_size = outlen;
+	} else if ( (size_t)c->block_size < outlen) {
+		c->block_data = avro_realloc(c->block_data, c->block_size, outlen);
+		c->block_size = outlen;
+	}
+
+	if (!c->block_data)
+	{
+		avro_set_error("Cannot allocate memory for snappy");
+		return 1;
+	}
+
+        if (snappy_uncompress(data, len-4, c->block_data, &outlen) != SNAPPY_OK)
+        {
+                avro_set_error("Error uncompressing block with Snappy");
+		return 1;
+	}
+
+        crc = __bswap_32(crc32(0, c->block_data, outlen));
+        if (memcmp(&crc, (char*)data+len-4, 4))
+        {
+                avro_set_error("CRC32 check failure uncompressing block with Snappy");
+		return 1;
+	}
+
+        c->used_size = outlen;
+
+	return 0;
+}
+
+static int reset_snappy(avro_codec_t c)
+{
+	if (c->block_data) {
+		avro_free(c->block_data, c->block_size);
+	}
+
+	c->block_data = NULL;
+	c->block_size = 0;
+	c->used_size = 0;
+	c->codec_data = NULL;
+
+	return 0;
+}
+
+#endif // SNAPPY_CODEC
+
 /* Deflate codec */
 
 #ifdef DEFLATE_CODEC
@@ -403,6 +513,12 @@ int avro_codec(avro_codec_t codec, const
 		return codec_null(codec);
 	}
 
+#ifdef SNAPPY_CODEC
+	if (strcmp("snappy", type) == 0) {
+		return codec_snappy(codec);
+	}
+#endif
+
 #ifdef DEFLATE_CODEC
 	if (strcmp("deflate", type) == 0) {
 		return codec_deflate(codec);
@@ -429,6 +545,10 @@ int avro_codec_encode(avro_codec_t c, vo
 	{
 	case AVRO_CODEC_NULL:
 		return encode_null(c, data, len);
+#ifdef SNAPPY_CODEC
+	case AVRO_CODEC_SNAPPY:
+		return encode_snappy(c, data, len);
+#endif
 #ifdef DEFLATE_CODEC
 	case AVRO_CODEC_DEFLATE:
 		return encode_deflate(c, data, len);
@@ -448,6 +568,10 @@ int avro_codec_decode(avro_codec_t c, vo
 	{
 	case AVRO_CODEC_NULL:
 		return decode_null(c, data, len);
+#ifdef SNAPPY_CODEC
+	case AVRO_CODEC_SNAPPY:
+		return decode_snappy(c, data, len);
+#endif
 #ifdef DEFLATE_CODEC
 	case AVRO_CODEC_DEFLATE:
 		return decode_deflate(c, data, len);
@@ -467,6 +591,10 @@ int avro_codec_reset(avro_codec_t c)
 	{
 	case AVRO_CODEC_NULL:
 		return reset_null(c);
+#ifdef SNAPPY_CODEC
+	case AVRO_CODEC_SNAPPY:
+		return reset_snappy(c);
+#endif
 #ifdef DEFLATE_CODEC
 	case AVRO_CODEC_DEFLATE:
 		return reset_deflate(c);

Modified: avro/trunk/lang/c/src/codec.h
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c/src/codec.h?rev=1507309&r1=1507308&r2=1507309&view=diff
==============================================================================
--- avro/trunk/lang/c/src/codec.h (original)
+++ avro/trunk/lang/c/src/codec.h Fri Jul 26 13:40:47 2013
@@ -29,7 +29,8 @@ extern "C" {
 enum avro_codec_type_t {
 	AVRO_CODEC_NULL,
 	AVRO_CODEC_DEFLATE,
-	AVRO_CODEC_LZMA
+	AVRO_CODEC_LZMA,
+	AVRO_CODEC_SNAPPY
 };
 typedef enum avro_codec_type_t avro_codec_type_t;
 



Mime
View raw message