avro-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dcrea...@apache.org
Subject svn commit: r1551926 - in /avro/trunk: CHANGES.txt lang/c/src/avro/io.h lang/c/src/datafile.c lang/c/tests/CMakeLists.txt lang/c/tests/test_avro_1379.c
Date Wed, 18 Dec 2013 12:50:07 GMT
Author: dcreager
Date: Wed Dec 18 12:50:06 2013
New Revision: 1551926

URL: http://svn.apache.org/r1551926
Log:
AVRO-1379. C: avro_file_writer_append_encoded() function

Allows you to append to a file an Avro value that you already have the
binary encoding for.  Contributed by Mark Teodoro.

Added:
    avro/trunk/lang/c/tests/test_avro_1379.c
Modified:
    avro/trunk/CHANGES.txt
    avro/trunk/lang/c/src/avro/io.h
    avro/trunk/lang/c/src/datafile.c
    avro/trunk/lang/c/tests/CMakeLists.txt

Modified: avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1551926&r1=1551925&r2=1551926&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Wed Dec 18 12:50:06 2013
@@ -24,6 +24,9 @@ Trunk (not yet released)
     AVRO-1409. Java: Add an API for testing schema compatibility.
     (Christophe Taton via cutting)
 
+    AVRO-1379. C: avro_file_writer_append_encoded() function.
+    (Mark Teodoro via dcreager)
+
   IMPROVEMENTS
 
     AVRO-1355. Java: Reject schemas with duplicate field

Modified: avro/trunk/lang/c/src/avro/io.h
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c/src/avro/io.h?rev=1551926&r1=1551925&r2=1551926&view=diff
==============================================================================
--- avro/trunk/lang/c/src/avro/io.h (original)
+++ avro/trunk/lang/c/src/avro/io.h Wed Dec 18 12:50:06 2013
@@ -130,6 +130,10 @@ avro_file_reader_read_value(avro_file_re
 int
 avro_file_writer_append_value(avro_file_writer_t writer, avro_value_t *src);
 
+int
+avro_file_writer_append_encoded(avro_file_writer_t writer,
+				const void *buf, int64_t len);
+
 /*
  * Legacy avro_datum_t API
  */

Modified: avro/trunk/lang/c/src/datafile.c
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c/src/datafile.c?rev=1551926&r1=1551925&r2=1551926&view=diff
==============================================================================
--- avro/trunk/lang/c/src/datafile.c (original)
+++ avro/trunk/lang/c/src/datafile.c Wed Dec 18 12:50:06 2013
@@ -623,6 +623,29 @@ avro_file_writer_append_value(avro_file_
 	return 0;
 }
 
+int
+avro_file_writer_append_encoded(avro_file_writer_t w,
+				const void *buf, int64_t len)
+{
+	int rval;
+	check_param(EINVAL, w, "writer");
+
+	rval = avro_write(w->datum_writer, (void *) buf, len);
+	if (rval) {
+		check(rval, file_write_block(w));
+		rval = avro_write(w->datum_writer, (void *) buf, len);
+		if (rval) {
+			avro_set_error("Value too large for file block size");
+			/* TODO: if the value encoder larger than our buffer,
+			   just write a single large datum */
+			return rval;
+		}
+	}
+	w->block_count++;
+	w->block_size = avro_writer_tell(w->datum_writer);
+	return 0;
+}
+
 int avro_file_writer_sync(avro_file_writer_t w)
 {
 	return file_write_block(w);

Modified: avro/trunk/lang/c/tests/CMakeLists.txt
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c/tests/CMakeLists.txt?rev=1551926&r1=1551925&r2=1551926&view=diff
==============================================================================
--- avro/trunk/lang/c/tests/CMakeLists.txt (original)
+++ avro/trunk/lang/c/tests/CMakeLists.txt Wed Dec 18 12:50:06 2013
@@ -59,3 +59,4 @@ add_avro_test(test_avro_1279)
 add_avro_test(test_avro_data)
 add_avro_test(test_refcount)
 add_avro_test(test_cpp test_cpp.cpp)
+add_avro_test(test_avro_1379)

Added: avro/trunk/lang/c/tests/test_avro_1379.c
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c/tests/test_avro_1379.c?rev=1551926&view=auto
==============================================================================
--- avro/trunk/lang/c/tests/test_avro_1379.c (added)
+++ avro/trunk/lang/c/tests/test_avro_1379.c Wed Dec 18 12:50:06 2013
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied.  See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+#include "avro.h"
+#include "avro_private.h"
+#include <stdio.h>
+#include <stdlib.h>
+
+static const char  *schema_json =
+	"{"
+	"  \"type\": \"record\","
+	"  \"name\": \"test\","
+	"  \"fields\": ["
+	"    { \"name\": \"i\", \"type\": \"int\" },"
+	"    { \"name\": \"l\", \"type\": \"long\" },"
+	"    { \"name\": \"s\", \"type\": \"string\" },"
+	"    {"
+	"      \"name\": \"subrec\","
+	"      \"type\": {"
+	"        \"type\": \"record\","
+	"        \"name\": \"sub\","
+	"        \"fields\": ["
+	"          { \"name\": \"f\", \"type\": \"float\" },"
+	"          { \"name\": \"d\", \"type\": \"double\" }"
+	"        ]"
+	"      }"
+	"    }"
+	"  ]"
+	"}";
+
+static void
+populate_complex_record(avro_value_t *p_val)
+{
+	avro_value_t  field;
+
+	avro_value_get_by_index(p_val, 0, &field, NULL);
+	avro_value_set_int(&field, 42);
+
+	avro_value_get_by_index(p_val, 1, &field, NULL);
+	avro_value_set_long(&field, 4242);
+
+	avro_wrapped_buffer_t  wbuf;
+	avro_wrapped_buffer_new_string(&wbuf, "Follow your bliss.");
+	avro_value_get_by_index(p_val, 2, &field, NULL);
+	avro_value_give_string_len(&field, &wbuf);
+
+	avro_value_t  subrec;
+	avro_value_get_by_index(p_val, 3, &subrec, NULL);
+
+	avro_value_get_by_index(&subrec, 0, &field, NULL);
+	avro_value_set_float(&field, 3.14159265);
+
+	avro_value_get_by_index(&subrec, 1, &field, NULL);
+	avro_value_set_double(&field, 2.71828183);
+}
+
+int main(void)
+{
+	int rval = 0;
+	size_t len;
+	static char  buf[4096];
+	avro_writer_t  writer;
+	avro_file_writer_t file_writer;
+	avro_file_reader_t file_reader;
+	const char *outpath = "test-1379.avro";
+
+	avro_schema_t  schema = NULL;
+	avro_schema_error_t  error = NULL;
+	check(rval, avro_schema_from_json(schema_json, strlen(schema_json), &schema, &error));
+
+	avro_value_iface_t  *iface = avro_generic_class_from_schema(schema);
+
+	avro_value_t  val;
+	avro_generic_value_new(iface, &val);
+
+	avro_value_t  out;
+	avro_generic_value_new(iface, &out);
+
+	/* create the val */
+	avro_value_reset(&val);
+	populate_complex_record(&val);
+
+	/* create the writers */
+	writer = avro_writer_memory(buf, sizeof(buf));
+	check(rval, avro_file_writer_create(outpath, schema, &file_writer));
+
+	fprintf(stderr, "Writing to buffer\n");
+	check(rval, avro_value_write(writer, &val));
+
+	fprintf(stderr, "Writing buffer to %s "
+		"using avro_file_writer_append_encoded()\n", outpath);
+	len = avro_writer_tell(writer);
+	check(rval, avro_file_writer_append_encoded(file_writer, buf, len));
+	check(rval, avro_file_writer_close(file_writer));
+
+	check(rval, avro_file_reader(outpath, &file_reader));
+	fprintf(stderr, "Re-reading value to verify\n");
+	check(rval, avro_file_reader_read_value(file_reader, &out));
+	fprintf(stderr, "Verifying value...");
+	if (!avro_value_equal(&val, &out)) {
+		fprintf(stderr, "fail!\n");
+		exit(EXIT_FAILURE);
+	}
+	fprintf(stderr, "ok\n");
+	check(rval, avro_file_reader_close(file_reader));
+	remove(outpath);
+
+	exit(EXIT_SUCCESS);
+}



Mime
View raw message