avro-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dcrea...@apache.org
Subject svn commit: r1340989 - in /avro/trunk: CHANGES.txt lang/c/src/datafile.c lang/c/tests/CMakeLists.txt lang/c/tests/test_avro_1087.c
Date Mon, 21 May 2012 12:28:03 GMT
Author: dcreager
Date: Mon May 21 12:28:02 2012
New Revision: 1340989

URL: http://svn.apache.org/viewvc?rev=1340989&view=rev
Log:
AVRO-1087. C: avro_file_writer_open() and appending values works correctly.

The avro_file_writer_open() function wasn't correctly setting up the
avro_file_writer_t instance to allow you to append data to an existing
file.  We now open up the file for read+write, read in the header, and
set up the writer's codec to match what's in the existing file.  The
patch includes a test case.

Contributed by Pugachev Maxim.

Added:
    avro/trunk/lang/c/tests/test_avro_1087.c
Modified:
    avro/trunk/CHANGES.txt
    avro/trunk/lang/c/src/datafile.c
    avro/trunk/lang/c/tests/CMakeLists.txt

Modified: avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1340989&r1=1340988&r2=1340989&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Mon May 21 12:28:02 2012
@@ -113,6 +113,9 @@ Avro 1.7.0 (unreleased)
     AVRO-1091. C: Helper scripts for calling CMake.
     (Vivek Nadkarni via dcreager)
 
+    AVRO-1087. C: avro_file_writer_open() and appending Avro values
+    works correctly.  (Pugachev Maxim via dcreager)
+
 Avro 1.6.3 (5 March 2012)
 
     AVRO-1077. Missing 'inline' for union set function. (thiru)

Modified: avro/trunk/lang/c/src/datafile.c
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c/src/datafile.c?rev=1340989&r1=1340988&r2=1340989&view=diff
==============================================================================
--- avro/trunk/lang/c/src/datafile.c (original)
+++ avro/trunk/lang/c/src/datafile.c Mon May 21 12:28:02 2012
@@ -310,13 +310,17 @@ static int file_writer_open(const char *
 	FILE *fp;
 	avro_reader_t reader;
 
-	fp = fopen(path, "rb");
+	/* Open for read AND write */
+	fp = fopen(path, "r+b");
 	if (!fp) {
 		avro_set_error("Error opening file: %s",
 			       strerror(errno));
 		return errno;
 	}
-	reader = avro_reader_file(fp);
+
+	/* Don`t close the underlying file descriptor, logrotate can
+	 * vanish it from sight. */
+	reader = avro_reader_file_fp(fp, 0);
 	if (!reader) {
 		fclose(fp);
 		avro_set_error("Cannot create file reader for %s", path);
@@ -325,13 +329,44 @@ static int file_writer_open(const char *
 	rval =
 	    file_read_header(reader, &w->writers_schema, w->codec, w->sync,
 			     sizeof(w->sync));
+
 	avro_reader_free(reader);
-	/* Position to end of file and get ready to write */
-	rval = file_writer_init_fp(path, "ab", w);
 	if (rval) {
-		avro_freet(struct avro_file_writer_t_, w);
+		fclose(fp);
+		return rval;
 	}
-	return rval;
+
+	w->block_count = 0;
+
+	/* Position to end of file and get ready to write */
+	fseek(fp, 0, SEEK_END);
+
+	w->writer = avro_writer_file(fp);
+	if (!w->writer) {
+		fclose(fp);
+		avro_set_error("Cannot create file writer for %s", path);
+		return ENOMEM;
+	}
+
+	w->datum_buffer_size = DEFAULT_BLOCK_SIZE;
+	w->datum_buffer = avro_malloc(w->datum_buffer_size);
+
+	if(!w->datum_buffer) {
+		avro_set_error("Could not allocate datum buffer\n");
+		avro_writer_free(w->writer);
+		return ENOMEM;
+	}
+
+	w->datum_writer =
+	    avro_writer_memory(w->datum_buffer, w->datum_buffer_size);
+	if (!w->datum_writer) {
+		avro_set_error("Cannot create datum writer for file %s", path);
+		avro_writer_free(w->writer);
+		avro_free(w->datum_buffer, w->datum_buffer_size);
+		return ENOMEM;
+	}
+
+	return 0;
 }
 
 int avro_file_writer_open(const char *path, avro_file_writer_t * writer)
@@ -346,8 +381,16 @@ int avro_file_writer_open(const char *pa
 		avro_set_error("Cannot create new file writer for %s", path);
 		return ENOMEM;
 	}
+	w->codec = avro_new(struct avro_codec_t_);
+	if (!w->codec) {
+		avro_set_error("Cannot allocate new codec");
+		avro_freet(struct avro_file_writer_t_, w);
+		return ENOMEM;
+	}
 	rval = file_writer_open(path, w);
 	if (rval) {
+		avro_codec_reset(w->codec);
+		avro_freet(struct avro_codec_t_, w->codec);
 		avro_freet(struct avro_file_writer_t_, w);
 		return rval;
 	}

Modified: avro/trunk/lang/c/tests/CMakeLists.txt
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c/tests/CMakeLists.txt?rev=1340989&r1=1340988&r2=1340989&view=diff
==============================================================================
--- avro/trunk/lang/c/tests/CMakeLists.txt (original)
+++ avro/trunk/lang/c/tests/CMakeLists.txt Mon May 21 12:28:02 2012
@@ -52,6 +52,7 @@ add_avro_test(test_avro_968)
 add_avro_test(test_avro_984)
 add_avro_test(test_avro_1034)
 add_avro_test(test_avro_1084)
+add_avro_test(test_avro_1087)
 add_avro_test(test_avro_data)
 add_avro_test(test_refcount)
 add_avro_test(test_cpp test_cpp.cpp)

Added: avro/trunk/lang/c/tests/test_avro_1087.c
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c/tests/test_avro_1087.c?rev=1340989&view=auto
==============================================================================
--- avro/trunk/lang/c/tests/test_avro_1087.c (added)
+++ avro/trunk/lang/c/tests/test_avro_1087.c Mon May 21 12:28:02 2012
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied.  See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+#include <avro.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+const char  PERSON_SCHEMA[] =
+"{\"type\":\"record\",\
+  \"name\":\"Person\",\
+  \"fields\":[\
+     {\"name\": \"ID\", \"type\": \"int\"}]}";
+
+const char *dbname = "test.db";
+avro_schema_t schema;
+
+void add_record (avro_file_writer_t writer)
+{
+	avro_datum_t main_datum = avro_record(schema);		
+	avro_datum_t id_datum = avro_int32(1);
+	
+	if (avro_record_set (main_datum, "ID", id_datum))
+	{
+		printf ("Unable to create datum");
+		exit (EXIT_FAILURE);
+	}
+
+	avro_file_writer_append (writer, main_datum);
+
+	avro_datum_decref (id_datum);
+	avro_datum_decref (main_datum);
+}
+
+void create_database()
+{
+	avro_file_writer_t writer;
+
+	if (avro_schema_from_json_literal (PERSON_SCHEMA, &schema))
+	{
+		printf ("Unable to parse schema\n");
+		exit (EXIT_FAILURE);
+	}
+
+	if (avro_file_writer_create ("test.db", schema, &writer))
+	{
+		printf ("There was an error creating db: %s\n", avro_strerror());
+		exit (EXIT_FAILURE);
+	}
+
+	add_record (writer);
+
+	avro_file_writer_flush (writer);
+	avro_file_writer_close (writer);
+}
+
+
+int main()
+{
+	avro_file_writer_t writer;
+
+	create_database();
+
+	avro_file_writer_open (dbname, &writer);
+	add_record (writer);
+	
+	avro_file_writer_flush (writer);
+	avro_file_writer_close (writer);
+
+	remove (dbname);
+
+	return EXIT_SUCCESS;
+}



Mime
View raw message