orc-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject orc git commit: ORC-231: Configurable capability to overwrite the file if it exists.
Date Tue, 29 Aug 2017 19:44:45 GMT
Repository: orc
Updated Branches:
  refs/heads/master 1b30e322a -> 22c6c5bf2


ORC-231: Configurable capability to overwrite the file if it exists.

The default value of this flag is false and this change is backward
compatible.

Fixes #162

Signed-off-by: Owen O'Malley <omalley@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/orc/repo
Commit: http://git-wip-us.apache.org/repos/asf/orc/commit/22c6c5bf
Tree: http://git-wip-us.apache.org/repos/asf/orc/tree/22c6c5bf
Diff: http://git-wip-us.apache.org/repos/asf/orc/diff/22c6c5bf

Branch: refs/heads/master
Commit: 22c6c5bf2463c6fdd7ecc546e669c4ea0cdeba3d
Parents: 1b30e32
Author: Ajay Yadava <ajayyadava@apache.org>
Authored: Mon Aug 21 23:49:54 2017 -0700
Committer: Owen O'Malley <omalley@apache.org>
Committed: Tue Aug 29 12:43:53 2017 -0700

----------------------------------------------------------------------
 java/core/src/java/org/apache/orc/OrcConf.java  |  4 +-
 java/core/src/java/org/apache/orc/OrcFile.java  | 15 +++++
 .../org/apache/orc/impl/PhysicalFsWriter.java   |  2 +-
 .../org/apache/orc/impl/TestWriterImpl.java     | 71 ++++++++++++++++++++
 4 files changed, 90 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/orc/blob/22c6c5bf/java/core/src/java/org/apache/orc/OrcConf.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/OrcConf.java b/java/core/src/java/org/apache/orc/OrcConf.java
index 10d4228..35328da 100644
--- a/java/core/src/java/org/apache/orc/OrcConf.java
+++ b/java/core/src/java/org/apache/orc/OrcConf.java
@@ -146,7 +146,9 @@ public enum OrcConf {
   ROWS_BETWEEN_CHECKS("orc.rows.between.memory.checks", "orc.rows.between.memory.checks",
5000,
     "How often should MemoryManager check the memory sizes? Measured in rows\n" +
       "added to all of the writers.  Valid range is [1,10000] and is primarily meant for"
+
-      "n\testing.  Setting this too low may negatively affect performance.")
+      "n\testing.  Setting this too low may negatively affect performance."),
+  OVERWRITE_OUTPUT_FILE("orc.overwrite.output.file", "orc.overwrite.output.file", false,
+    "A boolean flag to enable overwriting of the output file if it already exists.\n")
   ;
 
   private final String attribute;

http://git-wip-us.apache.org/repos/asf/orc/blob/22c6c5bf/java/core/src/java/org/apache/orc/OrcFile.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/OrcFile.java b/java/core/src/java/org/apache/orc/OrcFile.java
index 99f22dd..5375efe 100644
--- a/java/core/src/java/org/apache/orc/OrcFile.java
+++ b/java/core/src/java/org/apache/orc/OrcFile.java
@@ -384,10 +384,12 @@ public class OrcFile {
     private BloomFilterVersion bloomFilterVersion;
     private PhysicalWriter physicalWriter;
     private WriterVersion writerVersion = CURRENT_WRITER;
+    private boolean overwrite;
 
     protected WriterOptions(Properties tableProperties, Configuration conf) {
       configuration = conf;
       memoryManagerValue = getStaticMemoryManager(conf);
+      overwrite = OrcConf.OVERWRITE_OUTPUT_FILE.getBoolean(tableProperties, conf);
       stripeSizeValue = OrcConf.STRIPE_SIZE.getLong(tableProperties, conf);
       blockSizeValue = OrcConf.BLOCK_SIZE.getLong(tableProperties, conf);
       rowIndexStrideValue =
@@ -445,6 +447,15 @@ public class OrcFile {
     }
 
     /**
+     * If the output file already exists, should it be overwritten?
+     * If it is not provided, write operation will fail if the file already exists.
+     */
+    public WriterOptions overwrite(boolean value) {
+      overwrite = value;
+      return this;
+    }
+
+    /**
      * Set the stripe size for the file. The writer stores the contents of the
      * stripe in memory until this memory limit is reached and the stripe
      * is flushed to the HDFS file and the next stripe started.
@@ -631,6 +642,10 @@ public class OrcFile {
       return bloomFilterColumns;
     }
 
+    public boolean getOverwrite() {
+      return overwrite;
+    }
+
     public FileSystem getFileSystem() {
       return fileSystemValue;
     }

http://git-wip-us.apache.org/repos/asf/orc/blob/22c6c5bf/java/core/src/java/org/apache/orc/impl/PhysicalFsWriter.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/impl/PhysicalFsWriter.java b/java/core/src/java/org/apache/orc/impl/PhysicalFsWriter.java
index 1769182..06e762f 100644
--- a/java/core/src/java/org/apache/orc/impl/PhysicalFsWriter.java
+++ b/java/core/src/java/org/apache/orc/impl/PhysicalFsWriter.java
@@ -88,7 +88,7 @@ public class PhysicalFsWriter implements PhysicalWriter {
     LOG.info("ORC writer created for path: {} with stripeSize: {} blockSize: {}" +
         " compression: {} bufferSize: {}", path, defaultStripeSize, blockSize,
         compress, bufferSize);
-    rawWriter = fs.create(path, false, HDFS_BUFFER_SIZE,
+    rawWriter = fs.create(path, opts.getOverwrite(), HDFS_BUFFER_SIZE,
         fs.getDefaultReplication(path), blockSize);
     codec = OrcCodecPool.getCodec(compress);
     writer = new OutStream("metadata", bufferSize, codec,

http://git-wip-us.apache.org/repos/asf/orc/blob/22c6c5bf/java/core/src/test/org/apache/orc/impl/TestWriterImpl.java
----------------------------------------------------------------------
diff --git a/java/core/src/test/org/apache/orc/impl/TestWriterImpl.java b/java/core/src/test/org/apache/orc/impl/TestWriterImpl.java
new file mode 100644
index 0000000..22bb1d3
--- /dev/null
+++ b/java/core/src/test/org/apache/orc/impl/TestWriterImpl.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.orc.impl;
+
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.orc.OrcConf;
+import org.apache.orc.OrcFile;
+import org.apache.orc.TypeDescription;
+import org.apache.orc.Writer;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestWriterImpl {
+
+  Path workDir = new Path(System.getProperty("test.tmp.dir"));
+  Configuration conf;
+  FileSystem fs;
+  Path testFilePath;
+  TypeDescription schema;
+
+  @Before
+  public void openFileSystem() throws Exception {
+    conf = new Configuration();
+    fs = FileSystem.getLocal(conf);
+    fs.setWorkingDirectory(workDir);
+    testFilePath = new Path("testWriterImpl.orc");
+    fs.create(testFilePath, true);
+    schema = TypeDescription.fromString("struct<x:int,y:int>");
+  }
+
+  @After
+  public void deleteTestFile() throws Exception {
+    fs.delete(testFilePath, false);
+  }
+
+  @Test(expected = FileAlreadyExistsException.class)
+  public void testDefaultOverwriteFlagForWriter() throws Exception {
+    // default value of the overwrite flag is false, so this should fail
+    Writer w = OrcFile.createWriter(testFilePath, OrcFile.writerOptions(conf).setSchema(schema));
+    w.close();
+  }
+
+  @Test
+  public void testOverriddenOverwriteFlagForWriter() throws Exception {
+    // overriding the flag should result in a successful write (no exception)
+    conf.set(OrcConf.OVERWRITE_OUTPUT_FILE.getAttribute(), "true");
+    Writer w = OrcFile.createWriter(testFilePath, OrcFile.writerOptions(conf).setSchema(schema));
+    w.close();
+  }
+}


Mime
View raw message