sqoop-commits mailing list archives

Repository: sqoop
Updated Branches:
  refs/heads/trunk 184fdefac -> d96f867bc


SQOOP-3243: Importing BLOB data causes 'Stream closed' error on encrypted HDFS

(Boglarka Egyed)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/d96f867b
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/d96f867b
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/d96f867b

Branch: refs/heads/trunk
Commit: d96f867bc9f5206e0d14be282501bd4311644687
Parents: 184fdef
Author: Boglarka Egyed <bogi@apache.org>
Authored: Mon Nov 6 22:20:09 2017 +0100
Committer: Boglarka Egyed <bogi@apache.org>
Committed: Mon Nov 6 22:20:09 2017 +0100

----------------------------------------------------------------------
 src/java/org/apache/sqoop/io/LobFile.java       |  5 +--
 src/test/com/cloudera/sqoop/io/TestLobFile.java | 43 ++++++++++++++++++++
 2 files changed, 44 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/d96f867b/src/java/org/apache/sqoop/io/LobFile.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/io/LobFile.java b/src/java/org/apache/sqoop/io/LobFile.java
index 4e1bf2c..4edde12 100644
--- a/src/java/org/apache/sqoop/io/LobFile.java
+++ b/src/java/org/apache/sqoop/io/LobFile.java
@@ -1665,10 +1665,7 @@ public final class LobFile {
         this.out = null;
       }
 
-      if (this.countingOut != null) {
-        this.countingOut.close();
-        this.countingOut = null;
-      }
+      this.countingOut = null;
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/sqoop/blob/d96f867b/src/test/com/cloudera/sqoop/io/TestLobFile.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/io/TestLobFile.java b/src/test/com/cloudera/sqoop/io/TestLobFile.java
index 029758c..ac461fd 100644
--- a/src/test/com/cloudera/sqoop/io/TestLobFile.java
+++ b/src/test/com/cloudera/sqoop/io/TestLobFile.java
@@ -25,11 +25,15 @@ import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.Reader;
 import java.io.Writer;
+import java.net.URI;
 import java.nio.CharBuffer;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.CryptoOutputStream;
+import org.apache.hadoop.crypto.JceAesCtrCryptoCodec;
+import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.junit.Before;
@@ -39,6 +43,12 @@ import org.junit.Test;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
+import static org.mockito.Answers.CALLS_REAL_METHODS;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.withSettings;
 
 import org.junit.Rule;
 import org.junit.rules.ExpectedException;
@@ -595,5 +605,38 @@ public class TestLobFile {
     runCompressedTest(CodecMap.LZO);
   }
 
+  @Test
+  public void testCryptoOutputStreamClosingDoesNotThrowExceptionAndClosedProperly() throws
Exception {
+    // Tests that closing CryptoOutputStream doesn't throw exception neither with Java 7
nor with Java 8
+    // For a detailed explanation see SQOOP-3243
+    CryptoOutputStream cryptoOutputStream = createCryptoOutputStream();
+    FSDataOutputStream wrappedCryptoOutputStream = new FSDataOutputStream(cryptoOutputStream,
null);
+
+    Path mockPath = spy(new Path("file://" + TEMP_BASE_DIR, "binary.lob"));
+    FileSystem mockFileSystem = mock(FileSystem.class, withSettings().defaultAnswer(CALLS_REAL_METHODS.get()));
+
+    doReturn(mockFileSystem).when(mockPath).getFileSystem(conf);
+    doReturn(null).when(mockFileSystem).getWorkingDirectory();
+    doReturn(wrappedCryptoOutputStream).when(mockFileSystem).create(mockPath);
+    doReturn(new URI("file:///")).when(mockFileSystem).getUri();
+
+    LobFile.Writer writer = LobFile.create(mockPath, conf);
+
+    writer.close();
+
+    verify(cryptoOutputStream).close();
+  }
+
+
+  public CryptoOutputStream createCryptoOutputStream() throws Exception {
+    final byte[] BYTES = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};
+    Path p = new Path(TEMP_BASE_DIR, "binary.lob");
+
+    FSDataOutputStream fsDataOutputStream = fs.create(p);
+    CryptoOutputStream cryptoOutputStream = spy(new CryptoOutputStream(fsDataOutputStream,
new JceAesCtrCryptoCodec(), 512, BYTES, BYTES));
+
+    return cryptoOutputStream;
+  }
+
 }