db-derby-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bpendle...@apache.org
Subject svn commit: r1742057 - in /db/derby/code/trunk/java: build/org/apache/derbyBuild/lastgoodjarcontents/ engine/org/apache/derby/impl/load/ testing/org/apache/derbyTesting/functionTests/tests/largedata/
Date Tue, 03 May 2016 03:20:11 GMT
Author: bpendleton
Date: Tue May  3 03:20:11 2016
New Revision: 1742057

URL: http://svn.apache.org/viewvc?rev=1742057&view=rev
Log:
DERBY-6884: SYSCS_IMPORT_TABLE_LOBS_FROM_EXTFILE can't import lob data

This change modifies the ImportLobFile.getString() and
ImportReadData.initExternalLobFile() methods so that they use a Java "long"
variable for the offset into the external lob file; prior to this change
they were using a Java "int" variable and hence would malfunction when the
lob offsets exceeded Integer.MAX_VALUE ( 2,147,483,647 ).

The regression test which demonstrates these problems is a bit slow to run;
on my system, it takes approximately 15 minutes to execute, and requires
about 10 GB of available disk space during the test run. Therefore, the
test cases are placed in a new test program (Derby6884Test), which is not
listed in the "standard" system test suites, but rather is only added to the
"largedata" suite. The new test can also be run by itself, e.g.:

ant -Dderby.junit.testclass=org.apache.derbyTesting.functionTests.tests.largedata.Derby6884Test
junit-clean junit-single

Added:
    db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/largedata/Derby6884Test.java
  (with props)
Modified:
    db/derby/code/trunk/java/build/org/apache/derbyBuild/lastgoodjarcontents/insane.derbyTesting.jar.lastcontents
    db/derby/code/trunk/java/build/org/apache/derbyBuild/lastgoodjarcontents/sane.derbyTesting.jar.lastcontents
    db/derby/code/trunk/java/engine/org/apache/derby/impl/load/ImportLobFile.java
    db/derby/code/trunk/java/engine/org/apache/derby/impl/load/ImportReadData.java
    db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/largedata/_Suite.java

Modified: db/derby/code/trunk/java/build/org/apache/derbyBuild/lastgoodjarcontents/insane.derbyTesting.jar.lastcontents
URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/build/org/apache/derbyBuild/lastgoodjarcontents/insane.derbyTesting.jar.lastcontents?rev=1742057&r1=1742056&r2=1742057&view=diff
==============================================================================
--- db/derby/code/trunk/java/build/org/apache/derbyBuild/lastgoodjarcontents/insane.derbyTesting.jar.lastcontents
(original)
+++ db/derby/code/trunk/java/build/org/apache/derbyBuild/lastgoodjarcontents/insane.derbyTesting.jar.lastcontents
Tue May  3 03:20:11 2016
@@ -501,6 +501,7 @@ org.apache.derbyTesting.functionTests.te
 org.apache.derbyTesting.functionTests.tests.largedata.ClobImplT.class
 org.apache.derbyTesting.functionTests.tests.largedata.Derby5624Test.class
 org.apache.derbyTesting.functionTests.tests.largedata.Derby6317Test.class
+org.apache.derbyTesting.functionTests.tests.largedata.Derby6884Test.class
 org.apache.derbyTesting.functionTests.tests.largedata.LobLimitsClientTest.class
 org.apache.derbyTesting.functionTests.tests.largedata.LobLimitsLiteTest.class
 org.apache.derbyTesting.functionTests.tests.largedata.LobLimitsTest.class

Modified: db/derby/code/trunk/java/build/org/apache/derbyBuild/lastgoodjarcontents/sane.derbyTesting.jar.lastcontents
URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/build/org/apache/derbyBuild/lastgoodjarcontents/sane.derbyTesting.jar.lastcontents?rev=1742057&r1=1742056&r2=1742057&view=diff
==============================================================================
--- db/derby/code/trunk/java/build/org/apache/derbyBuild/lastgoodjarcontents/sane.derbyTesting.jar.lastcontents
(original)
+++ db/derby/code/trunk/java/build/org/apache/derbyBuild/lastgoodjarcontents/sane.derbyTesting.jar.lastcontents
Tue May  3 03:20:11 2016
@@ -501,6 +501,7 @@ org.apache.derbyTesting.functionTests.te
 org.apache.derbyTesting.functionTests.tests.largedata.ClobImplT.class
 org.apache.derbyTesting.functionTests.tests.largedata.Derby5624Test.class
 org.apache.derbyTesting.functionTests.tests.largedata.Derby6317Test.class
+org.apache.derbyTesting.functionTests.tests.largedata.Derby6884Test.class
 org.apache.derbyTesting.functionTests.tests.largedata.LobLimitsClientTest.class
 org.apache.derbyTesting.functionTests.tests.largedata.LobLimitsLiteTest.class
 org.apache.derbyTesting.functionTests.tests.largedata.LobLimitsTest.class

Modified: db/derby/code/trunk/java/engine/org/apache/derby/impl/load/ImportLobFile.java
URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/engine/org/apache/derby/impl/load/ImportLobFile.java?rev=1742057&r1=1742056&r2=1742057&view=diff
==============================================================================
--- db/derby/code/trunk/java/engine/org/apache/derby/impl/load/ImportLobFile.java (original)
+++ db/derby/code/trunk/java/engine/org/apache/derby/impl/load/ImportLobFile.java Tue May
 3 03:20:11 2016
@@ -128,10 +128,10 @@ class ImportLobFile
      * @param length  length of the the data.
      * @exception  IOException  on any I/O error.     
      */
-    public String getString(int offset, int length) throws IOException {
+    public String getString(long offset, int length) throws IOException {
         lobInputStream.seek(offset);
         lobLimitIn.clearLimit();
-        lobLimitIn.setLimit((int) length);
+        lobLimitIn.setLimit(length);
         
         // wrap a reader on top of the stream, so that calls 
         // to read the clob data from the file can read the 

Modified: db/derby/code/trunk/java/engine/org/apache/derby/impl/load/ImportReadData.java
URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/engine/org/apache/derby/impl/load/ImportReadData.java?rev=1742057&r1=1742056&r2=1742057&view=diff
==============================================================================
--- db/derby/code/trunk/java/engine/org/apache/derby/impl/load/ImportReadData.java (original)
+++ db/derby/code/trunk/java/engine/org/apache/derby/impl/load/ImportReadData.java Tue May
 3 03:20:11 2016
@@ -108,7 +108,7 @@ final class ImportReadData implements ja
   // variables realted to reading lob data from files.
   private ImportLobFile[] lobFileHandles; // lob file handle object 
   private String lobFileName; // current file name
-  private int lobOffset; // offset of the current large object
+  private long lobOffset; // offset of the current large object
   private int lobLength; //length of the current large object
 
 
@@ -1037,7 +1037,7 @@ final class ImportReadData implements ja
         lobLength = Integer.parseInt(lobLocationStr.substring(
                                      lengthIndex + 1, 
                                      lobLocationStr.length() -1));
-        lobOffset = Integer.parseInt(lobLocationStr.substring(
+        lobOffset = Long.parseLong(lobLocationStr.substring(
                                      offsetIndex+1, 
                                      lengthIndex));
         lobFileName = lobLocationStr.substring(0 , offsetIndex);

Added: db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/largedata/Derby6884Test.java
URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/largedata/Derby6884Test.java?rev=1742057&view=auto
==============================================================================
--- db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/largedata/Derby6884Test.java
(added)
+++ db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/largedata/Derby6884Test.java
Tue May  3 03:20:11 2016
@@ -0,0 +1,156 @@
+package org.apache.derbyTesting.functionTests.tests.largedata;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.sql.Statement;
+import junit.framework.Test;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.Reader;
+import org.apache.derbyTesting.functionTests.tests.tools.ImportExportBaseTest;
+import org.apache.derbyTesting.functionTests.util.streams.LoopingAlphabetReader;
+import org.apache.derbyTesting.functionTests.util.streams.LoopingAlphabetStream;
+import org.apache.derbyTesting.junit.BaseJDBCTestCase;
+import org.apache.derbyTesting.junit.BaseTestSuite;
+import org.apache.derbyTesting.junit.CleanDatabaseTestSetup;
+import org.apache.derbyTesting.junit.SupportFilesSetup;
+import org.apache.derbyTesting.junit.RuntimeStatisticsParser;
+import org.apache.derbyTesting.junit.SQLUtilities;
+
+
+/*
+Class org.apache.derbyTesting.functionTests.tests.largedata.Derby6884Test
+
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+*/
+
+
+public class Derby6884Test extends ImportExportBaseTest
+{
+    String fileName6884;
+    String lobFile6884;
+
+    public Derby6884Test(String name) 
+    {
+        super(name);
+        fileName6884 = (SupportFilesSetup.getReadWrite("table-data")).getPath();
+        lobFile6884 = (SupportFilesSetup.getReadWrite("lob-data")).getPath();
+    }
+
+    public static Test suite() 
+    {
+        BaseTestSuite suite = new BaseTestSuite("Derby6884Test");
+        suite.addTest(baseSuite("Derby6884Test:embedded"));
+        return suite;
+    }
+    
+    /*
+     * DERBY-6884: external file behaviors when the file contains more than
+     * Integer.MAX_VALUE bytes of data.
+     */
+    public void testDerby6884ImportLargeExtfileClob()
+        throws SQLException
+    {
+        Statement s  = createStatement();
+        PreparedStatement ps = prepareStatement(
+                     "insert into DERBY_6884_TESTCLOB values(? , ?)" );
+        int id = 1;
+        long byteCount = 0;
+        int clobSize = 0;
+        while ( byteCount < Integer.MAX_VALUE ) {
+            ps.setInt(1 , id++);
+            clobSize = ( 10000 * 1024 ) + ( 1024 * id );
+            byteCount += clobSize;
+            Reader reader = new LoopingAlphabetReader(clobSize);
+            ps.setCharacterStream(2, reader, clobSize);
+            ps.executeUpdate();
+        }
+        ps.setInt(1 , id++);
+        Reader reader = new LoopingAlphabetReader(clobSize);
+        ps.setCharacterStream(2, reader, clobSize);
+        ps.executeUpdate();
+
+        commit();
+        doExportTableLobsToExtFile("APP", "DERBY_6884_TESTCLOB", fileName6884, 
+                                   null, null , null, lobFile6884);
+        s.execute("TRUNCATE TABLE DERBY_6884_TESTCLOB");
+        doImportTableLobsFromExtFile("APP", "DERBY_6884_TESTCLOB",
+                                   fileName6884, null, null, null, 0);
+        SupportFilesSetup.deleteFile(fileName6884);
+        SupportFilesSetup.deleteFile(lobFile6884);
+    }
+
+    /*
+     * Same as the prior test, but with BLOB column, not CLOB column.
+     */
+    public void testDerby6884ImportLargeExtfileBlob()
+        throws SQLException
+    {
+        Statement s  = createStatement();
+        PreparedStatement ps = prepareStatement(
+                     "insert into DERBY_6884_TESTBLOB values(? , ?)" );
+        int id = 1;
+        long byteCount = 0;
+        int blobSize = 0;
+        while ( byteCount < Integer.MAX_VALUE ) {
+            ps.setInt(1 , id++);
+            blobSize = ( 50000 * 1024 ) + ( 1024 * id );
+            byteCount += blobSize;
+            InputStream stream = new LoopingAlphabetStream(blobSize);
+            ps.setBinaryStream(2, stream, blobSize);
+            ps.executeUpdate();
+        }
+        ps.setInt(1 , id++);
+        InputStream stream = new LoopingAlphabetStream(blobSize);
+        ps.setBinaryStream(2, stream, blobSize);
+        ps.executeUpdate();
+
+        commit();
+        doExportTableLobsToExtFile("APP", "DERBY_6884_TESTBLOB", fileName6884, 
+                                   null, null , null, lobFile6884);
+        s.execute("TRUNCATE TABLE DERBY_6884_TESTBLOB");
+        doImportTableLobsFromExtFile("APP", "DERBY_6884_TESTBLOB",
+                                   fileName6884, null, null, null, 0);
+        SupportFilesSetup.deleteFile(fileName6884);
+        SupportFilesSetup.deleteFile(lobFile6884);
+    }
+
+
+    protected static Test baseSuite(String name) 
+    {
+        BaseTestSuite suite = new BaseTestSuite(name);
+        suite.addTestSuite(Derby6884Test.class);
+        Test test = suite;
+        test = new SupportFilesSetup(test);
+        return new CleanDatabaseTestSetup(test)
+        {
+            protected void decorateSQL(Statement stmt) throws SQLException
+            {
+                Connection conn = stmt.getConnection();
+
+                stmt.execute("CREATE TABLE DERBY_6884_TESTBLOB" +
+                              "(id BIGINT, content BLOB)");
+                stmt.execute("CREATE TABLE DERBY_6884_TESTCLOB" +
+                              "(id BIGINT, content CLOB)");
+
+                conn.commit();
+            }
+        };
+    }
+}

Propchange: db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/largedata/Derby6884Test.java
------------------------------------------------------------------------------
    svn:eol-style = native

Modified: db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/largedata/_Suite.java
URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/largedata/_Suite.java?rev=1742057&r1=1742056&r2=1742057&view=diff
==============================================================================
--- db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/largedata/_Suite.java
(original)
+++ db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/largedata/_Suite.java
Tue May  3 03:20:11 2016
@@ -50,6 +50,7 @@ public class _Suite extends BaseJDBCTest
         suite.addTest(LobLimitsLiteTest.suite());
         suite.addTest(LobLimitsTest.suite());
         suite.addTest(LobLimitsClientTest.suite());
+        suite.addTest(Derby6884Test.suite());
         return suite;
     }
 }



Mime
View raw message