sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jar...@apache.org
Subject sqoop git commit: SQOOP-1125: Out of memory errors when number of records to import < 0.5 * splitSize
Date Wed, 19 Nov 2014 23:15:40 GMT
Repository: sqoop
Updated Branches:
  refs/heads/trunk 8704ff260 -> 2b4e4d9bf


SQOOP-1125: Out of memory errors when number of records to import < 0.5 * splitSize

(Sai Karthik Ganguru via Jarek Jarcec Cecho)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/2b4e4d9b
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/2b4e4d9b
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/2b4e4d9b

Branch: refs/heads/trunk
Commit: 2b4e4d9bf04e1b5686dbe54c467a816bf4a11a3e
Parents: 8704ff2
Author: Jarek Jarcec Cecho <jarcec@apache.org>
Authored: Wed Nov 19 15:15:05 2014 -0800
Committer: Jarek Jarcec Cecho <jarcec@apache.org>
Committed: Wed Nov 19 15:15:05 2014 -0800

----------------------------------------------------------------------
 .../sqoop/mapreduce/db/BigDecimalSplitter.java  |  2 +-
 .../mapreduce/db/TestBigDecimalSplitter.java    | 59 ++++++++++++++++++++
 2 files changed, 60 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/2b4e4d9b/src/java/org/apache/sqoop/mapreduce/db/BigDecimalSplitter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/mapreduce/db/BigDecimalSplitter.java b/src/java/org/apache/sqoop/mapreduce/db/BigDecimalSplitter.java
index 7e3048a..ebe6c40 100644
--- a/src/java/org/apache/sqoop/mapreduce/db/BigDecimalSplitter.java
+++ b/src/java/org/apache/sqoop/mapreduce/db/BigDecimalSplitter.java
@@ -102,7 +102,7 @@ public class BigDecimalSplitter implements DBSplitter  {
     try {
       return numerator.divide(denominator);
     } catch (ArithmeticException ae) {
-      return numerator.divide(denominator, BigDecimal.ROUND_HALF_UP);
+      return numerator.divide(denominator, BigDecimal.ROUND_UP);
     }
   }
 

http://git-wip-us.apache.org/repos/asf/sqoop/blob/2b4e4d9b/src/test/org/apache/sqoop/mapreduce/db/TestBigDecimalSplitter.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/mapreduce/db/TestBigDecimalSplitter.java b/src/test/org/apache/sqoop/mapreduce/db/TestBigDecimalSplitter.java
new file mode 100644
index 0000000..1e557a5
--- /dev/null
+++ b/src/test/org/apache/sqoop/mapreduce/db/TestBigDecimalSplitter.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.mapreduce.db;
+
+import java.math.BigDecimal;
+
+import junit.framework.TestCase;
+
+import com.cloudera.sqoop.mapreduce.db.BigDecimalSplitter;
+
+public class TestBigDecimalSplitter extends TestCase {
+
+  /* Test if the decimal split sizes are generated as expected */
+  public void testDecimalTryDivide() {
+    BigDecimal numerator = new BigDecimal("2");
+    BigDecimal denominator = new BigDecimal("4");
+	BigDecimal expected = new BigDecimal("0.5");
+	BigDecimalSplitter splitter = new BigDecimalSplitter();
+	BigDecimal out = splitter.tryDivide(numerator, denominator);
+	assertEquals(expected, out);
+  }
+
+  /* Test if the integer split sizes are generated as expected */
+  public void testIntegerTryDivide() {
+	BigDecimal numerator = new BigDecimal("99");
+	BigDecimal denominator = new BigDecimal("3");
+	BigDecimal expected = new BigDecimal("33");
+	BigDecimalSplitter splitter = new BigDecimalSplitter();
+	BigDecimal out = splitter.tryDivide(numerator, denominator);
+	assertEquals(expected, out);
+  }
+
+  /* Test if the recurring decimal split sizes are generated as expected */
+  public void testRecurringTryDivide() {
+	BigDecimal numerator = new BigDecimal("1");
+	BigDecimal denominator = new BigDecimal("3");
+	BigDecimal expected = new BigDecimal("1");
+	BigDecimalSplitter splitter = new BigDecimalSplitter();
+	BigDecimal out = splitter.tryDivide(numerator, denominator);
+	assertEquals(expected, out);
+  }
+
+}


Mime
View raw message