orc-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject orc git commit: ORC-228: Make MemoryManagerImpl.ROWS_BETWEEN_CHECKS configurable
Date Tue, 15 Aug 2017 17:53:46 GMT
Repository: orc
Updated Branches:
  refs/heads/master 4fc646725 -> b1d1fa132


ORC-228: Make MemoryManagerImpl.ROWS_BETWEEN_CHECKS configurable

Fixes #154

Signed-off-by: Owen O'Malley <omalley@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/orc/repo
Commit: http://git-wip-us.apache.org/repos/asf/orc/commit/b1d1fa13
Tree: http://git-wip-us.apache.org/repos/asf/orc/tree/b1d1fa13
Diff: http://git-wip-us.apache.org/repos/asf/orc/diff/b1d1fa13

Branch: refs/heads/master
Commit: b1d1fa1321b09aefe09b30a39c37fb12b985cc57
Parents: 4fc6467
Author: Eugene Koifman <ekoifman@hortonworks.com>
Authored: Fri Aug 11 16:58:10 2017 -0700
Committer: Owen O'Malley <omalley@apache.org>
Committed: Tue Aug 15 10:51:51 2017 -0700

----------------------------------------------------------------------
 java/core/src/java/org/apache/orc/OrcConf.java               | 6 +++++-
 .../core/src/java/org/apache/orc/impl/MemoryManagerImpl.java | 8 +++++++-
 .../core/src/test/org/apache/orc/impl/TestMemoryManager.java | 7 +++++++
 3 files changed, 19 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/orc/blob/b1d1fa13/java/core/src/java/org/apache/orc/OrcConf.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/OrcConf.java b/java/core/src/java/org/apache/orc/OrcConf.java
index 5eb2aa3..10d4228 100644
--- a/java/core/src/java/org/apache/orc/OrcConf.java
+++ b/java/core/src/java/org/apache/orc/OrcConf.java
@@ -142,7 +142,11 @@ public enum OrcConf {
       "orc.force.positional.evolution", false,
       "Require schema evolution to match the top level columns using position\n" +
       "rather than column names. This provides backwards compatibility with\n" +
-      "Hive 2.1.")
+      "Hive 2.1."),
+  ROWS_BETWEEN_CHECKS("orc.rows.between.memory.checks", "orc.rows.between.memory.checks",
5000,
+    "How often should MemoryManager check the memory sizes? Measured in rows\n" +
+      "added to all of the writers.  Valid range is [1,10000] and is primarily meant for"
+
+      "n\testing.  Setting this too low may negatively affect performance.")
   ;
 
   private final String attribute;

http://git-wip-us.apache.org/repos/asf/orc/blob/b1d1fa13/java/core/src/java/org/apache/orc/impl/MemoryManagerImpl.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/impl/MemoryManagerImpl.java b/java/core/src/java/org/apache/orc/impl/MemoryManagerImpl.java
index bd6c152..7f03ea9 100644
--- a/java/core/src/java/org/apache/orc/impl/MemoryManagerImpl.java
+++ b/java/core/src/java/org/apache/orc/impl/MemoryManagerImpl.java
@@ -49,7 +49,7 @@ public class MemoryManagerImpl implements MemoryManager {
    * How often should we check the memory sizes? Measured in rows added
    * to all of the writers.
    */
-  private static final int ROWS_BETWEEN_CHECKS = 5000;
+  final long ROWS_BETWEEN_CHECKS;
   private final long totalMemoryPool;
   private final Map<Path, WriterInfo> writerList =
       new HashMap<Path, WriterInfo>();
@@ -81,6 +81,12 @@ public class MemoryManagerImpl implements MemoryManager {
    */
   public MemoryManagerImpl(Configuration conf) {
     double maxLoad = OrcConf.MEMORY_POOL.getDouble(conf);
+    ROWS_BETWEEN_CHECKS = OrcConf.ROWS_BETWEEN_CHECKS.getLong(conf);
+    LOG.info(OrcConf.ROWS_BETWEEN_CHECKS.getAttribute() + "=" + ROWS_BETWEEN_CHECKS);
+    if(ROWS_BETWEEN_CHECKS < 1 || ROWS_BETWEEN_CHECKS > 10000) {
+      throw new IllegalArgumentException(OrcConf.ROWS_BETWEEN_CHECKS.getAttribute() + "="
+        + ROWS_BETWEEN_CHECKS + " is outside valid range [1,10000].");
+    }
     totalMemoryPool = Math.round(ManagementFactory.getMemoryMXBean().
         getHeapMemoryUsage().getMax() * maxLoad);
     ownerLock.lock();

http://git-wip-us.apache.org/repos/asf/orc/blob/b1d1fa13/java/core/src/test/org/apache/orc/impl/TestMemoryManager.java
----------------------------------------------------------------------
diff --git a/java/core/src/test/org/apache/orc/impl/TestMemoryManager.java b/java/core/src/test/org/apache/orc/impl/TestMemoryManager.java
index b84e6ed..109b95e 100644
--- a/java/core/src/test/org/apache/orc/impl/TestMemoryManager.java
+++ b/java/core/src/test/org/apache/orc/impl/TestMemoryManager.java
@@ -20,6 +20,7 @@ package org.apache.orc.impl;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.orc.MemoryManager;
+import org.apache.orc.OrcConf;
 import org.hamcrest.BaseMatcher;
 import org.hamcrest.Description;
 import org.junit.Test;
@@ -78,12 +79,18 @@ public class TestMemoryManager {
     Configuration conf = new Configuration();
     conf.set("hive.exec.orc.memory.pool", "0.9");
     MemoryManagerImpl mgr = new MemoryManagerImpl(conf);
+    assertEquals("Wrong default ",
+      OrcConf.ROWS_BETWEEN_CHECKS.getLong(conf), mgr.ROWS_BETWEEN_CHECKS);
     long mem =
         ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax();
     System.err.print("Memory = " + mem);
     long pool = mgr.getTotalMemoryPool();
     assertTrue("Pool too small: " + pool, mem * 0.899 < pool);
     assertTrue("Pool too big: " + pool, pool < mem * 0.901);
+
+    conf.setLong(OrcConf.ROWS_BETWEEN_CHECKS.getAttribute(), 1234);
+    mgr = new MemoryManagerImpl(conf);
+    assertEquals("Wrong default ", 1234, mgr.ROWS_BETWEEN_CHECKS);
   }
 
   private static class DoubleMatcher extends BaseMatcher<Double> {


Mime
View raw message