fluo-notifications mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From GitBox <...@apache.org>
Subject [GitHub] mikewalch closed pull request #15: fixes apache/fluo#671 made reserved mem configurable
Date Thu, 01 Jan 1970 00:00:00 GMT
mikewalch closed pull request #15: fixes apache/fluo#671 made reserved mem configurable
URL: https://github.com/apache/fluo-yarn/pull/15
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnEnv.java b/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnEnv.java
index c5a8629..15e77b0 100644
--- a/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnEnv.java
+++ b/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnEnv.java
@@ -116,6 +116,16 @@ public int getWorkerMaxMemory() {
         FluoYarnProperties.WORKER_MAX_MEMORY_MB_DEFAULT));
   }
 
+  public String getWorkerReservedMemory() {
+    String rmem = props.getProperty(FluoYarnProperties.WORKER_RESERVED_MEMORY_MB_PROP, null);
+    if (rmem == null) {
+      return rmem;
+    }
+
+    Preconditions.checkArgument(Integer.parseInt(rmem) > 0);
+    return rmem;
+  }
+
   public int getOracleCores() {
     return Integer.valueOf(props.getProperty(FluoYarnProperties.ORACLE_NUM_CORES_PROP,
         FluoYarnProperties.ORACLE_NUM_CORES_DEFAULT));
@@ -131,6 +141,16 @@ public int getOracleMaxMemory() {
         FluoYarnProperties.ORACLE_MAX_MEMORY_MB_DEFAULT));
   }
 
+  public String getOracleReservedMemory() {
+    String rmem = props.getProperty(FluoYarnProperties.ORACLE_RESERVED_MEMORY_MB_PROP, null);
+    if (rmem == null) {
+      return rmem;
+    }
+
+    Preconditions.checkArgument(Integer.parseInt(rmem) > 0);
+    return rmem;
+  }
+
   public String getYarnQueue() {
     return props.getProperty(FluoYarnProperties.QUEUE_PROP, null);
   }
diff --git a/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnLauncher.java b/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnLauncher.java
index e79f9dc..056604f 100644
--- a/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnLauncher.java
+++ b/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnLauncher.java
@@ -17,7 +17,9 @@
 
 import java.io.File;
 import java.util.Collection;
+import java.util.Collections;
 
+import org.apache.twill.api.Configs;
 import org.apache.twill.api.ResourceReport;
 import org.apache.twill.api.ResourceSpecification;
 import org.apache.twill.api.TwillApplication;
@@ -129,6 +131,16 @@ public static void main(String[] args) throws Exception {
       preparer.setSchedulerQueue(env.getYarnQueue());
     }
 
+    if (env.getWorkerReservedMemory() != null) {
+      preparer.withConfiguration(WORKER_ID, Collections
+          .singletonMap(Configs.Keys.JAVA_RESERVED_MEMORY_MB, env.getWorkerReservedMemory()));
+    }
+
+    if (env.getOracleReservedMemory() != null) {
+      preparer.withConfiguration(ORACLE_ID, Collections
+          .singletonMap(Configs.Keys.JAVA_RESERVED_MEMORY_MB, env.getOracleReservedMemory()));
+    }
+
     TwillController controller = preparer.start();
 
     ResourceReport report = controller.getResourceReport();
diff --git a/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnProperties.java b/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnProperties.java
index 990afa7..c707928 100644
--- a/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnProperties.java
+++ b/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnProperties.java
@@ -35,6 +35,8 @@
   // Worker properties
   public static final String WORKER_INSTANCES_PROP = YARN_PREFIX + ".worker.instances";
   public static final String WORKER_MAX_MEMORY_MB_PROP = YARN_PREFIX + ".worker.max.memory.mb";
+  public static final String WORKER_RESERVED_MEMORY_MB_PROP =
+      YARN_PREFIX + ".worker.reserved.memory.mb";
   public static final String WORKER_NUM_CORES_PROP = YARN_PREFIX + ".worker.num.cores";
   public static final String WORKER_INSTANCES_DEFAULT = "1";
   public static final String WORKER_MAX_MEMORY_MB_DEFAULT = "1024";
@@ -43,6 +45,8 @@
   // Oracle properties
   public static final String ORACLE_INSTANCES_PROP = YARN_PREFIX + ".oracle.instances";
   public static final String ORACLE_MAX_MEMORY_MB_PROP = YARN_PREFIX + ".oracle.max.memory.mb";
+  public static final String ORACLE_RESERVED_MEMORY_MB_PROP =
+      YARN_PREFIX + ".oracle.reserved.memory.mb";
   public static final String ORACLE_NUM_CORES_PROP = YARN_PREFIX + ".oracle.num.cores";
   public static final String ORACLE_INSTANCES_DEFAULT = "1";
   public static final String ORACLE_MAX_MEMORY_MB_DEFAULT = "512";
diff --git a/distribution/conf/fluo-yarn.properties b/distribution/conf/fluo-yarn.properties
index c2f3546..f2b1705 100644
--- a/distribution/conf/fluo-yarn.properties
+++ b/distribution/conf/fluo-yarn.properties
@@ -43,6 +43,9 @@
 ## Max memory of Oracle yarn containers (in MB)
 #fluo.yarn.oracle.max.memory.mb=512
 
+## Reserved memory.  Behaves same as worker property, so see its docs.
+#fluo.yarn.oracle.reserved.memory.mb=200
+
 ## Number of Oracle virtual cores
 #fluo.yarn.oracle.num.cores=1
 
@@ -51,12 +54,14 @@
 ## Number of Worker instances
 #fluo.yarn.worker.instances=1
 
-## Max memory of worker YARN containers (in MB). If YARN is killing worker processes consider
-## increasing twill.java.reserved.memory.mb (which defaults to 200 and is set in yarn-site.xml).
-## The twill.java.reserved.memory.mb config determines the gap between the YARN memory limit
set
-## below and the java -Xmx setting.  For example, if max memory is 1024 and twill reserved
memory
-## is 200, the java -Xmx setting will be 1024-200 = 824 MB.
+## Max memory of worker YARN containers (in MB).
 #fluo.yarn.worker.max.memory.mb=1024
 
+## This property is used to compute the Java heap size by subtracting it from the max memory
+## property.  For example, if max memory is 1024 and reserved memory is 200, then java -Xmx
setting
+## will be 1024-200 = 824 MB. If YARN is killing workers because too much memory is used,
 then
+## consider increasing this property. 
+#fluo.yarn.worker.reserved.memory.mb=200
+
 ## Number of worker virtual cores
 #fluo.yarn.worker.num.cores=1
diff --git a/distribution/lib/fetch.sh b/distribution/lib/fetch.sh
index 0881187..2c351f0 100755
--- a/distribution/lib/fetch.sh
+++ b/distribution/lib/fetch.sh
@@ -17,7 +17,7 @@ lib_dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
 maven_prefix=https://repo1.maven.org/maven2
 
 function download {
-  IFS=':' read -ra DEP <<< "$1" 
+  IFS=':' read -ra DEP <<< "$1"
   dir=$lib_dir/
   if [ -n "$2" ]; then
     dir=$lib_dir/$2
@@ -35,7 +35,7 @@ function download {
 
   if [ -f $dir/$fn ]; then
     echo "SUCCESS: Dependency exists - $dir/$fn"
-  else 
+  else
     wget -q $download_url -P $dir
     if [ $? == 0 ]; then
       echo "SUCCESS: Dependency downloaded from $download_url"
@@ -57,14 +57,14 @@ download com.yammer.metrics:metrics-annotation:jar:2.2.0
 download com.yammer.metrics:metrics-core:jar:2.2.0
 download net.sf.jopt-simple:jopt-simple:jar:3.2
 download org.apache.kafka:kafka_2.10:jar:0.8.0
-download org.apache.twill:twill-api:jar:0.11.0
-download org.apache.twill:twill-common:jar:0.11.0
-download org.apache.twill:twill-core:jar:0.11.0
-download org.apache.twill:twill-discovery-api:jar:0.11.0
-download org.apache.twill:twill-discovery-core:jar:0.11.0
-download org.apache.twill:twill-ext:jar:0.11.0
-download org.apache.twill:twill-yarn:jar:0.11.0
-download org.apache.twill:twill-zookeeper:jar:0.11.0
+download org.apache.twill:twill-api:jar:0.12.0
+download org.apache.twill:twill-common:jar:0.12.0
+download org.apache.twill:twill-core:jar:0.12.0
+download org.apache.twill:twill-discovery-api:jar:0.12.0
+download org.apache.twill:twill-discovery-core:jar:0.12.0
+download org.apache.twill:twill-ext:jar:0.12.0
+download org.apache.twill:twill-yarn:jar:0.12.0
+download org.apache.twill:twill-zookeeper:jar:0.12.0
 download org.ow2.asm:asm-all:jar:5.0.2
 download org.scala-lang:scala-compiler:jar:2.10.1
 download org.scala-lang:scala-library:jar:2.10.1
diff --git a/pom.xml b/pom.xml
index 00cda0d..ec732cb 100644
--- a/pom.xml
+++ b/pom.xml
@@ -42,7 +42,7 @@
     <hadoop.version>2.6.3</hadoop.version>
     <logback.version>1.1.3</logback.version>
     <slf4j.version>1.7.12</slf4j.version>
-    <twill.version>0.11.0</twill.version>
+    <twill.version>0.12.0</twill.version>
   </properties>
   <dependencyManagement>
     <dependencies>


 

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services

Mime
View raw message