oozie-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ry...@apache.org
Subject git commit: OOZIE-1958 address duplication of env variables in oozie.launcher.yarn.app.mapreduce.am.env when running with uber mode (ryota)
Date Mon, 11 Aug 2014 19:34:50 GMT
Repository: oozie
Updated Branches:
  refs/heads/branch-4.1 f4fdc0e9c -> f94b01b07


OOZIE-1958 address duplication of env variables in oozie.launcher.yarn.app.mapreduce.am.env
when running with uber mode (ryota)


Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/f94b01b0
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/f94b01b0
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/f94b01b0

Branch: refs/heads/branch-4.1
Commit: f94b01b07e2e4962d4a7097406a594b75096a977
Parents: f4fdc0e
Author: egashira <ryota.egashira@yahoo.com>
Authored: Mon Aug 11 12:33:43 2014 -0700
Committer: egashira <ryota.egashira@yahoo.com>
Committed: Mon Aug 11 12:33:43 2014 -0700

----------------------------------------------------------------------
 .../oozie/action/hadoop/JavaActionExecutor.java | 142 +++++++++++++------
 .../action/hadoop/TestJavaActionExecutor.java   |  69 +++++++++
 release-log.txt                                 |   1 +
 3 files changed, 171 insertions(+), 41 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/oozie/blob/f94b01b0/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
index 94b55cf..7f4d473 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
@@ -93,7 +93,7 @@ public class JavaActionExecutor extends ActionExecutor {
     public final static String MAX_EXTERNAL_STATS_SIZE = "oozie.external.stats.max.size";
     public static final String ACL_VIEW_JOB = "mapreduce.job.acl-view-job";
     public static final String ACL_MODIFY_JOB = "mapreduce.job.acl-modify-job";
-    private static final String HADOOP_YARN_UBER_MODE = "mapreduce.job.ubertask.enable";
+    public static final String HADOOP_YARN_UBER_MODE = "mapreduce.job.ubertask.enable";
     public static final String HADOOP_MAP_MEMORY_MB = "mapreduce.map.memory.mb";
     public static final String HADOOP_CHILD_JAVA_OPTS = "mapred.child.java.opts";
     public static final String HADOOP_MAP_JAVA_OPTS = "mapreduce.map.java.opts";
@@ -270,56 +270,116 @@ public class JavaActionExecutor extends ActionExecutor {
 
     void updateConfForUberMode(Configuration launcherConf) {
 
-        // memory.mb
-        int launcherMapMemoryMB = launcherConf.getInt(HADOOP_MAP_MEMORY_MB, 1536);
-        int amMemoryMB = launcherConf.getInt(YARN_AM_RESOURCE_MB, 1536);
-        // YARN_MEMORY_MB_MIN to provide buffer.
-        // suppose launcher map aggressively use high memory, need some headroom for AM
-        int memoryMB = Math.max(launcherMapMemoryMB, amMemoryMB) + YARN_MEMORY_MB_MIN;
-        // limit to 4096 in case of 32 bit
-        if(launcherMapMemoryMB < 4096 && amMemoryMB < 4096 && memoryMB
> 4096){
-            memoryMB = 4096;
-        }
-        launcherConf.setInt(YARN_AM_RESOURCE_MB, memoryMB);
-
-        // We already made mapred.child.java.opts and mapreduce.map.java.opts equal, so just
start with one of them
-        String launcherMapOpts = launcherConf.get(HADOOP_MAP_JAVA_OPTS, "");
-        String amChildOpts = launcherConf.get(YARN_AM_COMMAND_OPTS);
-        StringBuilder optsStr = new StringBuilder();
-        int heapSizeForMap = extractHeapSizeMB(launcherMapOpts);
-        int heapSizeForAm = extractHeapSizeMB(amChildOpts);
-        int heapSize = Math.max(heapSizeForMap, heapSizeForAm) + YARN_MEMORY_MB_MIN;
-        // limit to 3584 in case of 32 bit
-        if(heapSizeForMap < 4096 && heapSizeForAm < 4096 && heapSize
> 3584) {
-            heapSize = 3584;
-        }
-        if (amChildOpts != null) {
-            optsStr.append(amChildOpts);
-        }
-        optsStr.append(" ").append(launcherMapOpts.trim());
-        if (heapSize > 0) {
-            // append calculated total heap size to the end
-            optsStr.append(" ").append("-Xmx").append(heapSize).append("m");
-        }
-        launcherConf.set(YARN_AM_COMMAND_OPTS, optsStr.toString().trim());
-
         // child.env
+        boolean hasConflictEnv = false;
         String launcherMapEnv = launcherConf.get(HADOOP_MAP_JAVA_ENV);
         if (launcherMapEnv == null) {
             launcherMapEnv = launcherConf.get(HADOOP_CHILD_JAVA_ENV);
         }
-        String envForAm = launcherConf.get(YARN_AM_ENV);
+        String amEnv = launcherConf.get(YARN_AM_ENV);
         StringBuffer envStr = new StringBuffer();
-        if (envForAm != null) {
-            envStr.append(envForAm);
+        HashMap<String, List<String>> amEnvMap = null;
+        HashMap<String, List<String>> launcherMapEnvMap = null;
+        if (amEnv != null) {
+            envStr.append(amEnv);
+            amEnvMap = populateEnvMap(amEnv);
         }
         if (launcherMapEnv != null) {
-            if (envForAm != null) {
-                envStr.append(",");
+            launcherMapEnvMap = populateEnvMap(launcherMapEnv);
+            if (amEnvMap != null) {
+                Iterator<String> envKeyItr = launcherMapEnvMap.keySet().iterator();
+                while (envKeyItr.hasNext()) {
+                    String envKey = envKeyItr.next();
+                    if (amEnvMap.containsKey(envKey)) {
+                        List<String> amValList = amEnvMap.get(envKey);
+                        List<String> launcherValList = launcherMapEnvMap.get(envKey);
+                        Iterator<String> valItr = launcherValList.iterator();
+                        while (valItr.hasNext()) {
+                            String val = valItr.next();
+                            if (!amValList.contains(val)) {
+                                hasConflictEnv = true;
+                                break;
+                            }
+                            else {
+                                valItr.remove();
+                            }
+                        }
+                        if (launcherValList.isEmpty()) {
+                            envKeyItr.remove();
+                        }
+                    }
+                }
+            }
+        }
+        if (hasConflictEnv) {
+            launcherConf.setBoolean(HADOOP_YARN_UBER_MODE, false);
+        }
+        else {
+            if (launcherMapEnvMap != null) {
+                for (String key : launcherMapEnvMap.keySet()) {
+                    List<String> launcherValList = launcherMapEnvMap.get(key);
+                    for (String val : launcherValList) {
+                        if (envStr.length() > 0) {
+                            envStr.append(",");
+                        }
+                        envStr.append(key).append("=").append(val);
+                    }
+                }
+            }
+
+            launcherConf.set(YARN_AM_ENV, envStr.toString());
+
+            // memory.mb
+            int launcherMapMemoryMB = launcherConf.getInt(HADOOP_MAP_MEMORY_MB, 1536);
+            int amMemoryMB = launcherConf.getInt(YARN_AM_RESOURCE_MB, 1536);
+            // YARN_MEMORY_MB_MIN to provide buffer.
+            // suppose launcher map aggressively use high memory, need some
+            // headroom for AM
+            int memoryMB = Math.max(launcherMapMemoryMB, amMemoryMB) + YARN_MEMORY_MB_MIN;
+            // limit to 4096 in case of 32 bit
+            if (launcherMapMemoryMB < 4096 && amMemoryMB < 4096 &&
memoryMB > 4096) {
+                memoryMB = 4096;
+            }
+            launcherConf.setInt(YARN_AM_RESOURCE_MB, memoryMB);
+
+            // We already made mapred.child.java.opts and
+            // mapreduce.map.java.opts equal, so just start with one of them
+            String launcherMapOpts = launcherConf.get(HADOOP_MAP_JAVA_OPTS, "");
+            String amChildOpts = launcherConf.get(YARN_AM_COMMAND_OPTS);
+            StringBuilder optsStr = new StringBuilder();
+            int heapSizeForMap = extractHeapSizeMB(launcherMapOpts);
+            int heapSizeForAm = extractHeapSizeMB(amChildOpts);
+            int heapSize = Math.max(heapSizeForMap, heapSizeForAm) + YARN_MEMORY_MB_MIN;
+            // limit to 3584 in case of 32 bit
+            if (heapSizeForMap < 4096 && heapSizeForAm < 4096 && heapSize
> 3584) {
+                heapSize = 3584;
+            }
+            if (amChildOpts != null) {
+                optsStr.append(amChildOpts);
+            }
+            optsStr.append(" ").append(launcherMapOpts.trim());
+            if (heapSize > 0) {
+                // append calculated total heap size to the end
+                optsStr.append(" ").append("-Xmx").append(heapSize).append("m");
+            }
+            launcherConf.set(YARN_AM_COMMAND_OPTS, optsStr.toString().trim());
+        }
+    }
+
+    private HashMap<String, List<String>> populateEnvMap(String input) {
+        HashMap<String, List<String>> envMaps = new HashMap<String, List<String>>();
+        String[] envEntries = input.split(",");
+        for (String envEntry : envEntries) {
+            String[] envKeyVal = envEntry.split("=");
+            String envKey = envKeyVal[0].trim();
+            List<String> valList = envMaps.get(envKey);
+            if (valList == null) {
+                valList = new ArrayList<String>();
             }
-            envStr.append(launcherMapEnv);
+            valList.add(envKeyVal[1].trim());
+            envMaps.put(envKey, valList);
         }
-        launcherConf.set(YARN_AM_ENV, envStr.toString());
+        return envMaps;
     }
 
     public int extractHeapSizeMB(String input) {

http://git-wip-us.apache.org/repos/asf/oozie/blob/f94b01b0/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
index 72a137c..17c5fd9 100644
--- a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
@@ -1788,6 +1788,75 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase
{
         assertEquals("A=foo,B=bar", launcherConf.get(JavaActionExecutor.YARN_AM_ENV));
     }
 
+    public void testUpdateConfForUberModeWithEnvDup() throws Exception {
+
+        Services.get().getConf().setBoolean("oozie.action.launcher.mapreduce.job.ubertask.enable",
true);
+
+        Element actionXml1 = XmlUtils.parseXml("<java>" + "<job-tracker>" + getJobTrackerUri()
+ "</job-tracker>"
+                + "<name-node>" + getNameNodeUri() + "</name-node>" + "<configuration>"
+                + "<property><name>oozie.launcher.yarn.app.mapreduce.am.env</name>"
+                + "<value>JAVA_HOME=/home/blah/java/jdk64/current,A=foo,B=bar</value></property>"
+                + "<property><name>oozie.launcher.mapreduce.map.env</name>"
+                + "<value>JAVA_HOME=/home/blah/java/jdk64/latest,C=blah</value></property>"
+ "</configuration>"
+                + "<main-class>MAIN-CLASS</main-class>" + "</java>");
+        JavaActionExecutor ae = new JavaActionExecutor();
+        XConfiguration protoConf = new XConfiguration();
+        protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
+
+        WorkflowJobBean wf = createBaseWorkflow(protoConf, "action");
+        WorkflowActionBean action = (WorkflowActionBean) wf.getActions().get(0);
+        action.setType(ae.getType());
+
+        Context context = new Context(wf, action);
+        JobConf launcherConf = new JobConf();
+        launcherConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml1,
launcherConf);
+
+        // uber mode should be disabled since JAVA_HOME points to different paths in am.evn
and map.env
+        assertEquals("false", launcherConf.get(JavaActionExecutor.HADOOP_YARN_UBER_MODE));
+
+        // testing complicated env setting case
+        Element actionXml2 = XmlUtils.parseXml("<java>" + "<job-tracker>" + getJobTrackerUri()
+ "</job-tracker>"
+                + "<name-node>" + getNameNodeUri() + "</name-node>" + "<configuration>"
+ "<property>"
+                + "<name>oozie.launcher.yarn.app.mapreduce.am.env</name>"
+                + "<value>LD_LIBRARY_PATH=$HADOOP_HOME_1/lib/native/`$JAVA_HOME/bin/java
-d32 -version;"
+                + "if [ $? -eq 0 ]; then echo Linux-i386-32; else echo Linux-amd64-64;fi`</value></property>"
+                + "<property>" + "<name>oozie.launcher.mapreduce.map.env</name>"
+                + "<value>LD_LIBRARY_PATH=$HADOOP_HOME_2/lib/native/`$JAVA_HOME/bin/java
-d32 -version;"
+                + "if [ $? -eq 0 ]; then echo Linux-i386-32; else echo Linux-amd64-64;fi`</value></property>"
+                + "</configuration>" + "<main-class>MAIN-CLASS</main-class>"
+ "</java>");
+
+        launcherConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml2,
launcherConf);
+
+        // uber mode should be disabled since LD_LIBRARY_PATH is different in am.evn and
map.env
+        assertEquals("false", launcherConf.get(JavaActionExecutor.HADOOP_YARN_UBER_MODE));
+
+        Element actionXml3 = XmlUtils
+                .parseXml("<java>"
+                        + "<job-tracker>"
+                        + getJobTrackerUri()
+                        + "</job-tracker>"
+                        + "<name-node>"
+                        + getNameNodeUri()
+                        + "</name-node>"
+                        + "<configuration>"
+                        + "<property><name>oozie.launcher.yarn.app.mapreduce.am.env</name>"
+                        + "<value>JAVA_HOME=/home/blah/java/jdk64/current,PATH=A,PATH=B</value></property>"
+                        + "<property><name>oozie.launcher.mapreduce.map.env</name>"
+                        + "<value>JAVA_HOME=/home/blah/java/jdk64/current,PATH=A</value></property>"
+                        + "</configuration>" + "<main-class>MAIN-CLASS</main-class>"
+ "</java>");
+
+        launcherConf = ae.createBaseHadoopConf(context, actionXml3);
+        launcherConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml3,
launcherConf);
+
+        // uber mode should be enabled since JAVA_HOME is the same, and PATH doesn't conflict
+        assertEquals("true", launcherConf.get(JavaActionExecutor.HADOOP_YARN_UBER_MODE));
+
+        // JAVA_HOME, PATH=A duplication is removed
+        String a = launcherConf.get(JavaActionExecutor.YARN_AM_ENV);
+        assertEquals("JAVA_HOME=/home/blah/java/jdk64/current,PATH=A,PATH=B",
+                launcherConf.get(JavaActionExecutor.YARN_AM_ENV));
+    }
+
     public void testUpdateConfForUberModeForJavaOpts() throws Exception {
         Services.get().getConf().setBoolean("oozie.action.launcher.mapreduce.job.ubertask.enable",
true);
 

http://git-wip-us.apache.org/repos/asf/oozie/blob/f94b01b0/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index a6032c6..b2c59d7 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
 -- Oozie 4.1.0 release (4.1 - unreleased)
 
+OOZIE-1958 address duplication of env variables in oozie.launcher.yarn.app.mapreduce.am.env
when running with uber mode (ryota)
 OOZIE-1920 Capture Output for SSH Action doesn't work (Richard Williams via rkanter)
 OOZIE-1961 Remove requireJavaVersion from enforcer rules (lars_francke via rkanter)
 OOZIE-1883 hostnameFilter has invalid url-pattern (dvillegas via rkanter)


Mime
View raw message