drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From par...@apache.org
Subject [drill] 05/10: DRILL-6445: Fix existing test cases in TestScripts.java and add new test case for DRILLBIT_CONTEXT variable
Date Sat, 02 Jun 2018 04:50:13 GMT
This is an automated email from the ASF dual-hosted git repository.

parthc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git

commit f48894c7294b70d17ebd92d4c92cd5e60821beeb
Author: Sorabh Hamirwasia <shamirwasia@maprtech.com>
AuthorDate: Thu May 24 11:58:01 2018 -0700

    DRILL-6445: Fix existing test cases in TestScripts.java and add new test case for DRILLBIT_CONTEXT
variable
    
    This closes #1289
---
 .../org/apache/drill/yarn/scripts/ScriptUtils.java |  98 +++++++++++--
 .../org/apache/drill/yarn/scripts/TestScripts.java | 161 ++++++++++++++++-----
 2 files changed, 208 insertions(+), 51 deletions(-)

diff --git a/drill-yarn/src/test/java/org/apache/drill/yarn/scripts/ScriptUtils.java b/drill-yarn/src/test/java/org/apache/drill/yarn/scripts/ScriptUtils.java
index 3517cf8..8a909a5 100644
--- a/drill-yarn/src/test/java/org/apache/drill/yarn/scripts/ScriptUtils.java
+++ b/drill-yarn/src/test/java/org/apache/drill/yarn/scripts/ScriptUtils.java
@@ -38,6 +38,7 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.regex.Pattern;
@@ -66,7 +67,6 @@ public class ScriptUtils {
     String args[] = {
       "-Dlog.path=/.*/drill/log/sqlline\\.log",
       "-Dlog.query.path=/.*/drill/log/sqlline_queries\\.json",
-      "-XX:MaxPermSize=512M",
       "sqlline\\.SqlLine",
       "-d",
       "org\\.apache\\.drill\\.jdbc\\.Driver",
@@ -123,11 +123,10 @@ public class ScriptUtils {
       "-Xms4G",
       "-Xmx4G",
       "-XX:MaxDirectMemorySize=8G",
-      "-XX:MaxPermSize=512M",
       "-XX:ReservedCodeCacheSize=1G",
-      // Removed in Drill 1.8
-//      "-Ddrill\\.exec\\.enable-epoll=true",
-      "-XX:\\+CMSClassUnloadingEnabled",
+      "-Ddrill\\.exec\\.enable-epoll=false",
+      // Removed in Drill 1.14
+      //"-XX:\\+CMSClassUnloadingEnabled",
       "-XX:\\+UseG1GC",
       "org\\.apache\\.drill\\.exec\\.server\\.Drillbit",
       "-Dlog\\.path=/.*/script-test/drill/log/drillbit\\.log",
@@ -197,7 +196,8 @@ public class ScriptUtils {
       "sqlline",
       //sqlline.bat
       //submit_plan
-      "yarn-drillbit.sh"
+      "yarn-drillbit.sh",
+      "auto-setup.sh"
   };
 
   /**
@@ -286,24 +286,89 @@ public class ScriptUtils {
     }
   }
 
+  public void writeEnvFile(PrintWriter out, String key, String value, boolean overrideValue)
{
+    out.print("export ");
+    out.print(key);
+    out.print("=");
+
+    if (!overrideValue) {
+      out.print("${");
+      out.print(key);
+      out.print(":-");
+    }
+    out.print("\"");
+    out.print(value);
+    out.print("\"");
+
+    if (!overrideValue) {
+      out.print("}");
+    }
+    out.println();
+  }
+
   /**
    * Create a drill-env.sh or distrib-env.sh file with the given environment in
    * the recommended format.
+   * different formats based on overrideValue flag
+   *
+   * @param file - File instance to set environment variables in
+   * @param env - Environment to be placed inside File
+   * @param overrideValue - true - Set environment value such that it overrides previously
set value
+   *                      - false - Set environment value in recommended format.
    */
-
-  public void createEnvFile(File file, Map<String, String> env)
+  public void createEnvFile(File file, Map<String, String> env, boolean overrideValue)
       throws IOException {
     try (PrintWriter out = new PrintWriter(new FileWriter(file))) {
       out.println("#!/usr/bin/env bash");
       for (String key : env.keySet()) {
         String value = env.get(key);
-        out.print("export ");
-        out.print(key);
-        out.print("=${");
-        out.print(key);
-        out.print(":-\"");
-        out.print(value);
-        out.println("\"}");
+        writeEnvFile(out, key, value, overrideValue);
+      }
+    }
+  }
+
+  /**
+   * Creates a drill-env.sh or distrib-env.sh file with the given environment under
+   * a given condition. If size of env map is smaller than condition map then last
+   * env entry is repeated for rest of conditions.
+   *
+   * @param file - File instance to set environment and condition in
+   * @param condition - Conditions to guard environment variable
+   * @param env - Environment to be placed inside File
+   * @param overrideValue - true - Set environment value such that it overrides previously
set value
+   *                      - false - Set environment value in recommended format.
+   *
+   */
+  public void createEnvFileWithCondition(File file, Map<String, String> condition,
+                                         Map<String, String> env, boolean overrideValue)
throws IOException {
+    if (env.size() == 0 || condition.size() == 0) {
+      return;
+    }
+
+    final Iterator envIterator = env.entrySet().iterator();
+    Map.Entry currentEnv = (Map.Entry) envIterator.next();
+
+    try (PrintWriter out = new PrintWriter(new FileWriter(file))) {
+      out.println("#!/usr/bin/env bash");
+
+      for (String condKey : condition.keySet()) {
+        String condValue = condition.get(condKey);
+        out.print("if [ \"$");
+        out.print(condKey);
+        out.print("\" = \"");
+        out.print(condValue);
+        out.println("\" ]; then");
+
+        final String envKey = currentEnv.getKey().toString();
+        final String envValue = currentEnv.getValue().toString();
+        writeEnvFile(out, envKey, envValue, overrideValue);
+
+        out.println("fi");
+        out.println();
+
+        if (envIterator.hasNext()) {
+          currentEnv = (Map.Entry) envIterator.next();
+        }
       }
     }
   }
@@ -342,7 +407,8 @@ public class ScriptUtils {
    * Consume the input from a stream, specifically the stderr or stdout stream
    * from a process.
    *
-   * @see http://stackoverflow.com/questions/14165517/processbuilder-forwarding-stdout-and-stderr-of-started-processes-without-blocki
+   * @link http://stackoverflow.com/questions/14165517/processbuilder-forwarding-stdout-and-stderr-of-started-processes
+   * -without-blocki
    */
 
   private static class StreamGobbler extends Thread {
diff --git a/drill-yarn/src/test/java/org/apache/drill/yarn/scripts/TestScripts.java b/drill-yarn/src/test/java/org/apache/drill/yarn/scripts/TestScripts.java
index 5f6b5bb..38279f8 100644
--- a/drill-yarn/src/test/java/org/apache/drill/yarn/scripts/TestScripts.java
+++ b/drill-yarn/src/test/java/org/apache/drill/yarn/scripts/TestScripts.java
@@ -159,14 +159,6 @@ public class TestScripts {
       result.validateArgRegex("-Xloggc:.*/" + logTail);
     }
 
-    // Max Perm Size
-
-    {
-      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN)
-          .addEnv("DRILLBIT_MAX_PERM", "600M").run();
-      result.validateArg("-XX:MaxPermSize=600M");
-    }
-
     // Code cache size
 
     {
@@ -346,9 +338,8 @@ public class TestScripts {
     drillEnv.put("DRILL_HEAP", "5G");
     drillEnv.put("DRILL_MAX_DIRECT_MEMORY", "7G");
     drillEnv.put("SERVER_LOG_GC", "1");
-    drillEnv.put("DRILLBIT_MAX_PERM", "600M");
     drillEnv.put("DRILLBIT_CODE_CACHE_SIZE", "2G");
-    context.createEnvFile(new File(siteDir, fileName), drillEnv);
+    context.createEnvFile(new File(siteDir, fileName), drillEnv, false);
 
     {
       RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN).run();
@@ -358,8 +349,7 @@ public class TestScripts {
           propArg,
           "-Xms5G", "-Xmx5G",
           "-XX:MaxDirectMemorySize=7G",
-          "-XX:ReservedCodeCacheSize=2G",
-          "-XX:MaxPermSize=600M"
+          "-XX:ReservedCodeCacheSize=2G"
       };
 
       result.validateArgs(expectedArgs);
@@ -378,12 +368,50 @@ public class TestScripts {
           .run();
       assertEquals(0, result.returnCode);
       result.validateArg("-XX:MaxDirectMemorySize=9G");
-      result.validateArg("-XX:MaxPermSize=600M");
       String logTail = context.testDrillHome.getName() + "/log/drillbit.gc";
       assertFalse(result.containsArgRegex("-Xloggc:.*/" + logTail));
     }
   }
 
+  @Test
+  public void testDistribEnvWithNegativeCond() throws IOException {
+    // Construct condition map
+    final Map<String, String> conditions = new HashMap<>();
+    conditions.put("DRILLBIT_CONTEXT", "0");
+    final String expectedArgs[] = {"-XX:ReservedCodeCacheSize=1G"};
+    doEnvFileWithConditionTest("distrib-env.sh", conditions, expectedArgs);
+  }
+
+  @Test
+  public void testDistribEnvWithPositiveCond() throws IOException {
+    // Construct condition map
+    final Map<String, String> conditions = new HashMap<>();
+    conditions.put("DRILLBIT_CONTEXT", "1");
+    final String expectedArgs[] = {"-XX:ReservedCodeCacheSize=2G"};
+    doEnvFileWithConditionTest("distrib-env.sh", conditions, expectedArgs);
+  }
+
+  /**
+   * Implementation of the drill-env.sh or distrib-env.sh tests with conditions
+   * guarding environment variables.
+   */
+  private void doEnvFileWithConditionTest(String fileName, Map<String, String> conditions,
+                                          String[] expectedArgs) throws IOException {
+    context.createMockDistrib();
+    File siteDir = new File(context.testDrillHome, "conf");
+    context.createMockConf(siteDir);
+
+    // Set a property in the env file.
+    Map<String, String> drillEnv = new HashMap<>();
+    drillEnv.put("DRILLBIT_CODE_CACHE_SIZE", "2G");
+    context.createEnvFileWithCondition(new File(siteDir, fileName), conditions, drillEnv,
false);
+    {
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN).run();
+      assertEquals(0, result.returnCode);
+      result.validateArgs(expectedArgs);
+    }
+  }
+
   /**
    * Test that drill-env.sh overrides distrib-env.sh, and that the environment
    * overrides both. Assumes the basics were tested above.
@@ -400,13 +428,12 @@ public class TestScripts {
     Map<String, String> distribEnv = new HashMap<>();
     distribEnv.put("DRILL_HEAP", "5G");
     distribEnv.put("DRILL_MAX_DIRECT_MEMORY", "7G");
-    distribEnv.put("DRILLBIT_MAX_PERM", "600M");
-    context.createEnvFile(new File(siteDir, "distrib-env.sh"), distribEnv);
+    context.createEnvFile(new File(siteDir, "distrib-env.sh"), distribEnv, false);
 
     Map<String, String> drillEnv = new HashMap<>();
     drillEnv.put("DRILL_HEAP", "6G");
     drillEnv.put("DRILL_MAX_DIRECT_MEMORY", "9G");
-    context.createEnvFile(new File(siteDir, "drill-env.sh"), drillEnv);
+    context.createEnvFile(new File(siteDir, "drill-env.sh"), drillEnv, false);
 
     {
       RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN).run();
@@ -414,8 +441,7 @@ public class TestScripts {
       String expectedArgs[] = {
           "-Xms6G", "-Xmx6G",
           "-XX:MaxDirectMemorySize=9G",
-          "-XX:MaxPermSize=600M",
-          "-XX:ReservedCodeCacheSize=1G" // Default
+          "-XX:ReservedCodeCacheSize=1024m" // Default
       };
 
       result.validateArgs(expectedArgs);
@@ -428,8 +454,7 @@ public class TestScripts {
       String expectedArgs[] = {
           "-Xms6G", "-Xmx6G",
           "-XX:MaxDirectMemorySize=5G",
-          "-XX:MaxPermSize=600M",
-          "-XX:ReservedCodeCacheSize=1G" // Default
+          "-XX:ReservedCodeCacheSize=1024m" // Default
       };
 
       result.validateArgs(expectedArgs);
@@ -498,19 +523,17 @@ public class TestScripts {
     Map<String, String> distribEnv = new HashMap<>();
     distribEnv.put("DRILL_HEAP", "5G");
     distribEnv.put("DRILL_MAX_DIRECT_MEMORY", "7G");
-    distribEnv.put("DRILLBIT_MAX_PERM", "600M");
-    context.createEnvFile(new File(confDir, "distrib-env.sh"), distribEnv);
+    context.createEnvFile(new File(confDir, "distrib-env.sh"), distribEnv, false);
 
     Map<String, String> drillEnv = new HashMap<>();
     drillEnv.put("DRILL_HEAP", "6G");
     drillEnv.put("DRILL_MAX_DIRECT_MEMORY", "9G");
-    context.createEnvFile(new File(siteDir, "drill-env.sh"), drillEnv);
+    context.createEnvFile(new File(siteDir, "drill-env.sh"), drillEnv, false);
 
     String expectedArgs[] = {
         "-Xms6G", "-Xmx6G",
         "-XX:MaxDirectMemorySize=9G",
-        "-XX:MaxPermSize=600M",
-        "-XX:ReservedCodeCacheSize=1G" // Default
+        "-XX:ReservedCodeCacheSize=1024m" // Default
     };
 
     // Site set using argument
@@ -611,8 +634,7 @@ public class TestScripts {
 
     String prefix = "-Djava.library.path=";
     {
-      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN)
-          .run();
+      RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN).run();
       assertFalse(result.containsArgRegex(prefix + ".*"));
       assertNull(result.libPath);
     }
@@ -874,7 +896,7 @@ public class TestScripts {
     File pidDir = context.createDir(new File(context.testDir, "pid"));
     Map<String, String> drillEnv = new HashMap<>();
     drillEnv.put("DRILL_PID_DIR", pidDir.getAbsolutePath());
-    context.createEnvFile(new File(siteDir, "drill-env.sh"), drillEnv);
+    context.createEnvFile(new File(siteDir, "drill-env.sh"), drillEnv, false);
 
     {
       RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_START)
@@ -905,7 +927,7 @@ public class TestScripts {
 
     Map<String, String> drillEnv = new HashMap<>();
     drillEnv.put("DRILL_MAX_DIRECT_MEMORY", "9G");
-    context.createEnvFile(new File(siteDir, "drill-env.sh"), drillEnv);
+    context.createEnvFile(new File(siteDir, "drill-env.sh"), drillEnv, false);
 
     // Use the -site (--config) option.
 
@@ -948,7 +970,7 @@ public class TestScripts {
     context.removeDir(new File(context.testDrillHome, "log"));
     Map<String, String> drillEnv = new HashMap<>();
     drillEnv.put("DRILL_LOG_DIR", logsDir.getAbsolutePath());
-    context.createEnvFile(new File(siteDir, "drill-env.sh"), drillEnv);
+    context.createEnvFile(new File(siteDir, "drill-env.sh"), drillEnv, false);
 
     {
       DrillbitRun runner = new DrillbitRun(DrillbitRun.DRILLBIT_START);
@@ -1122,7 +1144,6 @@ public class TestScripts {
       drillEnv.put("DRILL_HEAP", "5G");
       drillEnv.put("DRILL_MAX_DIRECT_MEMORY", "7G");
       drillEnv.put("SERVER_LOG_GC", "1");
-      drillEnv.put("DRILLBIT_MAX_PERM", "600M");
       drillEnv.put("DRILLBIT_CODE_CACHE_SIZE", "2G");
       RunResult result = new ScriptRunner("sqlline")
           .withEnvironment(drillEnv)
@@ -1138,11 +1159,9 @@ public class TestScripts {
 
       Map<String, String> shellEnv = new HashMap<>();
       shellEnv.put("CLIENT_GC_OPTS", "-XX:+UseG1GC");
-      shellEnv.put("SQLLINE_JAVA_OPTS", "-XX:MaxPermSize=256M");
       RunResult result = new ScriptRunner("sqlline")
           .withEnvironment(shellEnv)
           .run();
-      assertTrue(result.containsArg("-XX:MaxPermSize=256M"));
       assertTrue(result.containsArg("-XX:+UseG1GC"));
     }
     {
@@ -1156,7 +1175,6 @@ public class TestScripts {
       drillEnv.put("DRILL_HEAP", "5G");
       drillEnv.put("DRILL_MAX_DIRECT_MEMORY", "7G");
       drillEnv.put("SERVER_LOG_GC", "1");
-      drillEnv.put("DRILLBIT_MAX_PERM", "600M");
       drillEnv.put("DRILLBIT_CODE_CACHE_SIZE", "2G");
       drillEnv.put("DRILL_EMBEDDED", "1");
       RunResult result = new ScriptRunner("sqlline")
@@ -1168,7 +1186,6 @@ public class TestScripts {
           "-Xms5G", "-Xmx5G",
           "-XX:MaxDirectMemorySize=7G",
           "-XX:ReservedCodeCacheSize=2G",
-          "-XX:MaxPermSize=600M"
       };
 
       result.validateArgs(expectedArgs);
@@ -1177,6 +1194,80 @@ public class TestScripts {
   }
 
   /**
+   * Test to verify no effect of DRILLBIT_CONTEXT for Sqlline.
+   * @throws IOException
+   */
+  @Test
+  public void testSqllineWithDrillbitContextEnv() throws IOException {
+    context.createMockDistrib();
+    File siteDir = new File(context.testDrillHome, "conf");
+    context.createMockConf(siteDir);
+
+    // Test when SQLLINE_JAVA_OPTS is overriden inside a condition for
+    // DRILLBIT_CONTEXT = 0, then there is no effect
+    {
+      // Create a condition variable to be placed in distrib-env.sh
+      Map<String, String> conditions = new HashMap<>();
+      conditions.put("DRILLBIT_CONTEXT", "0");
+
+      // Create environment variable to be placed inside a condition in distrib-env.sh
+      Map<String, String> drillEnv = new HashMap<>();
+      drillEnv.put("SQLLINE_JAVA_OPTS", "-XX:MaxPermSize=256M");
+
+      // Create the environment variable file overriding SQLLINE_JAVA_OPTS
+      context.createEnvFileWithCondition(new File(siteDir, "distrib-env.sh"), conditions,
drillEnv, true);
+
+      // Expected value of the property
+      String expectedArgs[] = {"-XX:MaxPermSize=256M"};
+
+      // Run the test and match the output with expectedArgs
+      RunResult result = new ScriptRunner("sqlline").run();
+      assertEquals(0, result.returnCode);
+      result.validateJava();
+      result.validateClassPath(ScriptUtils.stdCp);
+      // Since by default MaxPermSize is not set anymore for Sqlline. It's removed in 1.13
+      assertFalse(result.containsArgsRegex(expectedArgs));
+    }
+
+    // Test when SQLLINE_JAVA_OPTS is overriden inside a condition for
+    // DRILLBIT_CONTEXT = 1, then there is no effect
+    {
+      Map<String, String> conditions = new HashMap<>();
+      conditions.put("DRILLBIT_CONTEXT", "1");
+
+      Map<String, String> drillEnv = new HashMap<>();
+      drillEnv.put("SQLLINE_JAVA_OPTS", "-XX:MaxPermSize=256M");
+      String expectedArgs[] = {"-XX:MaxPermSize=256M"};
+
+      // Create the environment variable file overriding SQLLINE_JAVA_OPTS
+      context.createEnvFileWithCondition(new File(siteDir, "distrib-env.sh"), conditions,
drillEnv, true);
+      RunResult result = new ScriptRunner("sqlline").run();
+      assertEquals(0, result.returnCode);
+      result.validateJava();
+      result.validateClassPath(ScriptUtils.stdCp);
+      // Since by default MaxPermSize is not set anymore for Sqlline. It's removed in 1.13
+      assertFalse(result.containsArgsRegex(expectedArgs));
+    }
+
+    // Test when SQLLINE_JAVA_OPTS is overriden without condition for
+    // DRILLBIT_CONTEXT then the environment variable is updated
+    {
+      Map<String, String> drillEnv = new HashMap<>();
+      drillEnv.put("SQLLINE_JAVA_OPTS", "-XX:MaxPermSize=256M");
+
+      // Create the environment variable file overriding SQLLINE_JAVA_OPTS without any condition
+      // around it.
+      String expectedArgs[] = {"-XX:MaxPermSize=256M"};
+      context.createEnvFile(new File(siteDir, "distrib-env.sh"), drillEnv, true);
+      RunResult result = new ScriptRunner("sqlline").run();
+      assertEquals(0, result.returnCode);
+      result.validateJava();
+      result.validateClassPath(ScriptUtils.stdCp);
+      assertTrue(result.containsArgsRegex(expectedArgs));
+    }
+  }
+
+  /**
    * Verify that the sqlline client works with the --site option by customizing
    * items in the site directory.
    *

-- 
To stop receiving notification emails like this one, please contact
parthc@apache.org.

Mime
View raw message