toree-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lbust...@apache.org
Subject incubator-toree git commit: Removing --spark-configuration cmd arg
Date Wed, 30 Mar 2016 16:39:41 GMT
Repository: incubator-toree
Updated Branches:
  refs/heads/master 2aa8ce12b -> 7129ce222


Removing --spark-configuration cmd arg


Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/7129ce22
Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/7129ce22
Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/7129ce22

Branch: refs/heads/master
Commit: 7129ce222521141b747f0a490e8228be5f417f93
Parents: 2aa8ce1
Author: Gino Bustelo <lbustelo@apache.org>
Authored: Wed Mar 30 11:19:12 2016 -0500
Committer: Gino Bustelo <lbustelo@apache.org>
Committed: Wed Mar 30 11:19:12 2016 -0500

----------------------------------------------------------------------
 Vagrantfile                                     |  3 +-
 .../apache/toree/boot/CommandLineOptions.scala  |  8 ----
 .../org/apache/toree/kernel/api/Kernel.scala    |  7 ----
 .../toree/boot/CommandLineOptionsSpec.scala     | 44 --------------------
 .../apache/toree/kernel/api/KernelSpec.scala    |  1 -
 resources/compile/reference.conf                |  3 --
 resources/test/reference.conf                   |  3 --
 7 files changed, 1 insertion(+), 68 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/7129ce22/Vagrantfile
----------------------------------------------------------------------
diff --git a/Vagrantfile b/Vagrantfile
index 499194f..2f9d804 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -130,9 +130,8 @@ cat << EOF > /home/vagrant/.ipython/kernels/toree-kernel/kernel.json
     ],
     "codemirror_mode": "scala",
     "env": {
-        "SPARK_OPTS": "--driver-java-options=-Xms1024M --driver-java-options=-Xmx4096M --driver-java-options=-Dlog4j.logLevel=trace",
+        "SPARK_OPTS": "--conf=spark.cores.max=4 --driver-java-options='-Xms1024M -Xmx4096M
-Dlog4j.logLevel=trace'",
         "MAX_INTERPRETER_THREADS": "16",
-        "SPARK_CONFIGURATION": "spark.cores.max=4",
         "CAPTURE_STANDARD_OUT": "true",
         "CAPTURE_STANDARD_ERR": "true",
         "SEND_EMPTY_OUTPUT": "false",

http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/7129ce22/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala b/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala
index 68d2458..da3ef49 100644
--- a/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala
+++ b/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala
@@ -67,11 +67,6 @@ class CommandLineOptions(args: Seq[String]) {
     "heartbeat-port", "port of the heartbeat socket"
   ).withRequiredArg().ofType(classOf[Int])
 
-  private val _spark_configuration = parser.acceptsAll(
-    Seq("spark-configuration", "S").asJava,
-    "configuration setting for Apache Spark"
-  ).withRequiredArg().ofType(classOf[KeyValuePair])
-
   private val _magic_url =
     parser.accepts("magic-url", "path to a magic jar")
       .withRequiredArg().ofType(classOf[String])
@@ -145,9 +140,6 @@ class CommandLineOptions(args: Seq[String]) {
       "interpreter_args" -> interpreterArgs,
       "magic_urls" -> getAll(_magic_url).map(_.asJava)
         .flatMap(list => if (list.isEmpty) None else Some(list)),
-      "spark_configuration" -> getAll(_spark_configuration)
-        .map(list => KeyValuePairUtils.keyValuePairSeqToString(list))
-        .flatMap(str => if (str.nonEmpty) Some(str) else None),
       "max_interpreter_threads" -> get(_max_interpreter_threads),
       "jar_dir" -> get(_jar_dir),
       "default_interpreter" -> get(_default_interpreter),

http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/7129ce22/kernel/src/main/scala/org/apache/toree/kernel/api/Kernel.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/kernel/api/Kernel.scala b/kernel/src/main/scala/org/apache/toree/kernel/api/Kernel.scala
index 2d67189..bf7f56a 100644
--- a/kernel/src/main/scala/org/apache/toree/kernel/api/Kernel.scala
+++ b/kernel/src/main/scala/org/apache/toree/kernel/api/Kernel.scala
@@ -382,13 +382,6 @@ class Kernel (
     logger.info("Setting deployMode to client")
     conf.set("spark.submit.deployMode", "client")
 
-    KeyValuePairUtils.stringToKeyValuePairSeq(
-      _config.getString("spark_configuration")
-    ).foreach { keyValuePair =>
-      logger.info(s"Setting ${keyValuePair.key} to ${keyValuePair.value}")
-      Try(conf.set(keyValuePair.key, keyValuePair.value))
-    }
-
     // TODO: Move SparkIMain to private and insert in a different way
     logger.warn("Locked to Scala interpreter with SparkIMain until decoupled!")
 

http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/7129ce22/kernel/src/test/scala/org/apache/toree/boot/CommandLineOptionsSpec.scala
----------------------------------------------------------------------
diff --git a/kernel/src/test/scala/org/apache/toree/boot/CommandLineOptionsSpec.scala b/kernel/src/test/scala/org/apache/toree/boot/CommandLineOptionsSpec.scala
index cda360a..f88dd7d 100644
--- a/kernel/src/test/scala/org/apache/toree/boot/CommandLineOptionsSpec.scala
+++ b/kernel/src/test/scala/org/apache/toree/boot/CommandLineOptionsSpec.scala
@@ -89,50 +89,6 @@ class CommandLineOptionsSpec extends FunSpec with Matchers {
       }
     }
 
-    describe("when received --spark-conf=<key>=<value>") {
-      it("should add the key-value pair to the string representation") {
-        val expected = "key=value"
-        val options = new CommandLineOptions(s"--spark-conf=$expected" :: Nil)
-        val actual = options.toConfig.getString("spark_configuration")
-
-        actual should be (expected)
-      }
-
-      it("should append to existing command line key-value pairs") {
-        val expected = "key1=value1" :: "key2=value2" :: Nil
-        val options = new CommandLineOptions(
-          s"--spark-conf=${expected(0)}" ::
-          s"--spark-conf=${expected(1)}" ::
-          Nil
-        )
-        val actual = options.toConfig.getString("spark_configuration")
-
-        actual should be (expected.mkString(","))
-      }
-    }
-
-    describe("when received -S<key>=<value>") {
-      it("should add the key-value pair to the string representation") {
-        val expected = "key=value"
-        val options = new CommandLineOptions(s"-S$expected" :: Nil)
-        val actual = options.toConfig.getString("spark_configuration")
-
-        actual should be(expected)
-      }
-
-      it("should append to existing command line key-value pairs") {
-        val expected = "key1=value1" :: "key2=value2" :: Nil
-        val options = new CommandLineOptions(
-          s"-S${expected(0)}" ::
-          s"-S${expected(1)}" ::
-          Nil
-        )
-        val actual = options.toConfig.getString("spark_configuration")
-
-        actual should be (expected.mkString(","))
-      }
-    }
-
     describe("when received --profile=<path>") {
       it("should error if path is not set") {
         intercept[OptionException] {

http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/7129ce22/kernel/src/test/scala/org/apache/toree/kernel/api/KernelSpec.scala
----------------------------------------------------------------------
diff --git a/kernel/src/test/scala/org/apache/toree/kernel/api/KernelSpec.scala b/kernel/src/test/scala/org/apache/toree/kernel/api/KernelSpec.scala
index f8ae1eb..5f43c07 100644
--- a/kernel/src/test/scala/org/apache/toree/kernel/api/KernelSpec.scala
+++ b/kernel/src/test/scala/org/apache/toree/kernel/api/KernelSpec.scala
@@ -197,7 +197,6 @@ class KernelSpec extends FunSpec with Matchers with MockitoSugar
       it("should create SparkConf") {
         val expected = "some value"
         doReturn(expected).when(mockConfig).getString("spark.master")
-        doReturn("").when(mockConfig).getString("spark_configuration")
 
         // Provide stub for interpreter classServerURI since also executed
         doReturn("").when(mockInterpreter).classServerURI

http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/7129ce22/resources/compile/reference.conf
----------------------------------------------------------------------
diff --git a/resources/compile/reference.conf b/resources/compile/reference.conf
index 5857133..18efe52 100644
--- a/resources/compile/reference.conf
+++ b/resources/compile/reference.conf
@@ -45,9 +45,6 @@ interpreter_args = []
 magic_urls = []
 magic_urls = [${?MAGIC_URLS}]
 
-spark_configuration = ""
-spark_configuration = ${?SPARK_CONFIGURATION}
-
 max_interpreter_threads = 4
 max_interpreter_threads = ${?MAX_INTERPRETER_THREADS}
 

http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/7129ce22/resources/test/reference.conf
----------------------------------------------------------------------
diff --git a/resources/test/reference.conf b/resources/test/reference.conf
index 806a977..461d4e8 100644
--- a/resources/test/reference.conf
+++ b/resources/test/reference.conf
@@ -46,9 +46,6 @@ interpreter_args = []
 magic_urls = []
 magic_urls = [${?MAGIC_URLS}]
 
-spark_configuration = ""
-spark_configuration = ${?SPARK_CONFIGURATION}
-
 max_interpreter_threads = 4
 max_interpreter_threads = ${?MAX_INTERPRETER_THREADS}
 


Mime
View raw message