toree-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lrese...@apache.org
Subject [2/2] incubator-toree git commit: [TOREE-480] Add support for spark-context-initialization-mode none
Date Thu, 16 Aug 2018 03:25:09 GMT
[TOREE-480] Add support for spark-context-initialization-mode none

Deprecate nosparkcontext in favor of spark-context-initialization-mode none.
Note that old usage of nosparkcontext is automatically converted to use
spark-context-initialization-mode none.


Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/b8cca2cb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/b8cca2cb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/b8cca2cb

Branch: refs/heads/master
Commit: b8cca2cb55fc17359a6cb89477bb58bef0962196
Parents: bff1c6b
Author: Luciano Resende <lresende@apache.org>
Authored: Thu Aug 2 13:45:31 2018 -0700
Committer: Luciano Resende <lresende@apache.org>
Committed: Wed Aug 15 20:24:07 2018 -0700

----------------------------------------------------------------------
 etc/pip_install/toree/toreeapp.py                    |  2 +-
 .../org/apache/toree/boot/CommandLineOptions.scala   | 14 ++++++++------
 .../apache/toree/boot/CommandLineOptionsSpec.scala   | 15 ++++++++++++++-
 3 files changed, 23 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/b8cca2cb/etc/pip_install/toree/toreeapp.py
----------------------------------------------------------------------
diff --git a/etc/pip_install/toree/toreeapp.py b/etc/pip_install/toree/toreeapp.py
index 84dad4c..c4dc11d 100644
--- a/etc/pip_install/toree/toreeapp.py
+++ b/etc/pip_install/toree/toreeapp.py
@@ -49,7 +49,7 @@ class ToreeInstall(InstallKernelSpec):
     jupyter toree install --spark_home=/spark/home/dir
     jupyter toree install --spark_opts='--master=local[4]'
     jupyter toree install --kernel_name=toree_special
-    jupyter toree install --toree_opts='--nosparkcontext'
+    jupyter toree install --toree_opts='--spark-context-initialization-mode none'
     jupyter toree install --interpreters=PySpark,SQL
     jupyter toree install --python=python
     '''

http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/b8cca2cb/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala b/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala
index ae7f220..39376ae 100644
--- a/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala
+++ b/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala
@@ -93,14 +93,15 @@ class CommandLineOptions(args: Seq[String]) {
       .withRequiredArg().ofType(classOf[String])
 
   private val _nosparkcontext =
-    parser.accepts("nosparkcontext", "kernel should not create a spark context")
+    parser.accepts("nosparkcontext", "Deprecated (see spark-context-initialization-mode).
\n Kernel should not create a spark context")
 
   private val _spark_context_initialization_mode = parser.accepts(
     "spark-context-initialization-mode",
     "Identify how the Spark context initialization occurs. " +
+      "NONE will not initialize a Spark Context," +
       "EAGER initialization will happen during runtime initialization,  " +
-      "LAZY initialization will happen when the context is used for the first time ."
-  ).withRequiredArg().ofType(classOf[String]).withValuesConvertedBy( regex("(lazy)|(eager)")).defaultsTo("lazy")
+      "LAZY initialization will happen when the context is used for the first time."
+  ).withRequiredArg().ofType(classOf[String]).withValuesConvertedBy( regex("(none)|(lazy)|(eager)")).defaultsTo("lazy")
 
   private val _spark_context_initialization_timeout = parser.accepts(
     "spark-context-initialization-timeout",
@@ -171,9 +172,10 @@ class CommandLineOptions(args: Seq[String]) {
       "alternate_sigint" -> get(_alternate_sigint),
       "jar_dir" -> get(_jar_dir),
       "default_interpreter" -> get(_default_interpreter),
-      "nosparkcontext" -> (if (has(_nosparkcontext)) Some(true) else Some(false)),
-      "spark_context_initialization_mode" -> (if( has(_spark_context_initialization_mode))
-        get(_spark_context_initialization_mode) else Some("lazy")),
+      // deprecated in favor of spark-context-initialization-mode none
+      // "nosparkcontext" -> (if (has(_nosparkcontext)) Some(true) else Some(false)),
+      "spark_context_initialization_mode" -> (if( has(_spark_context_initialization_mode))
get(_spark_context_initialization_mode)
+        else ( if (has(_nosparkcontext)) Some("none") else Some("lazy"))),
       "spark_context_initialization_timeout" -> get(_spark_context_initialization_timeout),
       "interpreter_plugins" -> interpreterPlugins,
       "default_repositories" -> getAll(_default_repositories).map(_.asJava)

http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/b8cca2cb/kernel/src/test/scala/org/apache/toree/boot/CommandLineOptionsSpec.scala
----------------------------------------------------------------------
diff --git a/kernel/src/test/scala/org/apache/toree/boot/CommandLineOptionsSpec.scala b/kernel/src/test/scala/org/apache/toree/boot/CommandLineOptionsSpec.scala
index 029f46b..2ec7020 100644
--- a/kernel/src/test/scala/org/apache/toree/boot/CommandLineOptionsSpec.scala
+++ b/kernel/src/test/scala/org/apache/toree/boot/CommandLineOptionsSpec.scala
@@ -387,7 +387,20 @@ class CommandLineOptionsSpec extends FunSpec with Matchers {
         config.getString(key) should be("eager")
       }
 
-      it("when an invalid value is specified, an exception must be thrown") {
+      it("when the option nosparkcontext is specified, it should properly set spark-context-initialization-mode
to none") {
+          val options = new CommandLineOptions(List(
+            "--stdin-port", "99999",
+            "--shell-port", "88888",
+            "--iopub-port", "77777",
+            "--control-port", "55555",
+            "--heartbeat-port", "44444",
+            "--nosparkcontext", "foo"
+          ))
+          val config: Config = options.toConfig
+          config.getString(key) should be("none")
+        }
+
+        it("when an invalid value is specified, an exception must be thrown") {
         intercept [OptionException] {
           val options = new CommandLineOptions(List(
             "--stdin-port", "99999",


Mime
View raw message