toree-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lrese...@apache.org
Subject [1/2] incubator-toree git commit: [TOREE-487][TOREE-488] Remove PySpark and SparkR interpreters
Date Fri, 05 Oct 2018 13:55:29 GMT
Repository: incubator-toree
Updated Branches:
  refs/heads/master 9ee665d68 -> 276165ae2


http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/276165ae/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRService.scala
----------------------------------------------------------------------
diff --git a/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRService.scala
b/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRService.scala
deleted file mode 100644
index 350aee0..0000000
--- a/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRService.scala
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- *  Licensed to the Apache Software Foundation (ASF) under one or more
- *  contributor license agreements.  See the NOTICE file distributed with
- *  this work for additional information regarding copyright ownership.
- *  The ASF licenses this file to You under the Apache License, Version 2.0
- *  (the "License"); you may not use this file except in compliance with
- *  the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License
- */
-package org.apache.toree.kernel.interpreter.sparkr
-
-import java.util.concurrent.{Semaphore, TimeUnit}
-
-import org.apache.toree.interpreter.broker.BrokerService
-import org.apache.toree.kernel.interpreter.sparkr.SparkRTypes.{Code, CodeResults}
-import org.slf4j.LoggerFactory
-
-import scala.concurrent.Future
-import scala.tools.nsc.interpreter._
-
-/**
- * Represents the service that provides the high-level interface between the
- * JVM and R.
- *
- * @param processName The name of the Rscript process to run.
- * @param rBackend The backend to start to communicate between the JVM and R
- * @param sparkRBridge The bridge to use for communication between the JVM and R
- * @param sparkRProcessHandler The handler used for events that occur with the
- *                             SparkR process
- */
-class SparkRService(
-  processName: String,
-  private val rBackend: ReflectiveRBackend,
-  private val sparkRBridge: SparkRBridge,
-  private val sparkRProcessHandler: SparkRProcessHandler
-) extends BrokerService {
-  private val logger = LoggerFactory.getLogger(this.getClass)
-  @volatile private var rBackendPort: Int = -1
-  @volatile private var _isRunning: Boolean = false
-  override def isRunning: Boolean = _isRunning
-
-  /** Represents the process used to execute R code via the bridge. */
-  private lazy val sparkRProcess: SparkRProcess = {
-    val p = new SparkRProcess(
-      processName,
-      sparkRBridge,
-      sparkRProcessHandler,
-      rBackendPort
-    )
-
-    // Update handlers to correctly reset and restart the process
-    sparkRProcessHandler.setResetMethod(message => {
-      p.stop()
-      sparkRBridge.state.reset(message)
-    })
-    sparkRProcessHandler.setRestartMethod(() => p.start())
-
-    p
-  }
-
-  /** Starts the SparkR service. */
-  override def start(): Unit = {
-    logger.debug("Initializing statically-accessible SparkR bridge")
-    SparkRBridge.sparkRBridge = sparkRBridge
-
-    val initialized = new Semaphore(0)
-    val classLoader = SparkRBridge.getClass.getClassLoader
-    import scala.concurrent.ExecutionContext.Implicits.global
-    val rBackendRun = Future {
-      logger.debug("Initializing RBackend")
-      rBackendPort = rBackend.init(classLoader)
-      logger.debug(s"RBackend running on port $rBackendPort")
-      initialized.release()
-      logger.debug("Running RBackend")
-      rBackend.run()
-      logger.debug("RBackend has finished")
-    }
-
-    // Wait for backend to start before starting R process to connect
-    val backendTimeout =
-      sys.env.getOrElse("SPARKR_BACKEND_TIMEOUT", "120").toInt
-    if (initialized.tryAcquire(backendTimeout, TimeUnit.SECONDS)) {
-      // Start the R process used to execute code
-      logger.debug("Launching process to execute R code")
-      sparkRProcess.start()
-      _isRunning = true
-    } else {
-      // Unable to initialize, so throw an exception
-      throw new SparkRException(
-        s"Unable to initialize R backend in $backendTimeout seconds!")
-    }
-  }
-
-  /**
-   * Submits code to the SparkR service to be executed and return a result.
-   *
-   * @param code The code to execute
-   *
-   * @return The result as a future to eventually return
-   */
-  override def submitCode(code: Code, kernelOutputStream: Option[OutputStream]): Future[CodeResults]
= {
-    sparkRBridge.state.pushCode(code, kernelOutputStream)
-  }
-
-  /** Stops the running SparkR service. */
-  override def stop(): Unit = {
-    // Stop the R process used to execute code
-    sparkRProcess.stop()
-
-    // Stop the server used as an entrypoint for R
-    rBackend.close()
-
-    // Clear the bridge
-    SparkRBridge.reset()
-
-    _isRunning = false
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/276165ae/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRState.scala
----------------------------------------------------------------------
diff --git a/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRState.scala
b/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRState.scala
deleted file mode 100644
index 2f7ddc7..0000000
--- a/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRState.scala
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- *  Licensed to the Apache Software Foundation (ASF) under one or more
- *  contributor license agreements.  See the NOTICE file distributed with
- *  this work for additional information regarding copyright ownership.
- *  The ASF licenses this file to You under the Apache License, Version 2.0
- *  (the "License"); you may not use this file except in compliance with
- *  the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License
- */
-package org.apache.toree.kernel.interpreter.sparkr
-
-import org.apache.toree.interpreter.broker.BrokerState
-
-/**
- * Represents the state structure of SparkR.
- *
- * @param maxQueuedCode The maximum amount of code to support being queued
- *                      at the same time for SparkR execution
- */
-class SparkRState(private val maxQueuedCode: Int)
-  extends BrokerState(maxQueuedCode)

http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/276165ae/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRTransformer.scala
----------------------------------------------------------------------
diff --git a/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRTransformer.scala
b/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRTransformer.scala
deleted file mode 100644
index c35e342..0000000
--- a/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRTransformer.scala
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- *  Licensed to the Apache Software Foundation (ASF) under one or more
- *  contributor license agreements.  See the NOTICE file distributed with
- *  this work for additional information regarding copyright ownership.
- *  The ASF licenses this file to You under the Apache License, Version 2.0
- *  (the "License"); you may not use this file except in compliance with
- *  the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License
- */
-package org.apache.toree.kernel.interpreter.sparkr
-
-import org.apache.toree.interpreter.broker.BrokerTransformer
-
-/**
- * Represents the transformer used by SparkR.
- */
-class SparkRTransformer extends BrokerTransformer

http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/276165ae/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRTypes.scala
----------------------------------------------------------------------
diff --git a/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRTypes.scala
b/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRTypes.scala
deleted file mode 100644
index 370704f..0000000
--- a/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/SparkRTypes.scala
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- *  Licensed to the Apache Software Foundation (ASF) under one or more
- *  contributor license agreements.  See the NOTICE file distributed with
- *  this work for additional information regarding copyright ownership.
- *  The ASF licenses this file to You under the Apache License, Version 2.0
- *  (the "License"); you may not use this file except in compliance with
- *  the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License
- */
-package org.apache.toree.kernel.interpreter.sparkr
-
-import org.apache.toree.interpreter.broker.BrokerTypesProvider
-
-/**
- * Represents all types associated with the SparkR interface.
- */
-object SparkRTypes extends BrokerTypesProvider

http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/276165ae/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/package.scala
----------------------------------------------------------------------
diff --git a/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/package.scala
b/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/package.scala
deleted file mode 100644
index e5d3709..0000000
--- a/sparkr-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/sparkr/package.scala
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- *  Licensed to the Apache Software Foundation (ASF) under one or more
- *  contributor license agreements.  See the NOTICE file distributed with
- *  this work for additional information regarding copyright ownership.
- *  The ASF licenses this file to You under the Apache License, Version 2.0
- *  (the "License"); you may not use this file except in compliance with
- *  the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License
- */
-package org.apache.toree.kernel.interpreter
-
-import org.apache.toree.interpreter.broker.{BrokerCode, BrokerPromise}
-
-/**
- * Contains aliases to broker types.
- */
-package object sparkr {
-  /**
-   * Represents a promise made regarding the completion of SparkR code
-   * execution.
-   */
-  type SparkRPromise = BrokerPromise
-
-  /**
-   * Represents a block of SparkR code to be evaluated.
-   */
-  type SparkRCode = BrokerCode
-}

http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/276165ae/sparkr-interpreter/src/main/scala/org/apache/toree/magic/builtin/SparkR.scala
----------------------------------------------------------------------
diff --git a/sparkr-interpreter/src/main/scala/org/apache/toree/magic/builtin/SparkR.scala
b/sparkr-interpreter/src/main/scala/org/apache/toree/magic/builtin/SparkR.scala
deleted file mode 100644
index 361ea20..0000000
--- a/sparkr-interpreter/src/main/scala/org/apache/toree/magic/builtin/SparkR.scala
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- *  Licensed to the Apache Software Foundation (ASF) under one or more
- *  contributor license agreements.  See the NOTICE file distributed with
- *  this work for additional information regarding copyright ownership.
- *  The ASF licenses this file to You under the Apache License, Version 2.0
- *  (the "License"); you may not use this file except in compliance with
- *  the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License
- */
-package org.apache.toree.magic.builtin
-
-import org.apache.toree.interpreter.{ExecuteError, ExecuteAborted}
-import org.apache.toree.kernel.interpreter.sparkr.{SparkRInterpreter, SparkRException}
-import org.apache.toree.magic.{MagicOutput, CellMagic}
-import org.apache.toree.magic.dependencies.IncludeKernel
-import org.apache.toree.plugins.annotations.Event
-
-/**
- * Represents the magic interface to use the SparkR interpreter.
- */
-class SparkR extends CellMagic with IncludeKernel {
-
-  @Event(name = "sparkr")
-  override def execute(code: String): MagicOutput = {
-    val sparkR = kernel.interpreter("SparkR")
-
-    if (sparkR.isEmpty || sparkR.get == null)
-      throw new SparkRException("SparkR is not available!")
-
-    sparkR.get match {
-      case sparkRInterpreter: SparkRInterpreter =>
-        val (_, output) = sparkRInterpreter.interpret(code)
-        output match {
-          case Left(executeOutput) =>
-            MagicOutput(executeOutput.toSeq:_*)
-          case Right(executeFailure) => executeFailure match {
-            case executeAborted: ExecuteAborted =>
-              throw new SparkRException("SparkR code was aborted!")
-            case executeError: ExecuteError =>
-              throw new SparkRException(executeError.value)
-          }
-        }
-      case otherInterpreter =>
-        val className = otherInterpreter.getClass.getName
-        throw new SparkRException(s"Invalid SparkR interpreter: $className")
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/276165ae/test_toree.py
----------------------------------------------------------------------
diff --git a/test_toree.py b/test_toree.py
index 82b515a..0856b9a 100644
--- a/test_toree.py
+++ b/test_toree.py
@@ -138,47 +138,5 @@ class ToreeScalaKernelTests(jupyter_kernel_test.KernelTests):
 
         return reply, output_msgs
 
-class ToreePythonKernelTests(jupyter_kernel_test.KernelTests):
-    # Required --------------------------------------
-
-    # The name identifying an installed kernel to run the tests against
-    kernel_name = "apache_toree_pyspark"
-
-    # language_info.name in a kernel_info_reply should match this
-    language_name = "python"
-
-    # Optional --------------------------------------
-
-    # Code in the kernel's language to write "hello, world" to stdout
-    # These tests fail randomly on travis.....
-    # code_hello_world = "print(\"hello, world\")"
-
-    # Samples of code which generate a result value (ie, some text
-    # displayed as Out[n])
-    code_execute_result = [
-        {'code': '6*7', 'result': '42'}
-    ]
-
-class ToreeSparkRKernelTests(jupyter_kernel_test.KernelTests):
-    # Required --------------------------------------
-
-    # The name identifying an installed kernel to run the tests against
-    kernel_name = "apache_toree_sparkr"
-
-    # language_info.name in a kernel_info_reply should match this
-    language_name = "R"
-
-    # Optional --------------------------------------
-
-    # Code in the kernel's language to write "hello, world" to stdout
-    # Something weird goes on with R.	
-    # code_hello_world = r'write("hello, world", stdout())' 
-
-    # Samples of code which generate a result value (ie, some text
-    # displayed as Out[n])
-    code_execute_result = [
-        {'code': '6*7', 'result': '[1] 42'}
-    ]
-
 if __name__ == '__main__':
     unittest.main()


Mime
View raw message