carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [carbondata] branch master updated: [HOTFIX] Remove spark dependency in Processing module
Date Tue, 03 Mar 2020 02:10:28 GMT
This is an automated email from the ASF dual-hosted git repository.

jackylk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git


The following commit(s) were added to refs/heads/master by this push:
     new e40912c  [HOTFIX] Remove spark dependency in Processing module
e40912c is described below

commit e40912caa4aad58d71f1e1195c68a27927ea9a57
Author: Indhumathi27 <indhumathim27@gmail.com>
AuthorDate: Mon Mar 2 13:54:20 2020 +0530

    [HOTFIX] Remove spark dependency in Processing module
    
    Why is this PR needed?
    Remove spark dependency in Processing module, as it is not required
    
    What changes were proposed in this PR?
    Removed spark dependency in Processing module
    
    Does this PR introduce any user interface change?
    No
    
    Is any new testcase added?
    No
    
    This closes #3650
---
 .../org/apache/carbondata/streaming/StreamSinkFactory.scala    |  3 +--
 .../sql/execution/command/management/CommonLoadUtils.scala     |  3 +--
 .../sql/execution/streaming/CarbonAppendableStreamSink.scala   |  3 +--
 processing/pom.xml                                             |  5 -----
 .../carbondata/processing/loading/events/LoadEvents.java       | 10 +---------
 5 files changed, 4 insertions(+), 20 deletions(-)

diff --git a/integration/spark/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala
b/integration/spark/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala
index b22d4aa..152c6e7 100644
--- a/integration/spark/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala
+++ b/integration/spark/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala
@@ -102,8 +102,7 @@ object StreamSinkFactory {
       false,
       parameters.asJava,
       parameters.asJava,
-      false,
-      sparkSession
+      false
     )
     OperationListenerBus.getInstance().fireEvent(loadTablePreExecutionEvent, operationContext)
     // prepare the stream segment
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CommonLoadUtils.scala
b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CommonLoadUtils.scala
index 0aada09..9e01fb1 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CommonLoadUtils.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CommonLoadUtils.scala
@@ -289,8 +289,7 @@ object CommonLoadUtils {
         isDataFrame,
         optionsFinal,
         options,
-        isOverwriteTable,
-        sparkSession)
+        isOverwriteTable)
     operationContext.setProperty("isOverwrite", isOverwriteTable)
     OperationListenerBus.getInstance.fireEvent(loadTablePreExecutionEvent, operationContext)
     // Add pre event listener for index datamap
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala
b/integration/spark/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala
index 272d107..039c946 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala
@@ -133,8 +133,7 @@ class CarbonAppendableStreamSink(
         false,
         parameters.asJava,
         parameters.asJava,
-        false,
-        sparkSession
+        false
       )
       OperationListenerBus.getInstance().fireEvent(loadTablePreExecutionEvent, operationContext)
       checkOrHandOffSegment()
diff --git a/processing/pom.xml b/processing/pom.xml
index 3c178dd..5a73405 100644
--- a/processing/pom.xml
+++ b/processing/pom.xml
@@ -41,11 +41,6 @@
       <version>${project.version}</version>
     </dependency>
     <dependency>
-      <groupId>org.apache.spark</groupId>
-      <artifactId>spark-sql_${scala.binary.version}</artifactId>
-      <version>${spark.version}</version>
-    </dependency>
-    <dependency>
       <groupId>com.univocity</groupId>
       <artifactId>univocity-parsers</artifactId>
       <version>2.2.1</version>
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/events/LoadEvents.java
b/processing/src/main/java/org/apache/carbondata/processing/loading/events/LoadEvents.java
index 7f6f8fd..7759b02 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/events/LoadEvents.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/events/LoadEvents.java
@@ -24,8 +24,6 @@ import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
 import org.apache.carbondata.events.Event;
 import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
 
-import org.apache.spark.sql.SparkSession;
-
 public class LoadEvents {
   /**
    * Class for handling operations before start of a load process.
@@ -40,12 +38,11 @@ public class LoadEvents {
     // userProvidedOptions are needed if we need only the load options given by user
     private Map<String, String> userProvidedOptions;
     private boolean isOverWriteTable;
-    private SparkSession sparkSession;
 
     public LoadTablePreExecutionEvent(CarbonTableIdentifier carbonTableIdentifier,
         CarbonLoadModel carbonLoadModel, String factPath, boolean isDataFrameDefined,
         Map<String, String> optionsFinal, Map<String, String> userProvidedOptions,
-        boolean isOverWriteTable, SparkSession sparkSession) {
+        boolean isOverWriteTable) {
       this.carbonTableIdentifier = carbonTableIdentifier;
       this.carbonLoadModel = carbonLoadModel;
       this.factPath = factPath;
@@ -53,7 +50,6 @@ public class LoadEvents {
       this.optionsFinal = optionsFinal;
       this.userProvidedOptions = userProvidedOptions;
       this.isOverWriteTable = isOverWriteTable;
-      this.sparkSession = sparkSession;
     }
 
     public LoadTablePreExecutionEvent(CarbonTableIdentifier carbonTableIdentifier,
@@ -62,10 +58,6 @@ public class LoadEvents {
       this.carbonLoadModel = carbonLoadModel;
     }
 
-    public SparkSession getSparkSession() {
-      return sparkSession;
-    }
-
     public CarbonTableIdentifier getCarbonTableIdentifier() {
       return carbonTableIdentifier;
     }


Mime
View raw message