datafu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e...@apache.org
Subject datafu git commit: Simplify spark gradle build [Forced Update!]
Date Tue, 09 Oct 2018 06:45:00 GMT
Repository: datafu
Updated Branches:
  refs/heads/spark-tmp 009418cd0 -> 65b49fade (forced update)


Simplify spark gradle build


Project: http://git-wip-us.apache.org/repos/asf/datafu/repo
Commit: http://git-wip-us.apache.org/repos/asf/datafu/commit/65b49fad
Tree: http://git-wip-us.apache.org/repos/asf/datafu/tree/65b49fad
Diff: http://git-wip-us.apache.org/repos/asf/datafu/diff/65b49fad

Branch: refs/heads/spark-tmp
Commit: 65b49fadeac48ee30b312ecc915d7fb47413bf62
Parents: fd5868d
Author: Eyal Allweil <eyal@apache.org>
Authored: Tue Oct 9 09:44:46 2018 +0300
Committer: Eyal Allweil <eyal@apache.org>
Committed: Tue Oct 9 09:44:46 2018 +0300

----------------------------------------------------------------------
 datafu-spark/build.gradle                       | 103 ++-----------------
 .../dependency-versions-scala-2.10.gradle       |   1 +
 .../dependency-versions-scala-2.11.gradle       |   1 +
 .../dependency-versions-scala-2.12.gradle       |   1 +
 4 files changed, 9 insertions(+), 97 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/datafu/blob/65b49fad/datafu-spark/build.gradle
----------------------------------------------------------------------
diff --git a/datafu-spark/build.gradle b/datafu-spark/build.gradle
index 6ca209c..3a9d181 100644
--- a/datafu-spark/build.gradle
+++ b/datafu-spark/build.gradle
@@ -17,22 +17,18 @@
  * under the License.
  */
 
+// Much of this file is a variation on the Apache Samza build.gradle file
+
 buildscript {
   repositories {
       mavenCentral()
   }
 }
 
-plugins {
-    id "de.undercouch.download" version "3.3.0"
-}
-
 apply from: file("gradle/dependency-versions-scala-" + scalaVersion + ".gradle")
 
 apply plugin: 'scala'
-apply plugin: datafu.autojar.GradleAutojarPlugin
 
-// allprojects is from samza
 allprojects {
 	// For all scala compilation, add extra compiler options, taken from version-specific
   // dependency-versions-scala file applied above.
@@ -45,18 +41,6 @@ archivesBaseName = 'datafu-spark_' + scalaVersion
 
 import groovy.xml.MarkupBuilder
 
-// the autojarred configuration includes all JARs that will be included
-// in the final JAR via autojar
-configurations.create('autojarred')
-
-configurations.create('jarjar')
-
-configurations {
-  compile {
-    extendsFrom autojarred
-  }
-}
-
 cleanEclipse {
   doLast {
     delete ".apt_generated"
@@ -66,62 +50,6 @@ cleanEclipse {
   }
 }
 
-jar
-{
-  // initial jar only includes the main classes of datafu, not the dependencies.
-  // this is not the one we'll publish.
-  classifier = "core"
-}
-
-ext
-{
-  autojarBuildDir = tasks.jar.destinationDir
-}
-
-task jarWithDependencies(type: Autojar) {
-  description 'Creates a jar that includes the dependencies (under their own namespaces)'
-  autojarFiles = [
-    tasks.jar.getArchivePath().absoluteFile,
-    "META-INF/LICENSE",
-    "META-INF/DISCLAIMER",
-    "META-INF/NOTICE"
-  ]
-  targetConfiguration = configurations.autojarred
-  autojarExtra = '-baeq'
-}
-
-def outputFile = file(tasks.jar.getArchivePath().absoluteFile.toString().replace("-core","-jarjar"))
-
-task jarWithDependenciesNamespaced(dependsOn: jarWithDependencies) {
-  description 'Creates the jar that includes dependencies (under a datafu namespace)'
-
-  doLast {
-    project.ant {
-      taskdef name: "jarjar", classname: "com.tonicsystems.jarjar.JarJarTask", classpath:
configurations.jarjar.asPath
-      jarjar(jarfile: outputFile, filesetmanifest: "merge") {
-        zipfileset(src: tasks.jarWithDependencies.autojarOutput)
-        rule pattern: "org.apache.commons.math.**", result: "datafu.org.apache.commons.math.@1"
-        rule pattern: "com.google.common.**", result: "datafu.com.google.common.@1"
-      }
-    }
-  }
-}
-
-task finalJar(type: Jar, dependsOn: jarWithDependenciesNamespaced) {
-  description 'Creates the final jar'
-
-  from(zipTree(outputFile))
-}
-
-// don't publish the core archive, as this doesn't have the dependencies
-configurations.archives.artifacts.removeAll { return it.classifier == "core"; }
-
-artifacts {
-  archives finalJar
-}
-
-// forcing scala joint compilation is from samza -------------------------
-
 // Force scala joint compilation
 sourceSets.main.scala.srcDir "src/main/java"
 sourceSets.test.scala.srcDir "src/test/java"
@@ -132,42 +60,23 @@ sourceSets.main.java.srcDirs = []
 sourceSets.test.java.srcDirs = []
 
 dependencies {
-		// all these dependencies are taken from samza	
     compile "org.scala-lang:scala-library:$scalaLibVersion"
 
-  // dependencies that are packaged into the jar using autojar
-  // autojar only includes what is needed
-  autojarred "org.apache.commons:commons-math:$commonsMathVersion"
-  autojarred "com.google.guava:guava:$guavaVersion"
-
-  // needed to run jarjar
-  jarjar "com.googlecode.jarjar:jarjar:1.3"
-
-	testCompile "com.holdenkarau:spark-testing-base_" + scalaVersion + ":2.2.0_0.10.0"
-	testCompile "org.scalatest:scalatest_" + scalaVersion + ":2.2.0"
-}
-
-modifyPom {
-  project {
-    dependencies {
-      // No dependencies because everything we need is autojarred.
-    }
-  }
+    testCompile "com.holdenkarau:spark-testing-base_" + scalaVersion + ":" + sparkVersion
+ "_0.10.0"
+	  testCompile "org.scalatest:scalatest_" + scalaVersion + ":" + sparkVersion
 }
 
 // we need to set up the build for hadoop 3
 if (hadoopVersion.startsWith("2.")) {
   dependencies {
-    // needed for compilation only.  obviously don't need to autojar this.
     compile "org.apache.hadoop:hadoop-common:$hadoopVersion"
     compile "org.apache.hadoop:hadoop-hdfs:$hadoopVersion"
     compile "org.apache.hadoop:hadoop-mapreduce-client-jobclient:$hadoopVersion"
-		compile "org.apache.spark:spark-core_" + scalaVersion + ":2.2.0"
-		compile "org.apache.spark:spark-hive_" + scalaVersion + ":2.2.0"
+    compile "org.apache.spark:spark-core_" + scalaVersion + ":" + sparkVersion
+    compile "org.apache.spark:spark-hive_" + scalaVersion + ":" + sparkVersion
   }
 } else {
   dependencies {
-    // needed for compilation only.  obviously don't need to autojar this.
     compile "org.apache.hadoop:hadoop-core:$hadoopVersion"
   }
 }

http://git-wip-us.apache.org/repos/asf/datafu/blob/65b49fad/datafu-spark/gradle/dependency-versions-scala-2.10.gradle
----------------------------------------------------------------------
diff --git a/datafu-spark/gradle/dependency-versions-scala-2.10.gradle b/datafu-spark/gradle/dependency-versions-scala-2.10.gradle
index d128886..3fff3ce 100644
--- a/datafu-spark/gradle/dependency-versions-scala-2.10.gradle
+++ b/datafu-spark/gradle/dependency-versions-scala-2.10.gradle
@@ -17,6 +17,7 @@
  * under the License.
  */
 ext {
+  sparkVersion = "2.3.0"
   scalaVersion = "2.10"
   scalaLibVersion = "2.10.4"
   // Extra options for the compiler:

http://git-wip-us.apache.org/repos/asf/datafu/blob/65b49fad/datafu-spark/gradle/dependency-versions-scala-2.11.gradle
----------------------------------------------------------------------
diff --git a/datafu-spark/gradle/dependency-versions-scala-2.11.gradle b/datafu-spark/gradle/dependency-versions-scala-2.11.gradle
index 5306df0..b3c6599 100644
--- a/datafu-spark/gradle/dependency-versions-scala-2.11.gradle
+++ b/datafu-spark/gradle/dependency-versions-scala-2.11.gradle
@@ -17,6 +17,7 @@
  * under the License.
  */
 ext {
+  sparkVersion = "2.3.0"
   scalaVersion = "2.11"
   scalaLibVersion = "2.11.8"
   // Extra options for the compiler:

http://git-wip-us.apache.org/repos/asf/datafu/blob/65b49fad/datafu-spark/gradle/dependency-versions-scala-2.12.gradle
----------------------------------------------------------------------
diff --git a/datafu-spark/gradle/dependency-versions-scala-2.12.gradle b/datafu-spark/gradle/dependency-versions-scala-2.12.gradle
index fc8c6cd..7ac980e 100644
--- a/datafu-spark/gradle/dependency-versions-scala-2.12.gradle
+++ b/datafu-spark/gradle/dependency-versions-scala-2.12.gradle
@@ -17,6 +17,7 @@
  * under the License.
  */
 ext {
+  sparkVersion = "2.3.0"
   scalaVersion = "2.12"
   scalaLibVersion = "2.12.1"
   // Extra options for the compiler:


Mime
View raw message