Repository: datafu
Updated Branches:
refs/heads/spark-tmp 009418cd0 -> 65b49fade (forced update)
Simplify spark gradle build
Project: http://git-wip-us.apache.org/repos/asf/datafu/repo
Commit: http://git-wip-us.apache.org/repos/asf/datafu/commit/65b49fad
Tree: http://git-wip-us.apache.org/repos/asf/datafu/tree/65b49fad
Diff: http://git-wip-us.apache.org/repos/asf/datafu/diff/65b49fad
Branch: refs/heads/spark-tmp
Commit: 65b49fadeac48ee30b312ecc915d7fb47413bf62
Parents: fd5868d
Author: Eyal Allweil <eyal@apache.org>
Authored: Tue Oct 9 09:44:46 2018 +0300
Committer: Eyal Allweil <eyal@apache.org>
Committed: Tue Oct 9 09:44:46 2018 +0300
----------------------------------------------------------------------
datafu-spark/build.gradle | 103 ++-----------------
.../dependency-versions-scala-2.10.gradle | 1 +
.../dependency-versions-scala-2.11.gradle | 1 +
.../dependency-versions-scala-2.12.gradle | 1 +
4 files changed, 9 insertions(+), 97 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/datafu/blob/65b49fad/datafu-spark/build.gradle
----------------------------------------------------------------------
diff --git a/datafu-spark/build.gradle b/datafu-spark/build.gradle
index 6ca209c..3a9d181 100644
--- a/datafu-spark/build.gradle
+++ b/datafu-spark/build.gradle
@@ -17,22 +17,18 @@
* under the License.
*/
+// Much of this file is a variation on the Apache Samza build.gradle file
+
buildscript {
repositories {
mavenCentral()
}
}
-plugins {
- id "de.undercouch.download" version "3.3.0"
-}
-
apply from: file("gradle/dependency-versions-scala-" + scalaVersion + ".gradle")
apply plugin: 'scala'
-apply plugin: datafu.autojar.GradleAutojarPlugin
-// allprojects is from samza
allprojects {
// For all scala compilation, add extra compiler options, taken from version-specific
// dependency-versions-scala file applied above.
@@ -45,18 +41,6 @@ archivesBaseName = 'datafu-spark_' + scalaVersion
import groovy.xml.MarkupBuilder
-// the autojarred configuration includes all JARs that will be included
-// in the final JAR via autojar
-configurations.create('autojarred')
-
-configurations.create('jarjar')
-
-configurations {
- compile {
- extendsFrom autojarred
- }
-}
-
cleanEclipse {
doLast {
delete ".apt_generated"
@@ -66,62 +50,6 @@ cleanEclipse {
}
}
-jar
-{
- // initial jar only includes the main classes of datafu, not the dependencies.
- // this is not the one we'll publish.
- classifier = "core"
-}
-
-ext
-{
- autojarBuildDir = tasks.jar.destinationDir
-}
-
-task jarWithDependencies(type: Autojar) {
- description 'Creates a jar that includes the dependencies (under their own namespaces)'
- autojarFiles = [
- tasks.jar.getArchivePath().absoluteFile,
- "META-INF/LICENSE",
- "META-INF/DISCLAIMER",
- "META-INF/NOTICE"
- ]
- targetConfiguration = configurations.autojarred
- autojarExtra = '-baeq'
-}
-
-def outputFile = file(tasks.jar.getArchivePath().absoluteFile.toString().replace("-core","-jarjar"))
-
-task jarWithDependenciesNamespaced(dependsOn: jarWithDependencies) {
- description 'Creates the jar that includes dependencies (under a datafu namespace)'
-
- doLast {
- project.ant {
- taskdef name: "jarjar", classname: "com.tonicsystems.jarjar.JarJarTask", classpath:
configurations.jarjar.asPath
- jarjar(jarfile: outputFile, filesetmanifest: "merge") {
- zipfileset(src: tasks.jarWithDependencies.autojarOutput)
- rule pattern: "org.apache.commons.math.**", result: "datafu.org.apache.commons.math.@1"
- rule pattern: "com.google.common.**", result: "datafu.com.google.common.@1"
- }
- }
- }
-}
-
-task finalJar(type: Jar, dependsOn: jarWithDependenciesNamespaced) {
- description 'Creates the final jar'
-
- from(zipTree(outputFile))
-}
-
-// don't publish the core archive, as this doesn't have the dependencies
-configurations.archives.artifacts.removeAll { return it.classifier == "core"; }
-
-artifacts {
- archives finalJar
-}
-
-// forcing scala joint compilation is from samza -------------------------
-
// Force scala joint compilation
sourceSets.main.scala.srcDir "src/main/java"
sourceSets.test.scala.srcDir "src/test/java"
@@ -132,42 +60,23 @@ sourceSets.main.java.srcDirs = []
sourceSets.test.java.srcDirs = []
dependencies {
- // all these dependencies are taken from samza
compile "org.scala-lang:scala-library:$scalaLibVersion"
- // dependencies that are packaged into the jar using autojar
- // autojar only includes what is needed
- autojarred "org.apache.commons:commons-math:$commonsMathVersion"
- autojarred "com.google.guava:guava:$guavaVersion"
-
- // needed to run jarjar
- jarjar "com.googlecode.jarjar:jarjar:1.3"
-
- testCompile "com.holdenkarau:spark-testing-base_" + scalaVersion + ":2.2.0_0.10.0"
- testCompile "org.scalatest:scalatest_" + scalaVersion + ":2.2.0"
-}
-
-modifyPom {
- project {
- dependencies {
- // No dependencies because everything we need is autojarred.
- }
- }
+ testCompile "com.holdenkarau:spark-testing-base_" + scalaVersion + ":" + sparkVersion
+ "_0.10.0"
+ testCompile "org.scalatest:scalatest_" + scalaVersion + ":" + sparkVersion
}
// we need to set up the build for hadoop 3
if (hadoopVersion.startsWith("2.")) {
dependencies {
- // needed for compilation only. obviously don't need to autojar this.
compile "org.apache.hadoop:hadoop-common:$hadoopVersion"
compile "org.apache.hadoop:hadoop-hdfs:$hadoopVersion"
compile "org.apache.hadoop:hadoop-mapreduce-client-jobclient:$hadoopVersion"
- compile "org.apache.spark:spark-core_" + scalaVersion + ":2.2.0"
- compile "org.apache.spark:spark-hive_" + scalaVersion + ":2.2.0"
+ compile "org.apache.spark:spark-core_" + scalaVersion + ":" + sparkVersion
+ compile "org.apache.spark:spark-hive_" + scalaVersion + ":" + sparkVersion
}
} else {
dependencies {
- // needed for compilation only. obviously don't need to autojar this.
compile "org.apache.hadoop:hadoop-core:$hadoopVersion"
}
}
http://git-wip-us.apache.org/repos/asf/datafu/blob/65b49fad/datafu-spark/gradle/dependency-versions-scala-2.10.gradle
----------------------------------------------------------------------
diff --git a/datafu-spark/gradle/dependency-versions-scala-2.10.gradle b/datafu-spark/gradle/dependency-versions-scala-2.10.gradle
index d128886..3fff3ce 100644
--- a/datafu-spark/gradle/dependency-versions-scala-2.10.gradle
+++ b/datafu-spark/gradle/dependency-versions-scala-2.10.gradle
@@ -17,6 +17,7 @@
* under the License.
*/
ext {
+ sparkVersion = "2.3.0"
scalaVersion = "2.10"
scalaLibVersion = "2.10.4"
// Extra options for the compiler:
http://git-wip-us.apache.org/repos/asf/datafu/blob/65b49fad/datafu-spark/gradle/dependency-versions-scala-2.11.gradle
----------------------------------------------------------------------
diff --git a/datafu-spark/gradle/dependency-versions-scala-2.11.gradle b/datafu-spark/gradle/dependency-versions-scala-2.11.gradle
index 5306df0..b3c6599 100644
--- a/datafu-spark/gradle/dependency-versions-scala-2.11.gradle
+++ b/datafu-spark/gradle/dependency-versions-scala-2.11.gradle
@@ -17,6 +17,7 @@
* under the License.
*/
ext {
+ sparkVersion = "2.3.0"
scalaVersion = "2.11"
scalaLibVersion = "2.11.8"
// Extra options for the compiler:
http://git-wip-us.apache.org/repos/asf/datafu/blob/65b49fad/datafu-spark/gradle/dependency-versions-scala-2.12.gradle
----------------------------------------------------------------------
diff --git a/datafu-spark/gradle/dependency-versions-scala-2.12.gradle b/datafu-spark/gradle/dependency-versions-scala-2.12.gradle
index fc8c6cd..7ac980e 100644
--- a/datafu-spark/gradle/dependency-versions-scala-2.12.gradle
+++ b/datafu-spark/gradle/dependency-versions-scala-2.12.gradle
@@ -17,6 +17,7 @@
* under the License.
*/
ext {
+ sparkVersion = "2.3.0"
scalaVersion = "2.12"
scalaLibVersion = "2.12.1"
// Extra options for the compiler:
|