directory-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From plusplusjia...@apache.org
Subject directory-kerby git commit: Update the support of Spark.
Date Mon, 25 Dec 2017 02:45:37 GMT
Repository: directory-kerby
Updated Branches:
  refs/heads/has-project 510eebfb6 -> 2247b985e


Update the support of Spark.


Project: http://git-wip-us.apache.org/repos/asf/directory-kerby/repo
Commit: http://git-wip-us.apache.org/repos/asf/directory-kerby/commit/2247b985
Tree: http://git-wip-us.apache.org/repos/asf/directory-kerby/tree/2247b985
Diff: http://git-wip-us.apache.org/repos/asf/directory-kerby/diff/2247b985

Branch: refs/heads/has-project
Commit: 2247b985ef10f3ad5d6420e40c4e35ae14435ec4
Parents: 510eebf
Author: plusplusjiajia <jiajia.li@intel.com>
Authored: Mon Dec 25 10:42:02 2017 +0800
Committer: plusplusjiajia <jiajia.li@intel.com>
Committed: Mon Dec 25 10:42:02 2017 +0800

----------------------------------------------------------------------
 has/supports/spark/README.md          | 25 ++++++++++++-
 has/supports/spark/spark-v2.0.0.patch | 59 ++++++++++++++++++++++++++++++
 has/supports/spark/spark-v2.1.1.patch | 51 --------------------------
 3 files changed, 82 insertions(+), 53 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/directory-kerby/blob/2247b985/has/supports/spark/README.md
----------------------------------------------------------------------
diff --git a/has/supports/spark/README.md b/has/supports/spark/README.md
index f08ce50..9ccbbba 100644
--- a/has/supports/spark/README.md
+++ b/has/supports/spark/README.md
@@ -1,7 +1,28 @@
 Enable Spark
 ===============
 
-## 1. Update spark-env.sh
+## 1. Apply the [patch](https://github.com/apache/directory-kerby/blob/has-project/has/supports/hadoop/hadoop-2.7.2.patch)
to hadoop-2.7.2 source code
+```
+git apply hadoop-2.7.2.patch
+```
+
+## 2. Install
+```
+mvn clean install -DskipTests
+```
+
+
+## 3. Apply the patch to spark-v2.0.0 source code
+```
+git apply spark-v2.0.0.patch
+```
+
+## 4. Build
+```
+./build/mvn -Pyarn -Phadoop-2.7 -Dhadoop.version=2.7.2 -DskipTests clean package
+```
+
+## 5. Update spark-env.sh
 ```
 SPARK_HISTORY_OPTS=-Dspark.history.kerberos.enabled=true \
 -Dspark.history.kerberos.principal=<spark/_HOST@HADOOP.COM> \
@@ -10,7 +31,7 @@ SPARK_HISTORY_OPTS=-Dspark.history.kerberos.enabled=true \
 
 > Note "_HOST" should be replaced with the specific hostname.
 
-## 2. Spark-submit job
+## 6. Spark-submit job
 > YARN mode supported only
 ```
 /bin/spark-submit \

http://git-wip-us.apache.org/repos/asf/directory-kerby/blob/2247b985/has/supports/spark/spark-v2.0.0.patch
----------------------------------------------------------------------
diff --git a/has/supports/spark/spark-v2.0.0.patch b/has/supports/spark/spark-v2.0.0.patch
new file mode 100644
index 0000000..9e560a2
--- /dev/null
+++ b/has/supports/spark/spark-v2.0.0.patch
@@ -0,0 +1,59 @@
+diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
+index 9feafc9..5501721 100644
+--- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
++++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
+@@ -549,7 +549,9 @@ object SparkSubmit {
+ 
+     // assure a keytab is available from any place in a JVM
+     if (clusterManager == YARN || clusterManager == LOCAL) {
+-      if (args.principal != null) {
++      if (args.useHas) {
++        UserGroupInformation.loginUserFromHas()
++      } else if (args.principal != null) {
+         require(args.keytab != null, "Keytab must be specified when principal is specified")
+         if (!new File(args.keytab).exists()) {
+           throw new SparkException(s"Keytab file: ${args.keytab} does not exist")
+diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
+index f1761e7..835f4c1 100644
+--- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
++++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
+@@ -78,6 +78,8 @@ private[deploy] class SparkSubmitArguments(args: Seq[String], env: Map[String,
S
+   var submissionToRequestStatusFor: String = null
+   var useRest: Boolean = true // used internally
+ 
++  var useHas: Boolean = false
++
+   /** Default properties present in the currently defined defaults file. */
+   lazy val defaultSparkProperties: HashMap[String, String] = {
+     val defaultProperties = new HashMap[String, String]()
+@@ -435,6 +437,9 @@ private[deploy] class SparkSubmitArguments(args: Seq[String], env: Map[String,
S
+       case VERSION =>
+         SparkSubmit.printVersionAndExit()
+ 
++      case USE_HAS =>
++        useHas = true
++
+       case USAGE_ERROR =>
+         printUsageAndExit(1)
+ 
+diff --git a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitOptionParser.java
b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitOptionParser.java
+index 6767cc5..ec42de5 100644
+--- a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitOptionParser.java
++++ b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitOptionParser.java
+@@ -76,6 +76,8 @@ class SparkSubmitOptionParser {
+   protected final String PRINCIPAL = "--principal";
+   protected final String QUEUE = "--queue";
+ 
++  protected final String USE_HAS = "--use-has";
++
+   /**
+    * This is the canonical list of spark-submit options. Each entry in the array contains
the
+    * different aliases for the same option; the first element of each entry is the "official"
+@@ -115,6 +117,7 @@ class SparkSubmitOptionParser {
+     { REPOSITORIES },
+     { STATUS },
+     { TOTAL_EXECUTOR_CORES },
++    { USE_HAS },
+   };
+ 
+   /**

http://git-wip-us.apache.org/repos/asf/directory-kerby/blob/2247b985/has/supports/spark/spark-v2.1.1.patch
----------------------------------------------------------------------
diff --git a/has/supports/spark/spark-v2.1.1.patch b/has/supports/spark/spark-v2.1.1.patch
deleted file mode 100644
index c7e40b7..0000000
--- a/has/supports/spark/spark-v2.1.1.patch
+++ /dev/null
@@ -1,51 +0,0 @@
-diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
-index 443f1f5..1fc66f0 100644
---- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
-+++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
-@@ -553,7 +553,9 @@ object SparkSubmit {
- 
-     // assure a keytab is available from any place in a JVM
-     if (clusterManager == YARN || clusterManager == LOCAL) {
--      if (args.principal != null) {
-+      if (args.useHas) {
-+        UserGroupInformation.loginUserFromHas()
-+      } else if (args.principal != null) {
-         require(args.keytab != null, "Keytab must be specified when principal is specified")
-         if (!new File(args.keytab).exists()) {
-           throw new SparkException(s"Keytab file: ${args.keytab} does not exist")
-diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
-index f1761e7..5e48419 100644
---- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
-+++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
-@@ -78,6 +78,8 @@ private[deploy] class SparkSubmitArguments(args: Seq[String], env: Map[String,
S
-   var submissionToRequestStatusFor: String = null
-   var useRest: Boolean = true // used internally
- 
-+  var useHas: Boolean = false
-+
-   /** Default properties present in the currently defined defaults file. */
-   lazy val defaultSparkProperties: HashMap[String, String] = {
-     val defaultProperties = new HashMap[String, String]()
-@@ -438,6 +440,9 @@ private[deploy] class SparkSubmitArguments(args: Seq[String], env: Map[String,
S
-       case USAGE_ERROR =>
-         printUsageAndExit(1)
- 
-+      case USE_HAS =>
-+        useHas = true
-+
-       case _ =>
-         throw new IllegalArgumentException(s"Unexpected argument '$opt'.")
-     }
-diff --git a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitOptionParser.java
b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitOptionParser.java
-index 6767cc5..49a7678 100644
---- a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitOptionParser.java
-+++ b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitOptionParser.java
-@@ -76,6 +76,8 @@ class SparkSubmitOptionParser {
-   protected final String PRINCIPAL = "--principal";
-   protected final String QUEUE = "--queue";
- 
-+  protected final String USE_HAS = "--use-has";
-+
-   /**
-    * This is the canonical list of spark-submit options. Each entry in the array contains
the
-    * different aliases for the same option; the first element of each entry is the "official"


Mime
View raw message