knox-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lmc...@apache.org
Subject knox git commit: KNOX-743 - Add submitSqoop via knox shell api
Date Mon, 31 Oct 2016 20:00:00 GMT
Repository: knox
Updated Branches:
  refs/heads/master c6caebd4b -> 43d07683b


KNOX-743 - Add submitSqoop via knox shell api

Project: http://git-wip-us.apache.org/repos/asf/knox/repo
Commit: http://git-wip-us.apache.org/repos/asf/knox/commit/43d07683
Tree: http://git-wip-us.apache.org/repos/asf/knox/tree/43d07683
Diff: http://git-wip-us.apache.org/repos/asf/knox/diff/43d07683

Branch: refs/heads/master
Commit: 43d07683b687ffffe9302e390669bfeccdcf98fc
Parents: c6caebd
Author: Larry McCay <lmccay@hortonworks.com>
Authored: Mon Oct 31 15:59:47 2016 -0400
Committer: Larry McCay <lmccay@hortonworks.com>
Committed: Mon Oct 31 15:59:47 2016 -0400

----------------------------------------------------------------------
 .../home/samples/ExampleWebHCatSqoop.groovy     | 80 ++++++++++++++++
 .../apache/hadoop/gateway/shell/job/Job.java    |  4 +
 .../apache/hadoop/gateway/shell/job/Sqoop.java  | 99 ++++++++++++++++++++
 3 files changed, 183 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/knox/blob/43d07683/gateway-release/home/samples/ExampleWebHCatSqoop.groovy
----------------------------------------------------------------------
diff --git a/gateway-release/home/samples/ExampleWebHCatSqoop.groovy b/gateway-release/home/samples/ExampleWebHCatSqoop.groovy
new file mode 100644
index 0000000..a5629e3
--- /dev/null
+++ b/gateway-release/home/samples/ExampleWebHCatSqoop.groovy
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import com.jayway.jsonpath.JsonPath
+import groovy.json.JsonSlurper
+import org.apache.hadoop.gateway.shell.Hadoop
+import org.apache.hadoop.gateway.shell.hdfs.Hdfs
+import org.apache.hadoop.gateway.shell.job.Job
+
+import static java.util.concurrent.TimeUnit.SECONDS
+import org.apache.hadoop.gateway.shell.Credentials
+
+gateway = "https://localhost:8443/gateway/sandbox"
+
+credentials = new Credentials()
+credentials.add("ClearInput", "Enter username: ", "user")
+                .add("HiddenInput", "Enter pas" + "sword: ", "pass")
+credentials.collect()
+
+username = credentials.get("user").string()
+pass = credentials.get("pass").string()
+
+jobDir = "/user/" + username + "/test"
+
+session = Hadoop.login( gateway, username, pass )
+
+println "Delete " + jobDir + ": " + Hdfs.rm( session ).file( jobDir ).recursive().now().statusCode
+println "Create " + jobDir + ": " + Hdfs.mkdir( session ).dir( jobDir ).now().statusCode
+
+// Define sqoop options value for sqoop command.
+// This use publicly available Genome mysql database.
+// If the database is unavailable, setup an alternate database and update the
+// db information below.
+db = [ driver:"com.mysql.jdbc.Driver", url:"jdbc:mysql://genome-mysql.cse.ucsc.edu/hg38",
user:"genome", password:"", name:"hg38", table:"scBlastTab", split:"query" ]
+
+targetdir = jobDir + "/" + db.table
+
+sqoop_command = "import --driver ${db.driver} --connect ${db.url} --username ${db.user} --password
${db.password} --table ${db.table} --split-by ${db.split} --target-dir ${targetdir}"
+
+jobId = Job.submitSqoop(session) \
+            .command(sqoop_command) \
+            .statusDir("${jobDir}/output") \
+            .now().jobId
+
+println "Submitted job: " + jobId
+
+println "Polling up to 60s for job completion..."
+done = false
+count = 0
+while( !done && count++ < 180 ) {
+  sleep( 1000 )
+  json = Job.queryStatus(session).jobId(jobId).now().string
+  done = JsonPath.read( json, "\$.status.jobComplete" )
+  print "."; System.out.flush();
+}
+println ""
+println "Job status: " + done
+
+text = Hdfs.ls( session ).dir( jobDir + "/output" ).now().string
+json = (new JsonSlurper()).parseText( text )
+println json.FileStatuses.FileStatus.pathSuffix
+
+println "Content of stderr:"
+println Hdfs.get( session ).from( jobDir + "/output/stderr" ).now().string
+
+println "Session closed: " + session.shutdown( 10, SECONDS )

http://git-wip-us.apache.org/repos/asf/knox/blob/43d07683/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Job.java
----------------------------------------------------------------------
diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Job.java b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Job.java
index f374244..ec49fe1 100644
--- a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Job.java
+++ b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Job.java
@@ -27,6 +27,10 @@ public class Job {
     return new Java.Request( session );
   }
 
+  public static Sqoop.Request submitSqoop( Hadoop session ) {
+    return new Sqoop.Request( session );
+  }
+
   public static Pig.Request submitPig( Hadoop session ) {
     return new Pig.Request( session );
   }

http://git-wip-us.apache.org/repos/asf/knox/blob/43d07683/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Sqoop.java
----------------------------------------------------------------------
diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Sqoop.java b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Sqoop.java
new file mode 100644
index 0000000..124a00f
--- /dev/null
+++ b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Sqoop.java
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.gateway.shell.job;
+
+import com.jayway.jsonpath.JsonPath;
+import org.apache.hadoop.gateway.shell.AbstractRequest;
+import org.apache.hadoop.gateway.shell.BasicResponse;
+import org.apache.hadoop.gateway.shell.Hadoop;
+import org.apache.http.HttpResponse;
+import org.apache.http.NameValuePair;
+import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.utils.URIBuilder;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+
+public class Sqoop {
+
+  static class Request extends AbstractRequest<Response> {
+
+    private String statusDir;
+    List<NameValuePair> params = new ArrayList<NameValuePair>();
+
+    public Request( Hadoop session ) {
+      super( session );
+    }
+
+    public Request command( String command ) {
+      addParam( params, "command", command );
+      return this;
+    }
+
+    public Request libdir( String libdir ) {
+      addParam( params, "libdir", libdir );
+      return this;
+    }
+
+    public Request files( String files ) {
+      addParam( params, "files", files );
+      return this;
+    }
+
+    public Request optionsfile( String optionsFile ) {
+      addParam( params, "optionsfile", optionsFile );
+      return this;
+    }
+
+    public Request statusDir( String dir ) {
+      this.statusDir = dir;
+      return this;
+    }
+
+    protected Callable<Response> callable() {
+      return new Callable<Response>() {
+        @Override
+        public Response call() throws Exception {
+          URIBuilder uri = uri( Job.SERVICE_PATH, "/sqoop" );
+          addParam( params, "statusdir", statusDir );
+          UrlEncodedFormEntity form = new UrlEncodedFormEntity( params );
+          HttpPost request = new HttpPost( uri.build() );
+          request.setEntity( form );
+          return new Response( execute( request ) );
+        }
+      };
+    }
+
+  }
+
+  static class Response extends BasicResponse {
+
+    protected Response( HttpResponse response ) {
+      super( response );
+    }
+
+    public String getJobId() throws IOException {
+      return JsonPath.read( getString(), "$.id" );
+    }
+
+  }
+
+}


Mime
View raw message