knox-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lmc...@apache.org
Subject knox git commit: KNOX-712 - submitPig does not allow multiple argument via arg()
Date Fri, 20 May 2016 22:41:00 GMT
Repository: knox
Updated Branches:
  refs/heads/v0.9.0 dad3843ad -> 814c11a11


KNOX-712 - submitPig does not allow multiple argument via arg()


Project: http://git-wip-us.apache.org/repos/asf/knox/repo
Commit: http://git-wip-us.apache.org/repos/asf/knox/commit/814c11a1
Tree: http://git-wip-us.apache.org/repos/asf/knox/tree/814c11a1
Diff: http://git-wip-us.apache.org/repos/asf/knox/diff/814c11a1

Branch: refs/heads/v0.9.0
Commit: 814c11a111789dc17f8ebbc7b8cf510b172b305f
Parents: dad3843
Author: Larry McCay <lmccay@hortonworks.com>
Authored: Fri May 20 18:37:03 2016 -0400
Committer: Larry McCay <lmccay@hortonworks.com>
Committed: Fri May 20 18:40:36 2016 -0400

----------------------------------------------------------------------
 .../home/samples/ExampleWebHCatPig.groovy       | 84 ++++++++++++++++++++
 .../apache/hadoop/gateway/shell/job/Pig.java    |  8 +-
 2 files changed, 87 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/knox/blob/814c11a1/gateway-release/home/samples/ExampleWebHCatPig.groovy
----------------------------------------------------------------------
diff --git a/gateway-release/home/samples/ExampleWebHCatPig.groovy b/gateway-release/home/samples/ExampleWebHCatPig.groovy
new file mode 100644
index 0000000..8699608
--- /dev/null
+++ b/gateway-release/home/samples/ExampleWebHCatPig.groovy
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import com.jayway.jsonpath.JsonPath
+import groovy.json.JsonSlurper
+import org.apache.hadoop.gateway.shell.Hadoop
+import org.apache.hadoop.gateway.shell.hdfs.Hdfs
+import org.apache.hadoop.gateway.shell.job.Job
+
+import static java.util.concurrent.TimeUnit.SECONDS
+import org.apache.hadoop.gateway.shell.Credentials
+
+gateway = "https://localhost:8443/gateway/sandbox"
+
+credentials = new Credentials()
+credentials.add("ClearInput", "Enter username: ", "user")
+                .add("HiddenInput", "Enter pas" + "sword: ", "pass")
+credentials.collect()
+
+username = credentials.get("user").string()
+pass = credentials.get("pass").string()
+
+jobDir = "/user/" + username + "/test"
+
+session = Hadoop.login( gateway, username, pass )
+
+println "Delete " + jobDir + ": " + Hdfs.rm( session ).file( jobDir ).recursive().now().statusCode
+println "Create " + jobDir + ": " + Hdfs.mkdir( session ).dir( jobDir ).now().statusCode
+
+id_pig = '''
+A = load 'test/input/$filename' using PigStorage(':');
+B = foreach A generate $0 as id;
+dump B;
+'''
+
+fake_passwd = '''ctdean:Chris Dean:secret
+pauls:Paul Stolorz:good
+carmas:Carlos Armas:evil
+dra:Deirdre McClure:marvelous
+'''
+
+Hdfs.put(session).text( id_pig ).to( jobDir + "/input/id.pig" ).now()
+Hdfs.put(session).text( fake_passwd ).to( jobDir + "/input/passwd" ).now()
+
+jobId = Job.submitPig(session) \
+            .file("${jobDir}/input/id.pig") \
+            .arg("-v") \
+            .arg("-p").arg("filename=passwd") \
+            .statusDir("${jobDir}/output") \
+            .now().jobId
+
+println "Submitted job: " + jobId
+
+println "Polling up to 60s for job completion..."
+done = false
+count = 0
+while( !done && count++ < 60 ) {
+  sleep( 1000 )
+  json = Job.queryStatus(session).jobId(jobId).now().string
+  done = JsonPath.read( json, "\$.status.jobComplete" )
+  print "."; System.out.flush();
+}
+println ""
+println "Job status: " + done
+
+text = Hdfs.ls( session ).dir( jobDir + "/output" ).now().string
+json = (new JsonSlurper()).parseText( text )
+println json.FileStatuses.FileStatus.pathSuffix
+
+println "Session closed: " + session.shutdown( 10, SECONDS )

http://git-wip-us.apache.org/repos/asf/knox/blob/814c11a1/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Pig.java
----------------------------------------------------------------------
diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Pig.java b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Pig.java
index 67efba6..01d43e6 100644
--- a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Pig.java
+++ b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Pig.java
@@ -38,8 +38,8 @@ public class Pig {
 
     private String group;
     private String file;
-    private String arg;
     private String statusDir;
+    List<NameValuePair> params = new ArrayList<NameValuePair>();
 
     public Request( Hadoop session ) {
       super( session );
@@ -55,8 +55,8 @@ public class Pig {
       return this;
     }
 
-    public Request arg( String arg ) {
-      this.arg = arg;
+    public Request arg( String value ) {
+      addParam( params, "arg", value );
       return this;
     }
 
@@ -70,10 +70,8 @@ public class Pig {
         @Override
         public Response call() throws Exception {
           URIBuilder uri = uri( Job.SERVICE_PATH, "/pig" );
-          List<NameValuePair> params = new ArrayList<NameValuePair>();
           addParam( params, "group", group );
           addParam( params, "file", file );
-          addParam( params, "arg", arg );
           addParam( params, "statusdir", statusDir );
           UrlEncodedFormEntity form = new UrlEncodedFormEntity( params );
           HttpPost request = new HttpPost( uri.build() );


Mime
View raw message