knox-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kmin...@apache.org
Subject git commit: KNOX-172: Partial support for ~. WebHCat still outstanding.
Date Tue, 01 Oct 2013 12:31:54 GMT
Updated Branches:
  refs/heads/master 832e86c86 -> 34492a258


KNOX-172: Partial support for ~.  WebHCat still outstanding.


Project: http://git-wip-us.apache.org/repos/asf/incubator-knox/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-knox/commit/34492a25
Tree: http://git-wip-us.apache.org/repos/asf/incubator-knox/tree/34492a25
Diff: http://git-wip-us.apache.org/repos/asf/incubator-knox/diff/34492a25

Branch: refs/heads/master
Commit: 34492a25850271db552d84a83c61a58e90e216d6
Parents: 832e86c
Author: Kevin Minder <kevin.minder@hortonworks.com>
Authored: Tue Oct 1 08:31:46 2013 -0400
Committer: Kevin Minder <kevin.minder@hortonworks.com>
Committed: Tue Oct 1 08:31:46 2013 -0400

----------------------------------------------------------------------
 .../rewrite/api/UrlRewriteProcessorTest.java    | 31 ++++++++++++++-
 .../api/UrlRewriteProcessorTest/rewrite.xml     |  9 +++++
 .../home/samples/ExampleOozieWorkflow.groovy    | 40 ++++++++------------
 .../home/samples/ExampleWebHCatJob.groovy       |  2 +
 gateway-release/home/templates/sandbox-1-3.xml  |  5 ---
 .../OozieDeploymentContributor/rewrite.xml      | 19 ++++++++--
 .../hadoop/gateway/security/SubjectUtils.java   |  5 +--
 7 files changed, 75 insertions(+), 36 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/34492a25/gateway-provider-rewrite/src/test/java/org/apache/hadoop/gateway/filter/rewrite/api/UrlRewriteProcessorTest.java
----------------------------------------------------------------------
diff --git a/gateway-provider-rewrite/src/test/java/org/apache/hadoop/gateway/filter/rewrite/api/UrlRewriteProcessorTest.java
b/gateway-provider-rewrite/src/test/java/org/apache/hadoop/gateway/filter/rewrite/api/UrlRewriteProcessorTest.java
index b01b656..86d8460 100644
--- a/gateway-provider-rewrite/src/test/java/org/apache/hadoop/gateway/filter/rewrite/api/UrlRewriteProcessorTest.java
+++ b/gateway-provider-rewrite/src/test/java/org/apache/hadoop/gateway/filter/rewrite/api/UrlRewriteProcessorTest.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.gateway.filter.rewrite.api;
 import org.apache.hadoop.gateway.util.urltemplate.Parser;
 import org.apache.hadoop.gateway.util.urltemplate.Template;
 import org.easymock.EasyMock;
-import org.junit.Ignore;
 import org.junit.Test;
 
 import javax.servlet.http.HttpServletRequest;
@@ -152,6 +151,36 @@ public class UrlRewriteProcessorTest {
   }
 
   @Test
+  public void testRewriteViaRuleWithComplexFlow() throws Exception {
+    UrlRewriteEnvironment environment = EasyMock.createNiceMock( UrlRewriteEnvironment.class
);
+    HttpServletRequest request = EasyMock.createNiceMock( HttpServletRequest.class );
+    HttpServletResponse response = EasyMock.createNiceMock( HttpServletResponse.class );
+    EasyMock.replay( environment, request, response );
+
+    UrlRewriteProcessor processor = new UrlRewriteProcessor();
+    UrlRewriteRulesDescriptor config = UrlRewriteRulesDescriptorFactory.load(
+        "xml", getTestResourceReader( "rewrite.xml", "UTF-8" ) );
+    processor.initialize( environment, config );
+
+    Template inputUrl;
+    Template outputUrl;
+
+    inputUrl = Parser.parse( "test-scheme://test-host:777/test-path" );
+    outputUrl = processor.rewrite( null, inputUrl, UrlRewriter.Direction.IN, "test-rule-with-complex-flow"
);
+    assertThat(
+        "Expect rewrite to contain the correct path.",
+        outputUrl.toString(), is( "test-scheme-output://test-host-output:42/test-path-output/test-path"
) );
+
+    inputUrl = Parser.parse( "test-scheme://test-host:42/~/test-path" );
+    outputUrl = processor.rewrite( null, inputUrl, UrlRewriter.Direction.IN, "test-rule-with-complex-flow"
);
+    assertThat(
+        "Expect rewrite to contain the correct path.",
+        outputUrl.toString(), is( "test-scheme-output://test-host-output:777/test-path-output/test-home/test-path"
) );
+
+    processor.destroy();
+  }
+
+  @Test
   public void testRewriteViaRuleWithWildcardTemplateAndOptionalQuery() throws Exception {
     UrlRewriteEnvironment environment = EasyMock.createNiceMock( UrlRewriteEnvironment.class
);
     HttpServletRequest request = EasyMock.createNiceMock( HttpServletRequest.class );

http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/34492a25/gateway-provider-rewrite/src/test/resources/org/apache/hadoop/gateway/filter/rewrite/api/UrlRewriteProcessorTest/rewrite.xml
----------------------------------------------------------------------
diff --git a/gateway-provider-rewrite/src/test/resources/org/apache/hadoop/gateway/filter/rewrite/api/UrlRewriteProcessorTest/rewrite.xml
b/gateway-provider-rewrite/src/test/resources/org/apache/hadoop/gateway/filter/rewrite/api/UrlRewriteProcessorTest/rewrite.xml
index 998f54c..84836d5 100644
--- a/gateway-provider-rewrite/src/test/resources/org/apache/hadoop/gateway/filter/rewrite/api/UrlRewriteProcessorTest/rewrite.xml
+++ b/gateway-provider-rewrite/src/test/resources/org/apache/hadoop/gateway/filter/rewrite/api/UrlRewriteProcessorTest/rewrite.xml
@@ -24,4 +24,13 @@
         <rewrite template="test-scheme-output://test-host-output:777/test-path-output/{path}?{**}"/>
     </rule>
 
+    <rule name="test-rule-with-complex-flow" flow="OR">
+        <match pattern="*://*:*/~/{path=**}?{**}">
+            <rewrite template="test-scheme-output://test-host-output:777/test-path-output/test-home/{path}?{**}"/>
+        </match>
+        <match pattern="*://*:*/{path=**}?{**}">
+            <rewrite template="test-scheme-output://test-host-output:42/test-path-output/{path}?{**}"/>
+        </match>
+    </rule>
+
 </rules>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/34492a25/gateway-release/home/samples/ExampleOozieWorkflow.groovy
----------------------------------------------------------------------
diff --git a/gateway-release/home/samples/ExampleOozieWorkflow.groovy b/gateway-release/home/samples/ExampleOozieWorkflow.groovy
index 3f5c0ed..157e4ac 100644
--- a/gateway-release/home/samples/ExampleOozieWorkflow.groovy
+++ b/gateway-release/home/samples/ExampleOozieWorkflow.groovy
@@ -38,8 +38,8 @@ definition = """\
             <job-tracker>\${jobTracker}</job-tracker>
             <name-node>\${nameNode}</name-node>
             <main-class>org.apache.hadoop.examples.WordCount</main-class>
-            <arg>$jobDir/input</arg>
-            <arg>$jobDir/output</arg>
+            <arg>\${inputDir}</arg>
+            <arg>\${outputDir}</arg>
         </java>
         <ok to="end"/>
         <error to="fail"/>
@@ -54,7 +54,7 @@ definition = """\
 configuration = """\
 <configuration>
     <property>
-        <name>fs.default.name</name>
+        <name>user.name</name>
         <value>default</value>
     </property>
     <property>
@@ -62,33 +62,21 @@ configuration = """\
         <value>default</value>
     </property>
     <property>
-        <name>mapred.job.tracker</name>
-        <value>default</value>
-    </property>
-    <property>
         <name>jobTracker</name>
         <value>default</value>
     </property>
     <property>
-        <name>user.name</name>
-        <value>default</value>
+        <name>inputDir</name>
+        <value>$jobDir/input</value>
     </property>
     <property>
-        <name>mapreduce.job.user.name</name>
-        <value>default</value>
+        <name>outputDir</name>
+        <value>$jobDir/output</value>
     </property>
     <property>
         <name>oozie.wf.application.path</name>
         <value>$jobDir</value>
     </property>
-    <property>
-        <name>oozie.libpath</name>
-        <value>$jobDir/lib</value>
-    </property>
-    <property>
-        <name>oozie.proxysubmission</name>
-        <value>true</value>
-    </property>
 </configuration>
 """
 
@@ -112,17 +100,21 @@ jobId = Workflow.submit(session).text( configuration ).now().jobId
 println "Submitted job: " + jobId
 
 println "Polling up to 60s for job completion..."
-status = "UNKNOWN";
+status = "RUNNING";
 count = 0;
-while( status != "SUCCEEDED" && count++ < 60 ) {
+while( status == "RUNNING" && count++ < 60 ) {
   sleep( 1000 )
   json = Workflow.status(session).jobId( jobId ).now().string
   status = JsonPath.read( json, "\$.status" )
+  print "."; System.out.flush();
 }
+println ""
 println "Job status: " + status
 
-text = Hdfs.ls( session ).dir( jobDir + "/output" ).now().string
-json = (new JsonSlurper()).parseText( text )
-println json.FileStatuses.FileStatus.pathSuffix
+if( status == "SUCCEEDED" ) {
+  text = Hdfs.ls( session ).dir( jobDir + "/output" ).now().string
+  json = (new JsonSlurper()).parseText( text )
+  println json.FileStatuses.FileStatus.pathSuffix
+}
 
 println "Session closed: " + session.shutdown( 10, SECONDS )
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/34492a25/gateway-release/home/samples/ExampleWebHCatJob.groovy
----------------------------------------------------------------------
diff --git a/gateway-release/home/samples/ExampleWebHCatJob.groovy b/gateway-release/home/samples/ExampleWebHCatJob.groovy
index d0432ba..50e43fd 100644
--- a/gateway-release/home/samples/ExampleWebHCatJob.groovy
+++ b/gateway-release/home/samples/ExampleWebHCatJob.groovy
@@ -58,7 +58,9 @@ while( !done && count++ < 60 ) {
   sleep( 1000 )
   json = Job.queryStatus(session).jobId(jobId).now().string
   done = JsonPath.read( json, "\$.status.jobComplete" )
+  print "."; System.out.flush();
 }
+println ""
 println "Job status: " + done
 
 text = Hdfs.ls( session ).dir( jobDir + "/output" ).now().string

http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/34492a25/gateway-release/home/templates/sandbox-1-3.xml
----------------------------------------------------------------------
diff --git a/gateway-release/home/templates/sandbox-1-3.xml b/gateway-release/home/templates/sandbox-1-3.xml
index ba824e4..9f1ab8f 100644
--- a/gateway-release/home/templates/sandbox-1-3.xml
+++ b/gateway-release/home/templates/sandbox-1-3.xml
@@ -113,9 +113,4 @@
         <url>http://localhost:60080</url>
     </service>
 
-    <service>
-        <role>HIVE</role>
-        <url>http://localhost:10000</url>
-    </service>
-
 </topology>

http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/34492a25/gateway-service-oozie/src/main/resources/org/apache/hadoop/gateway/oozie/OozieDeploymentContributor/rewrite.xml
----------------------------------------------------------------------
diff --git a/gateway-service-oozie/src/main/resources/org/apache/hadoop/gateway/oozie/OozieDeploymentContributor/rewrite.xml
b/gateway-service-oozie/src/main/resources/org/apache/hadoop/gateway/oozie/OozieDeploymentContributor/rewrite.xml
index b7abde6..d3c52bb 100644
--- a/gateway-service-oozie/src/main/resources/org/apache/hadoop/gateway/oozie/OozieDeploymentContributor/rewrite.xml
+++ b/gateway-service-oozie/src/main/resources/org/apache/hadoop/gateway/oozie/OozieDeploymentContributor/rewrite.xml
@@ -40,9 +40,16 @@
         <rewrite template="{$serviceMappedAddr[JOBTRACKER]}"/>
     </rule>
 
-    <rule name="OOZIE/oozie/hdfs-path">
-        <match pattern="{path=**}"/>
-        <rewrite template="hdfs://{$serviceMappedHost[NAMENODE]}:{$servicePort[NAMENODE]}/{path=**}"/>
+    <rule name="OOZIE/oozie/hdfs-path" flow="OR">
+        <match pattern="/~">
+            <rewrite template="hdfs://{$serviceMappedHost[NAMENODE]}:{$servicePort[NAMENODE]}/user/{$username}"/>
+        </match>
+        <match pattern="/~/{path=**}">
+            <rewrite template="hdfs://{$serviceMappedHost[NAMENODE]}:{$servicePort[NAMENODE]}/user/{$username}/{path=**}"/>
+        </match>
+        <match pattern="{path=**}">
+            <rewrite template="hdfs://{$serviceMappedHost[NAMENODE]}:{$servicePort[NAMENODE]}/{path=**}"/>
+        </match>
     </rule>
 
     <filter name="OOZIE/oozie/configuration">
@@ -84,9 +91,15 @@
                 <detect path="name" value="mapred.input.dir">
                     <apply path="value" rule="OOZIE/oozie/hdfs-path"/>
                 </detect>
+                <detect path="name" value="inputDir">
+                    <apply path="value" rule="OOZIE/oozie/hdfs-path"/>
+                </detect>
                 <detect path="name" value="mapred.output.dir">
                     <apply path="value" rule="OOZIE/oozie/hdfs-path"/>
                 </detect>
+                <detect path="name" value="outputDir">
+                    <apply path="value" rule="OOZIE/oozie/hdfs-path"/>
+                </detect>
             </buffer>
         </content>
         <content type="*/json">

http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/34492a25/gateway-spi/src/main/java/org/apache/hadoop/gateway/security/SubjectUtils.java
----------------------------------------------------------------------
diff --git a/gateway-spi/src/main/java/org/apache/hadoop/gateway/security/SubjectUtils.java
b/gateway-spi/src/main/java/org/apache/hadoop/gateway/security/SubjectUtils.java
index d40613e..7116c0d 100644
--- a/gateway-spi/src/main/java/org/apache/hadoop/gateway/security/SubjectUtils.java
+++ b/gateway-spi/src/main/java/org/apache/hadoop/gateway/security/SubjectUtils.java
@@ -17,12 +17,11 @@
  */
 package org.apache.hadoop.gateway.security;
 
+import javax.security.auth.Subject;
 import java.security.AccessController;
 import java.security.Principal;
 import java.util.Set;
 
-import javax.security.auth.Subject;
-
 /**
  * General utility methods for interrogating the standard java Subject
  */
@@ -65,7 +64,7 @@ public class SubjectUtils {
 
     Set<ImpersonatedPrincipal> impPrincipals = subject.getPrincipals(ImpersonatedPrincipal.class);
     if (impPrincipals.size() > 0) {
-      return ((PrimaryPrincipal)impPrincipals.toArray()[0]).getName();
+      return ((Principal)impPrincipals.toArray()[0]).getName();
     }
     
     return name;


Mime
View raw message