knox-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From su...@apache.org
Subject knox git commit: KNOX-483 added webhdfs sans ha
Date Mon, 19 Jan 2015 17:21:25 GMT
Repository: knox
Updated Branches:
  refs/heads/KNOX-481 2a0f54169 -> ddeaefb78


KNOX-483 added webhdfs sans ha


Project: http://git-wip-us.apache.org/repos/asf/knox/repo
Commit: http://git-wip-us.apache.org/repos/asf/knox/commit/ddeaefb7
Tree: http://git-wip-us.apache.org/repos/asf/knox/tree/ddeaefb7
Diff: http://git-wip-us.apache.org/repos/asf/knox/diff/ddeaefb7

Branch: refs/heads/KNOX-481
Commit: ddeaefb7888f50344e810e6858939c500836c83c
Parents: 2a0f541
Author: Sumit Gupta <sumit@apache.org>
Authored: Mon Jan 19 12:21:03 2015 -0500
Committer: Sumit Gupta <sumit@apache.org>
Committed: Mon Jan 19 12:21:03 2015 -0500

----------------------------------------------------------------------
 .../services/webhdfs/2.4.0/rewrite.xml          | 70 ++++++++++++++++++++
 .../services/webhdfs/2.4.0/service.xml          | 38 +++++++++++
 .../services/yarn-rm/2.5.0/service.xml          |  2 -
 .../hdfs/WebHdfsDeploymentContributor.java      | 10 +--
 .../deploy/DeploymentFactoryFuncTest.java       | 26 +++++++-
 5 files changed, 137 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/knox/blob/ddeaefb7/gateway-service-definitions/src/main/resources/services/webhdfs/2.4.0/rewrite.xml
----------------------------------------------------------------------
diff --git a/gateway-service-definitions/src/main/resources/services/webhdfs/2.4.0/rewrite.xml
b/gateway-service-definitions/src/main/resources/services/webhdfs/2.4.0/rewrite.xml
new file mode 100644
index 0000000..efbd93d
--- /dev/null
+++ b/gateway-service-definitions/src/main/resources/services/webhdfs/2.4.0/rewrite.xml
@@ -0,0 +1,70 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<rules>
+
+    <rule dir="OUT" name="WEBHDFS/webhdfs/outbound" pattern="hdfs://*:*/{path=**}?{**}">
+        <rewrite template="{$frontend[url]}/webhdfs/v1/{path=**}?{**}"/>
+    </rule>
+
+    <rule dir="OUT" name="WEBHDFS/webhdfs/outbound" pattern="webhdfs://*:*/{path=**}?{**}">
+        <rewrite template="{$frontend[url]}/webhdfs/v1/{path=**}?{**}"/>
+    </rule>
+
+    <rule dir="OUT" name="WEBHDFS/webhdfs/outbound/namenode/headers/location">
+        <match pattern="{scheme}://{host}:{port}/{path=**}?{**}"/>
+        <rewrite template="{$frontend[url]}/webhdfs/data/v1/{path=**}?{scheme}?host={$hostmap(host)}?{port}?{**}"/>
+        <encrypt-query/>
+    </rule>
+
+    <rule dir="IN" name="WEBHDFS/webhdfs/inbound/hdfs" pattern="hdfs:/{path=**}?{**}">
+        <rewrite template="{$serviceMappedUrl[NAMENODE]}/{path=**}?{**}"/>
+    </rule>
+
+    <rule dir="IN" name="WEBHDFS/webhdfs/inbound/webhdfs" pattern="webhdfs:/{path=**}?{**}">
+        <rewrite template="{$serviceUrl[WEBHDFS]}/{path=**}?{**}"/>
+    </rule>
+
+    <rule dir="IN" name="WEBHDFS/webhdfs/inbound/namenode/root" pattern="*://*:*/**/webhdfs/{version}/?{**}">
+        <rewrite template="{$serviceUrl[WEBHDFS]}/{version}/?{**}"/>
+    </rule>
+
+    <rule dir="IN" name="WEBHDFS/webhdfs/inbound/namenode/file" pattern="*://*:*/**/webhdfs/{version}/{path=**}?{**}">
+        <rewrite template="{$serviceUrl[WEBHDFS]}/{version}/{path=**}?{**}"/>
+    </rule>
+
+    <rule dir="IN" name="WEBHDFS/webhdfs/inbound/namenode/home" pattern="*://*:*/**/webhdfs/{version}/~?{**}">
+        <rewrite template="{$serviceUrl[WEBHDFS]}/{version}/user/{$username}?{**}"/>
+    </rule>
+
+    <rule dir="IN" name="WEBHDFS/webhdfs/inbound/namenode/home/file" pattern="*://*:*/**/webhdfs/{version}/~/{path=**}?{**}">
+        <rewrite template="{$serviceUrl[WEBHDFS]}/{version}/user/{$username}/{path=**}?{**}"/>
+    </rule>
+
+    <rule dir="IN" name="WEBHDFS/webhdfs/inbound/datanode">
+        <decrypt-query/>
+        <match pattern="*://*:*/**/webhdfs/data/*/{path=**}?{scheme}?{host}?{port}?{**}"/>
+        <rewrite template="{scheme}://{host}:{port}/{path=**}?{**}"/>
+    </rule>
+
+    <filter name="WEBHDFS/webhdfs/outbound/namenode/headers">
+        <content type="application/x-http-headers">
+            <apply path="Location" rule="WEBHDFS/webhdfs/outbound/namenode/headers/location"/>
+        </content>
+    </filter>
+
+</rules>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/knox/blob/ddeaefb7/gateway-service-definitions/src/main/resources/services/webhdfs/2.4.0/service.xml
----------------------------------------------------------------------
diff --git a/gateway-service-definitions/src/main/resources/services/webhdfs/2.4.0/service.xml
b/gateway-service-definitions/src/main/resources/services/webhdfs/2.4.0/service.xml
new file mode 100644
index 0000000..1aeaf70
--- /dev/null
+++ b/gateway-service-definitions/src/main/resources/services/webhdfs/2.4.0/service.xml
@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<service role="WEBHDFS" name="webhdfs" version="2.4.0">
+    <urls>
+        <url pattern="/webhdfs/v1/?**">
+            <rewrite-filter ref="WEBHDFS/webhdfs/inbound/namenode/root" apply-to="request.url"/>
+        </url>
+        <url pattern="/webhdfs/v1/**?**">
+            <rewrite-filter ref="WEBHDFS/webhdfs/inbound/namenode/file" apply-to="request.url"/>
+            <rewrite-filter ref="WEBHDFS/webhdfs/outbound/namenode/headers" apply-to="response.headers"/>
+        </url>
+        <url pattern="/webhdfs/v1/~?**">
+            <rewrite-filter ref="WEBHDFS/webhdfs/inbound/namenode/home" apply-to="request.url"/>
+        </url>
+        <url pattern="/webhdfs/v1/~/**?**">
+            <rewrite-filter ref="WEBHDFS/webhdfs/inbound/namenode/home/file" apply-to="request.url"/>
+            <rewrite-filter ref="WEBHDFS/webhdfs/outbound/namenode/headers" apply-to="response.headers"/>
+        </url>
+        <url pattern="/webhdfs/data/v1/**?**">
+            <rewrite-filter ref="WEBHDFS/webhdfs/inbound/datanode" apply-to="request.url"/>
+        </url>
+    </urls>
+</service>

http://git-wip-us.apache.org/repos/asf/knox/blob/ddeaefb7/gateway-service-definitions/src/main/resources/services/yarn-rm/2.5.0/service.xml
----------------------------------------------------------------------
diff --git a/gateway-service-definitions/src/main/resources/services/yarn-rm/2.5.0/service.xml
b/gateway-service-definitions/src/main/resources/services/yarn-rm/2.5.0/service.xml
index 61a77f0..f77d94d 100644
--- a/gateway-service-definitions/src/main/resources/services/yarn-rm/2.5.0/service.xml
+++ b/gateway-service-definitions/src/main/resources/services/yarn-rm/2.5.0/service.xml
@@ -16,13 +16,11 @@
    limitations under the License.
 -->
 <service role="RESOURCEMANAGER" name="resourcemanager" version="2.5.0">
-    <policies></policies>
     <urls>
         <url pattern="/resourcemanager/v1/cluster/"/>
         <url pattern="/resourcemanager/v1/cluster/**?**"/>
         <url pattern="/resourcemanager/v1/cluster/apps?**">
             <rewrite-filter ref="RESOURCEMANAGER/resourcemanager/apps/outbound" apply-to="response.body"/>
-            <policies></policies>
         </url>
         <url pattern="/resourcemanager/v1/cluster/apps?**">
             <rewrite-filter ref="RESOURCEMANAGER/resourcemanager/apps/outbound" apply-to="response.body"/>

http://git-wip-us.apache.org/repos/asf/knox/blob/ddeaefb7/gateway-service-webhdfs/src/main/java/org/apache/hadoop/gateway/hdfs/WebHdfsDeploymentContributor.java
----------------------------------------------------------------------
diff --git a/gateway-service-webhdfs/src/main/java/org/apache/hadoop/gateway/hdfs/WebHdfsDeploymentContributor.java
b/gateway-service-webhdfs/src/main/java/org/apache/hadoop/gateway/hdfs/WebHdfsDeploymentContributor.java
index 0e63597..295e081 100644
--- a/gateway-service-webhdfs/src/main/java/org/apache/hadoop/gateway/hdfs/WebHdfsDeploymentContributor.java
+++ b/gateway-service-webhdfs/src/main/java/org/apache/hadoop/gateway/hdfs/WebHdfsDeploymentContributor.java
@@ -44,19 +44,19 @@ public class WebHdfsDeploymentContributor extends ServiceDeploymentContributorBa
 
   @Override
   public String getRole() {
-    return "WEBHDFS";
+    return "xWEBHDFS";
   }
 
   @Override
   public String getName() {
-    return "webhdfs";
+    return "xwebhdfs";
   }
 
   @Override
   public void contributeService( DeploymentContext context, Service service ) throws Exception
{
-    contributeRewriteRules( context, service );
-    contributeNameNodeResource( context, service );
-    contributeDataNodeResource( context, service );
+//    contributeRewriteRules( context, service );
+//    contributeNameNodeResource( context, service );
+//    contributeDataNodeResource( context, service );
   }
 
   private void contributeRewriteRules( DeploymentContext context, Service service ) throws
URISyntaxException, IOException {

http://git-wip-us.apache.org/repos/asf/knox/blob/ddeaefb7/gateway-test/src/test/java/org/apache/hadoop/gateway/deploy/DeploymentFactoryFuncTest.java
----------------------------------------------------------------------
diff --git a/gateway-test/src/test/java/org/apache/hadoop/gateway/deploy/DeploymentFactoryFuncTest.java
b/gateway-test/src/test/java/org/apache/hadoop/gateway/deploy/DeploymentFactoryFuncTest.java
index e2c1bae..8f8f236 100644
--- a/gateway-test/src/test/java/org/apache/hadoop/gateway/deploy/DeploymentFactoryFuncTest.java
+++ b/gateway-test/src/test/java/org/apache/hadoop/gateway/deploy/DeploymentFactoryFuncTest.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.gateway.deploy;
 
+import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.gateway.GatewayTestConfig;
 import org.apache.hadoop.gateway.config.GatewayConfig;
 import org.apache.hadoop.gateway.services.DefaultGatewayServices;
@@ -68,6 +69,7 @@ public class DeploymentFactoryFuncTest {
 
 //    ((GatewayTestConfig) config).setDeploymentDir( "clusters" );
 
+    addStacksDir(config, gatewayDir);
     DefaultGatewayServices srvcs = new DefaultGatewayServices();
     Map<String,String> options = new HashMap<String,String>();
     options.put("persist-master", "false");
@@ -121,6 +123,7 @@ public class DeploymentFactoryFuncTest {
     ((GatewayTestConfig) config).setGatewayHomeDir( gatewayDir.getAbsolutePath() );
     File deployDir = new File( config.getGatewayDeploymentDir() );
     deployDir.mkdirs();
+    addStacksDir(config, gatewayDir);
 
     DefaultGatewayServices srvcs = new DefaultGatewayServices();
     Map<String,String> options = new HashMap<String,String>();
@@ -175,6 +178,7 @@ public class DeploymentFactoryFuncTest {
     ((GatewayTestConfig) config).setGatewayHomeDir( gatewayDir.getAbsolutePath() );
     File deployDir = new File( config.getGatewayDeploymentDir() );
     deployDir.mkdirs();
+    addStacksDir(config, gatewayDir);
 
     DefaultGatewayServices srvcs = new DefaultGatewayServices();
     Map<String,String> options = new HashMap<String,String>();
@@ -214,8 +218,8 @@ public class DeploymentFactoryFuncTest {
     topology.addProvider( authorizer );
 
     WebArchive war = DeploymentFactory.createDeployment( config, topology );
-    //File dir = new File( System.getProperty( "user.dir" ) );
-    //File file = war.as( ExplodedExporter.class ).exportExploded( dir, "test-cluster.war"
);
+//    File dir = new File( System.getProperty( "user.dir" ) );
+//    File file = war.as( ExplodedExporter.class ).exportExploded( dir, "test-cluster.war"
);
 
     Document web = parse( war.get( "WEB-INF/web.xml" ).getAsset().openStream() );
     assertThat( web, hasXPath( "/web-app/servlet/servlet-name", equalTo( "test-cluster" )
) );
@@ -363,6 +367,24 @@ public class DeploymentFactoryFuncTest {
     return builder.parse( source );
   }
 
+  private void addStacksDir(GatewayConfig config, File targetDir) {
+    File stacksDir = new File( config.getGatewayStacksDir() );
+    stacksDir.mkdirs();
+    //TODO: [sumit] This is a hack for now, need to find a better way to locate the source
resources for 'stacks' to be tested
+    String pathToStacksSource = "gateway-service-definitions/src/main/resources/services";
+    File stacksSourceDir = new File( targetDir.getParent(), pathToStacksSource);
+    if (!stacksSourceDir.exists()) {
+      stacksSourceDir = new File( targetDir.getParentFile().getParent(), pathToStacksSource);
+    }
+    if (stacksSourceDir.exists()) {
+      try {
+        FileUtils.copyDirectoryToDirectory(stacksSourceDir, stacksDir);
+      } catch ( IOException e) {
+        fail(e.getMessage());
+      }
+    }
+
+  }
 //  private void dump( Document document ) throws TransformerException {
 //    Transformer transformer = TransformerFactory.newInstance().newTransformer();
 //    transformer.setOutputProperty( OutputKeys.INDENT, "yes" );


Mime
View raw message