knox-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From pzamp...@apache.org
Subject knox git commit: KNOX-1287 - Invalid federated namenode namespace declaration should not produce invalid URLs in topologies.
Date Thu, 03 May 2018 15:53:26 GMT
Repository: knox
Updated Branches:
  refs/heads/master fc0e59a9c -> 8a1f03b1c


KNOX-1287 - Invalid federated namenode namespace declaration should not produce invalid URLs
in topologies.


Project: http://git-wip-us.apache.org/repos/asf/knox/repo
Commit: http://git-wip-us.apache.org/repos/asf/knox/commit/8a1f03b1
Tree: http://git-wip-us.apache.org/repos/asf/knox/tree/8a1f03b1
Diff: http://git-wip-us.apache.org/repos/asf/knox/diff/8a1f03b1

Branch: refs/heads/master
Commit: 8a1f03b1c63d8b445302dc1ed46b033086fcb9be
Parents: fc0e59a
Author: Phil Zampino <pzampino@apache.org>
Authored: Wed May 2 22:55:16 2018 -0400
Committer: Phil Zampino <pzampino@apache.org>
Committed: Thu May 3 11:37:59 2018 -0400

----------------------------------------------------------------------
 .../discovery/ambari/HDFSURLCreatorBase.java    |  21 +-
 .../discovery/ambari/NameNodeUrlCreator.java    |  16 +-
 .../AmbariDynamicServiceURLCreatorTest.java     |  64 ++----
 .../discovery/ambari/WebHdfsUrlCreatorTest.java | 216 +++++++++++++++++--
 .../simple/SimpleDescriptorHandler.java         |  20 +-
 5 files changed, 255 insertions(+), 82 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/knox/blob/8a1f03b1/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/HDFSURLCreatorBase.java
----------------------------------------------------------------------
diff --git a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/HDFSURLCreatorBase.java
b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/HDFSURLCreatorBase.java
index 54fa87e..5a04e7e 100644
--- a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/HDFSURLCreatorBase.java
+++ b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/HDFSURLCreatorBase.java
@@ -59,22 +59,18 @@ public abstract class HDFSURLCreatorBase implements ServiceURLCreator
{
                                       cluster.getServiceConfiguration(CONFIG_SERVICE_HDFS,
CONFIG_TYPE_HDFS_SITE);
       if (sc != null) {
         // First, check if it's HA config
-        String nameServices = null;
-        AmbariComponent nameNodeComp = cluster.getComponent(CONFIG_SERVICE_NAMENODE);
-        if (nameNodeComp != null) {
-          nameServices = nameNodeComp.getConfigProperty("dfs.nameservices");
-        }
-
+        String nameServices = sc.getProperties().get("dfs.nameservices");
         if (nameServices != null && !nameServices.isEmpty()) {
           String ns = null;
 
           // Parse the nameservices value
           String[] namespaces = nameServices.split(",");
 
-          if (namespaces.length > 1) {
-            String nsParam = (serviceParams != null) ? serviceParams.get(NAMESERVICE_PARAM)
: null;
+          String nsParam = (serviceParams != null) ? serviceParams.get(NAMESERVICE_PARAM)
: null;
+
+          if (namespaces.length > 1 || nsParam != null) {
             if (nsParam != null) {
-              if (!validateDeclaredNameService(sc, nsParam)) {
+              if (!validateDeclaredNameService(namespaces, nsParam)) {
                 log.undefinedHDFSNameService(nsParam);
               }
               ns = nsParam;
@@ -136,12 +132,9 @@ public abstract class HDFSURLCreatorBase implements ServiceURLCreator
{
 
 
   // Verify whether the declared nameservice is among the configured nameservices in the
cluster
-  private static boolean validateDeclaredNameService(AmbariCluster.ServiceConfiguration hdfsSite,
-                                                     String                             declaredNameService)
{
+  private static boolean validateDeclaredNameService(String[] namespaces, String declaredNameService)
{
     boolean isValid = false;
-    String nameservices = hdfsSite.getProperties().get("dfs.nameservices");
-    if (nameservices != null) {
-      String[] namespaces = nameservices.split(",");
+    if (namespaces != null) {
       for (String ns : namespaces) {
         if (ns.equals(declaredNameService)) {
           isValid = true;

http://git-wip-us.apache.org/repos/asf/knox/blob/8a1f03b1/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/NameNodeUrlCreator.java
----------------------------------------------------------------------
diff --git a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/NameNodeUrlCreator.java
b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/NameNodeUrlCreator.java
index 478d12f..ff071e8 100644
--- a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/NameNodeUrlCreator.java
+++ b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/NameNodeUrlCreator.java
@@ -53,9 +53,9 @@ public class NameNodeUrlCreator implements ServiceURLCreator {
         // Validate declared nameservice against available nameservices
         if (!validateDeclaredNameService(cluster, declaredNameService)) {
           log.undefinedHDFSNameService(declaredNameService);
+        } else {
+          urls.add("hdfs://" + declaredNameService);
         }
-
-        urls.add("hdfs://" + declaredNameService);
       } else {
         // Add the default nameservice URL to the result
         AmbariCluster.ServiceConfiguration coreSite = cluster.getServiceConfiguration("HDFS",
"core-site");
@@ -74,11 +74,13 @@ public class NameNodeUrlCreator implements ServiceURLCreator {
   // Verify whether the declared nameservice is among the configured nameservices in the
cluster
   private static boolean validateDeclaredNameService(AmbariCluster cluster, String declaredNameService)
{
     boolean isValid = false;
-    AmbariCluster.ServiceConfiguration hdfsSite = cluster.getServiceConfiguration("HDFS",
"hdfs-site");
-    if (hdfsSite != null) {
-      String nameservices = hdfsSite.getProperties().get("dfs.nameservices");
-      if (nameservices != null) {
-        String[] namespaces = nameservices.split(",");
+
+    AmbariComponent nameNodeComp = cluster.getComponent(SERVICE);
+    if (nameNodeComp != null) {
+      String nameServices = nameNodeComp.getConfigProperty("dfs.nameservices");
+      if (nameServices != null && !nameServices.isEmpty()) {
+        // Parse the nameservices value
+        String[] namespaces = nameServices.split(",");
         for (String ns : namespaces) {
           if (ns.equals(declaredNameService)) {
             isValid = true;

http://git-wip-us.apache.org/repos/asf/knox/blob/8a1f03b1/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariDynamicServiceURLCreatorTest.java
----------------------------------------------------------------------
diff --git a/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariDynamicServiceURLCreatorTest.java
b/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariDynamicServiceURLCreatorTest.java
index 03612c1..2a63535 100644
--- a/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariDynamicServiceURLCreatorTest.java
+++ b/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariDynamicServiceURLCreatorTest.java
@@ -36,6 +36,7 @@ import static junit.framework.TestCase.fail;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
 
 
 public class AmbariDynamicServiceURLCreatorTest {
@@ -181,9 +182,7 @@ public class AmbariDynamicServiceURLCreatorTest {
         params.put("discovery-nameservice", delcaredNS);
 
         String url = getTestNameNodeURL("nn1", "ns1,ns2", params);
-        assertEquals("Invalid nameservice declaration should still yield a URL.",
-                     "hdfs://" + delcaredNS,
-                     url);
+        assertNull("Invalid nameservice declaration should still yield a URL.", url);
     }
 
     private String getTestNameNodeURL(String address) throws Exception {
@@ -208,17 +207,24 @@ public class AmbariDynamicServiceURLCreatorTest {
         }
         EasyMock.expect(hdfsSC.getProperties()).andReturn(hdfsProps).anyTimes();
         EasyMock.replay(hdfsSC);
+        AmbariComponent nnComp = EasyMock.createNiceMock(AmbariComponent.class);
+        EasyMock.expect(nnComp.getConfigProperty("dfs.nameservices")).andReturn(nameservices).anyTimes();
+        EasyMock.replay(nnComp);
 
         AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class);
         EasyMock.expect(cluster.getServiceConfiguration("HDFS", "core-site")).andReturn(coreSC).anyTimes();
         EasyMock.expect(cluster.getServiceConfiguration("HDFS", "hdfs-site")).andReturn(hdfsSC).anyTimes();
+        EasyMock.expect(cluster.getComponent("NAMENODE")).andReturn(nnComp).anyTimes();
         EasyMock.replay(cluster);
 
         // Create the URL
         List<String> urls = ServiceURLFactory.newInstance(cluster).create("NAMENODE",
serviceParams);
         assertNotNull(urls);
-        assertFalse(urls.isEmpty());
-        return urls.get(0);
+        String url = null;
+        if (!urls.isEmpty()) {
+            url = urls.get(0);
+        }
+        return url;
     }
 
 
@@ -399,19 +405,15 @@ public class AmbariDynamicServiceURLCreatorTest {
         final String EXPECTED_ADDR_1 = "http://" + HTTP_ADDRESS_1 + "/webhdfs";
         final String EXPECTED_ADDR_2 = "http://" + HTTP_ADDRESS_2 + "/webhdfs";
 
-        AmbariComponent namenode = EasyMock.createNiceMock(AmbariComponent.class);
-        EasyMock.expect(namenode.getConfigProperty("dfs.nameservices")).andReturn(NAMESERVICES).anyTimes();
-        EasyMock.replay(namenode);
-
         AmbariCluster.ServiceConfiguration hdfsSC = EasyMock.createNiceMock(AmbariCluster.ServiceConfiguration.class);
         Map<String, String> hdfsProps = new HashMap<>();
+        hdfsProps.put("dfs.nameservices", NAMESERVICES);
         hdfsProps.put("dfs.namenode.http-address." + NAMESERVICES + ".nn1", HTTP_ADDRESS_1);
         hdfsProps.put("dfs.namenode.http-address." + NAMESERVICES + ".nn2", HTTP_ADDRESS_2);
         EasyMock.expect(hdfsSC.getProperties()).andReturn(hdfsProps).anyTimes();
         EasyMock.replay(hdfsSC);
 
         AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class);
-        EasyMock.expect(cluster.getComponent("NAMENODE")).andReturn(namenode).anyTimes();
         EasyMock.expect(cluster.getServiceConfiguration("HDFS", "hdfs-site")).andReturn(hdfsSC).anyTimes();
         EasyMock.replay(cluster);
 
@@ -432,19 +434,15 @@ public class AmbariDynamicServiceURLCreatorTest {
         final String EXPECTED_ADDR_1 = "http://" + HTTP_ADDRESS_1;
         final String EXPECTED_ADDR_2 = "http://" + HTTP_ADDRESS_2;
 
-        AmbariComponent namenode = EasyMock.createNiceMock(AmbariComponent.class);
-        EasyMock.expect(namenode.getConfigProperty("dfs.nameservices")).andReturn(NAMESERVICES).anyTimes();
-        EasyMock.replay(namenode);
-
         AmbariCluster.ServiceConfiguration hdfsSC = EasyMock.createNiceMock(AmbariCluster.ServiceConfiguration.class);
         Map<String, String> hdfsProps = new HashMap<>();
+        hdfsProps.put("dfs.nameservices", NAMESERVICES);
         hdfsProps.put("dfs.namenode.http-address." + NAMESERVICES + ".nn1", HTTP_ADDRESS_1);
         hdfsProps.put("dfs.namenode.http-address." + NAMESERVICES + ".nn2", HTTP_ADDRESS_2);
         EasyMock.expect(hdfsSC.getProperties()).andReturn(hdfsProps).anyTimes();
         EasyMock.replay(hdfsSC);
 
         AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class);
-        EasyMock.expect(cluster.getComponent("NAMENODE")).andReturn(namenode).anyTimes();
         EasyMock.expect(cluster.getServiceConfiguration("HDFS", "hdfs-site")).andReturn(hdfsSC).anyTimes();
         EasyMock.replay(cluster);
 
@@ -473,12 +471,9 @@ public class AmbariDynamicServiceURLCreatorTest {
         final String EXPECTED_ADDR_1 = "http://" + HTTP_ADDRESS_11 + "/webhdfs";
         final String EXPECTED_ADDR_2 = "http://" + HTTP_ADDRESS_12 + "/webhdfs";
 
-        AmbariComponent namenode = EasyMock.createNiceMock(AmbariComponent.class);
-        EasyMock.expect(namenode.getConfigProperty("dfs.nameservices")).andReturn(NAMESERVICES).anyTimes();
-        EasyMock.replay(namenode);
-
         AmbariCluster.ServiceConfiguration hdfsSC = EasyMock.createNiceMock(AmbariCluster.ServiceConfiguration.class);
         Map<String, String> hdfsProps = new HashMap<>();
+        hdfsProps.put("dfs.nameservices", NAMESERVICES);
         hdfsProps.put("dfs.namenode.http-address." + NS1 + ".nn1", HTTP_ADDRESS_11);
         hdfsProps.put("dfs.namenode.http-address." + NS1 + ".nn2", HTTP_ADDRESS_12);
         hdfsProps.put("dfs.namenode.http-address." + NS2 + ".nn1", HTTP_ADDRESS_21);
@@ -487,7 +482,6 @@ public class AmbariDynamicServiceURLCreatorTest {
         EasyMock.replay(hdfsSC);
 
         AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class);
-        EasyMock.expect(cluster.getComponent("NAMENODE")).andReturn(namenode).anyTimes();
         EasyMock.expect(cluster.getServiceConfiguration("HDFS", "hdfs-site")).andReturn(hdfsSC).anyTimes();
         EasyMock.replay(cluster);
 
@@ -516,12 +510,9 @@ public class AmbariDynamicServiceURLCreatorTest {
         final String EXPECTED_ADDR_1 = "http://" + HTTP_ADDRESS_11;
         final String EXPECTED_ADDR_2 = "http://" + HTTP_ADDRESS_12;
 
-        AmbariComponent namenode = EasyMock.createNiceMock(AmbariComponent.class);
-        EasyMock.expect(namenode.getConfigProperty("dfs.nameservices")).andReturn(NAMESERVICES).anyTimes();
-        EasyMock.replay(namenode);
-
         AmbariCluster.ServiceConfiguration hdfsSC = EasyMock.createNiceMock(AmbariCluster.ServiceConfiguration.class);
         Map<String, String> hdfsProps = new HashMap<>();
+        hdfsProps.put("dfs.nameservices", NAMESERVICES);
         hdfsProps.put("dfs.namenode.http-address." + NS1 + ".nn1", HTTP_ADDRESS_11);
         hdfsProps.put("dfs.namenode.http-address." + NS1 + ".nn2", HTTP_ADDRESS_12);
         hdfsProps.put("dfs.namenode.http-address." + NS2 + ".nn1", HTTP_ADDRESS_21);
@@ -530,7 +521,6 @@ public class AmbariDynamicServiceURLCreatorTest {
         EasyMock.replay(hdfsSC);
 
         AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class);
-        EasyMock.expect(cluster.getComponent("NAMENODE")).andReturn(namenode).anyTimes();
         EasyMock.expect(cluster.getServiceConfiguration("HDFS", "hdfs-site")).andReturn(hdfsSC).anyTimes();
         EasyMock.replay(cluster);
 
@@ -558,12 +548,9 @@ public class AmbariDynamicServiceURLCreatorTest {
         final String EXPECTED_ADDR_1 = "http://" + HTTP_ADDRESS_21 + "/webhdfs";
         final String EXPECTED_ADDR_2 = "http://" + HTTP_ADDRESS_22 + "/webhdfs";
 
-        AmbariComponent namenode = EasyMock.createNiceMock(AmbariComponent.class);
-        EasyMock.expect(namenode.getConfigProperty("dfs.nameservices")).andReturn(NAMESERVICES).anyTimes();
-        EasyMock.replay(namenode);
-
         AmbariCluster.ServiceConfiguration hdfsSC = EasyMock.createNiceMock(AmbariCluster.ServiceConfiguration.class);
         Map<String, String> hdfsProps = new HashMap<>();
+        hdfsProps.put("dfs.nameservices", NAMESERVICES);
         hdfsProps.put("dfs.namenode.http-address." + NS1 + ".nn1", HTTP_ADDRESS_11);
         hdfsProps.put("dfs.namenode.http-address." + NS1 + ".nn2", HTTP_ADDRESS_12);
         hdfsProps.put("dfs.namenode.http-address." + NS2 + ".nn1", HTTP_ADDRESS_21);
@@ -578,7 +565,6 @@ public class AmbariDynamicServiceURLCreatorTest {
         EasyMock.replay(coreSC);
 
         AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class);
-        EasyMock.expect(cluster.getComponent("NAMENODE")).andReturn(namenode).anyTimes();
         EasyMock.expect(cluster.getServiceConfiguration("HDFS", "hdfs-site")).andReturn(hdfsSC).anyTimes();
         EasyMock.expect(cluster.getServiceConfiguration("HDFS", "core-site")).andReturn(coreSC).anyTimes();
         EasyMock.replay(cluster);
@@ -607,12 +593,9 @@ public class AmbariDynamicServiceURLCreatorTest {
         final String EXPECTED_ADDR_1 = "http://" + HTTP_ADDRESS_21;
         final String EXPECTED_ADDR_2 = "http://" + HTTP_ADDRESS_22;
 
-        AmbariComponent namenode = EasyMock.createNiceMock(AmbariComponent.class);
-        EasyMock.expect(namenode.getConfigProperty("dfs.nameservices")).andReturn(NAMESERVICES).anyTimes();
-        EasyMock.replay(namenode);
-
         AmbariCluster.ServiceConfiguration hdfsSC = EasyMock.createNiceMock(AmbariCluster.ServiceConfiguration.class);
         Map<String, String> hdfsProps = new HashMap<>();
+        hdfsProps.put("dfs.nameservices", NAMESERVICES);
         hdfsProps.put("dfs.namenode.http-address." + NS1 + ".nn1", HTTP_ADDRESS_11);
         hdfsProps.put("dfs.namenode.http-address." + NS1 + ".nn2", HTTP_ADDRESS_12);
         hdfsProps.put("dfs.namenode.http-address." + NS2 + ".nn1", HTTP_ADDRESS_21);
@@ -627,7 +610,6 @@ public class AmbariDynamicServiceURLCreatorTest {
         EasyMock.replay(coreSC);
 
         AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class);
-        EasyMock.expect(cluster.getComponent("NAMENODE")).andReturn(namenode).anyTimes();
         EasyMock.expect(cluster.getServiceConfiguration("HDFS", "hdfs-site")).andReturn(hdfsSC).anyTimes();
         EasyMock.expect(cluster.getServiceConfiguration("HDFS", "core-site")).andReturn(coreSC).anyTimes();
         EasyMock.replay(cluster);
@@ -658,12 +640,9 @@ public class AmbariDynamicServiceURLCreatorTest {
         final String EXPECTED_ADDR_1 = "http://" + HTTP_ADDRESS_21 + "/webhdfs";
         final String EXPECTED_ADDR_2 = "http://" + HTTP_ADDRESS_22 + "/webhdfs";
 
-        AmbariComponent namenode = EasyMock.createNiceMock(AmbariComponent.class);
-        EasyMock.expect(namenode.getConfigProperty("dfs.nameservices")).andReturn(NAMESERVICES).anyTimes();
-        EasyMock.replay(namenode);
-
         AmbariCluster.ServiceConfiguration hdfsSC = EasyMock.createNiceMock(AmbariCluster.ServiceConfiguration.class);
         Map<String, String> hdfsProps = new HashMap<>();
+        hdfsProps.put("dfs.nameservices", NAMESERVICES);
         hdfsProps.put("dfs.namenode.http-address." + NS1 + ".nn11", HTTP_ADDRESS_11);
         hdfsProps.put("dfs.namenode.http-address." + NS1 + ".nn12", HTTP_ADDRESS_12);
         hdfsProps.put("dfs.namenode.http-address." + NS2 + ".nn21", HTTP_ADDRESS_21);
@@ -680,7 +659,6 @@ public class AmbariDynamicServiceURLCreatorTest {
         EasyMock.replay(coreSC);
 
         AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class);
-        EasyMock.expect(cluster.getComponent("NAMENODE")).andReturn(namenode).anyTimes();
         EasyMock.expect(cluster.getServiceConfiguration("HDFS", "hdfs-site")).andReturn(hdfsSC).anyTimes();
         EasyMock.expect(cluster.getServiceConfiguration("HDFS", "core-site")).andReturn(coreSC).anyTimes();
         EasyMock.replay(cluster);
@@ -711,12 +689,9 @@ public class AmbariDynamicServiceURLCreatorTest {
         final String EXPECTED_ADDR_1 = "http://" + HTTP_ADDRESS_21;
         final String EXPECTED_ADDR_2 = "http://" + HTTP_ADDRESS_22;
 
-        AmbariComponent namenode = EasyMock.createNiceMock(AmbariComponent.class);
-        EasyMock.expect(namenode.getConfigProperty("dfs.nameservices")).andReturn(NAMESERVICES).anyTimes();
-        EasyMock.replay(namenode);
-
         AmbariCluster.ServiceConfiguration hdfsSC = EasyMock.createNiceMock(AmbariCluster.ServiceConfiguration.class);
         Map<String, String> hdfsProps = new HashMap<>();
+        hdfsProps.put("dfs.nameservices", NAMESERVICES);
         hdfsProps.put("dfs.namenode.http-address." + NS1 + ".nn11", HTTP_ADDRESS_11);
         hdfsProps.put("dfs.namenode.http-address." + NS1 + ".nn12", HTTP_ADDRESS_12);
         hdfsProps.put("dfs.namenode.http-address." + NS2 + ".nn21", HTTP_ADDRESS_21);
@@ -733,7 +708,6 @@ public class AmbariDynamicServiceURLCreatorTest {
         EasyMock.replay(coreSC);
 
         AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class);
-        EasyMock.expect(cluster.getComponent("NAMENODE")).andReturn(namenode).anyTimes();
         EasyMock.expect(cluster.getServiceConfiguration("HDFS", "hdfs-site")).andReturn(hdfsSC).anyTimes();
         EasyMock.expect(cluster.getServiceConfiguration("HDFS", "core-site")).andReturn(coreSC).anyTimes();
         EasyMock.replay(cluster);

http://git-wip-us.apache.org/repos/asf/knox/blob/8a1f03b1/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/WebHdfsUrlCreatorTest.java
----------------------------------------------------------------------
diff --git a/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/WebHdfsUrlCreatorTest.java
b/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/WebHdfsUrlCreatorTest.java
index 46a2474..e2f8db4 100644
--- a/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/WebHdfsUrlCreatorTest.java
+++ b/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/WebHdfsUrlCreatorTest.java
@@ -98,10 +98,6 @@ public class WebHdfsUrlCreatorTest {
     final String HTTP_ADDRESS  = "host1:20070";
     final String HTTPS_ADDRESS = "host2:20470";
 
-    AmbariComponent nnComp = EasyMock.createNiceMock(AmbariComponent.class);
-    EasyMock.expect(nnComp.getConfigProperty("dfs.nameservices")).andReturn("X,Y").anyTimes();
-    EasyMock.replay(nnComp);
-
     AmbariCluster.ServiceConfiguration coreSiteConfig = EasyMock.createNiceMock(AmbariCluster.ServiceConfiguration.class);
     Map<String, String> corSiteProps = new HashMap<>();
     corSiteProps.put("fs.defaultFS", "hdfs://X");
@@ -113,6 +109,7 @@ public class WebHdfsUrlCreatorTest {
     configProps.put(HDFSURLCreatorBase.HTTP_POLICY_PROPERTY, HDFSURLCreatorBase.HTTP_ONLY_POLICY);
     configProps.put(HDFSURLCreatorBase.HTTP_ADDRESS_PROPERTY, HTTP_ADDRESS);
     configProps.put(HDFSURLCreatorBase.HTTPS_ADDRESS_PROPERTY, HTTPS_ADDRESS);
+    configProps.put("dfs.nameservices", "X,Y");
     configProps.put("dfs.ha.namenodes.X", "nn1,nn2");
     configProps.put(HDFSURLCreatorBase.HTTP_ADDRESS_PROPERTY + ".X.nn1", "host3:20070");
     configProps.put(HDFSURLCreatorBase.HTTP_ADDRESS_PROPERTY + ".X.nn2", "host4:20070");
@@ -121,9 +118,6 @@ public class WebHdfsUrlCreatorTest {
     EasyMock.replay(hdfsSiteConfig);
 
     AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class);
-    EasyMock.expect(cluster.getComponent("NAMENODE"))
-            .andReturn(nnComp)
-            .anyTimes();
     EasyMock.expect(cluster.getServiceConfiguration("HDFS", "hdfs-site"))
             .andReturn(hdfsSiteConfig)
             .anyTimes();
@@ -148,6 +142,147 @@ public class WebHdfsUrlCreatorTest {
     final String HTTP_ADDRESS  = "host1:20070";
     final String HTTPS_ADDRESS = "host2:20470";
 
+    AmbariCluster.ServiceConfiguration coreSiteConfig = EasyMock.createNiceMock(AmbariCluster.ServiceConfiguration.class);
+    Map<String, String> corSiteProps = new HashMap<>();
+    corSiteProps.put("fs.defaultFS", "hdfs://Y");
+    EasyMock.expect(coreSiteConfig.getProperties()).andReturn(corSiteProps).anyTimes();
+    EasyMock.replay(coreSiteConfig);
+
+    AmbariCluster.ServiceConfiguration hdfsSiteConfig = EasyMock.createNiceMock(AmbariCluster.ServiceConfiguration.class);
+    Map<String, String> configProps = new HashMap<>();
+    configProps.put(HDFSURLCreatorBase.HTTP_POLICY_PROPERTY, HDFSURLCreatorBase.HTTPS_ONLY_POLICY);
+    configProps.put(HDFSURLCreatorBase.HTTP_ADDRESS_PROPERTY, HTTP_ADDRESS);
+    configProps.put(HDFSURLCreatorBase.HTTPS_ADDRESS_PROPERTY, HTTPS_ADDRESS);
+    configProps.put("dfs.nameservices", "X,Y");
+    configProps.put("dfs.ha.namenodes.Y", "nn7,nn8");
+    configProps.put(HDFSURLCreatorBase.HTTPS_ADDRESS_PROPERTY + ".Y.nn7", "host5:20470");
+    configProps.put(HDFSURLCreatorBase.HTTPS_ADDRESS_PROPERTY + ".Y.nn8", "host6:20470");
+
+    EasyMock.expect(hdfsSiteConfig.getProperties()).andReturn(configProps).anyTimes();
+    EasyMock.replay(hdfsSiteConfig);
+
+    AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class);
+    EasyMock.expect(cluster.getServiceConfiguration("HDFS", "hdfs-site"))
+            .andReturn(hdfsSiteConfig)
+            .anyTimes();
+    EasyMock.expect(cluster.getServiceConfiguration("HDFS", "core-site"))
+            .andReturn(coreSiteConfig)
+            .anyTimes();
+    EasyMock.replay(cluster);
+
+    WebHdfsUrlCreator c = new WebHdfsUrlCreator();
+    c.init(cluster);
+    List<String> urls = c.create("WEBHDFS", null);
+    assertNotNull(urls);
+    assertFalse(urls.isEmpty());
+    assertEquals(2, urls.size());
+    assertTrue(urls.contains("https://" + "host5:20470" + "/webhdfs"));
+    assertTrue(urls.contains("https://" + "host6:20470" + "/webhdfs"));
+  }
+
+
+  @Test
+  public void testFederatedHttpsWebHdfsEndpointAddressWithValidNS() {
+    final String HTTP_ADDRESS  = "host1:20070";
+    final String HTTPS_ADDRESS = "host2:20470";
+
+    AmbariCluster.ServiceConfiguration coreSiteConfig = EasyMock.createNiceMock(AmbariCluster.ServiceConfiguration.class);
+    Map<String, String> corSiteProps = new HashMap<>();
+    corSiteProps.put("fs.defaultFS", "hdfs://Y");
+    EasyMock.expect(coreSiteConfig.getProperties()).andReturn(corSiteProps).anyTimes();
+    EasyMock.replay(coreSiteConfig);
+
+    AmbariCluster.ServiceConfiguration hdfsSiteConfig = EasyMock.createNiceMock(AmbariCluster.ServiceConfiguration.class);
+    Map<String, String> configProps = new HashMap<>();
+    configProps.put(HDFSURLCreatorBase.HTTP_POLICY_PROPERTY, HDFSURLCreatorBase.HTTPS_ONLY_POLICY);
+    configProps.put(HDFSURLCreatorBase.HTTP_ADDRESS_PROPERTY, HTTP_ADDRESS);
+    configProps.put(HDFSURLCreatorBase.HTTPS_ADDRESS_PROPERTY, HTTPS_ADDRESS);
+    configProps.put("dfs.nameservices", "X,Y");
+    configProps.put("dfs.ha.namenodes.X", "nn2,nn3");
+    configProps.put("dfs.ha.namenodes.Y", "nn7,nn8");
+    configProps.put(HDFSURLCreatorBase.HTTPS_ADDRESS_PROPERTY + ".X.nn2", "host3:20470");
+    configProps.put(HDFSURLCreatorBase.HTTPS_ADDRESS_PROPERTY + ".X.nn3", "host4:20470");
+    configProps.put(HDFSURLCreatorBase.HTTPS_ADDRESS_PROPERTY + ".Y.nn7", "host5:20470");
+    configProps.put(HDFSURLCreatorBase.HTTPS_ADDRESS_PROPERTY + ".Y.nn8", "host6:20470");
+
+    EasyMock.expect(hdfsSiteConfig.getProperties()).andReturn(configProps).anyTimes();
+    EasyMock.replay(hdfsSiteConfig);
+
+    AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class);
+    EasyMock.expect(cluster.getServiceConfiguration("HDFS", "hdfs-site"))
+            .andReturn(hdfsSiteConfig)
+            .anyTimes();
+    EasyMock.expect(cluster.getServiceConfiguration("HDFS", "core-site"))
+            .andReturn(coreSiteConfig)
+            .anyTimes();
+    EasyMock.replay(cluster);
+
+    Map<String, String> serviceParams = new HashMap<>();
+    serviceParams.put("discovery-nameservice", "X");
+
+    WebHdfsUrlCreator c = new WebHdfsUrlCreator();
+    c.init(cluster);
+    List<String> urls = c.create("WEBHDFS", serviceParams);
+    assertNotNull(urls);
+    assertFalse(urls.isEmpty());
+    assertEquals(2, urls.size());
+    assertTrue(urls.contains("https://" + "host3:20470" + "/webhdfs"));
+    assertTrue(urls.contains("https://" + "host4:20470" + "/webhdfs"));
+  }
+
+
+  @Test
+  public void testFederatedHttpsWebHdfsEndpointAddressWithInvalidNS() {
+    final String HTTP_ADDRESS  = "host1:20070";
+    final String HTTPS_ADDRESS = "host2:20470";
+
+    AmbariCluster.ServiceConfiguration coreSiteConfig = EasyMock.createNiceMock(AmbariCluster.ServiceConfiguration.class);
+    Map<String, String> corSiteProps = new HashMap<>();
+    corSiteProps.put("fs.defaultFS", "hdfs://Y");
+    EasyMock.expect(coreSiteConfig.getProperties()).andReturn(corSiteProps).anyTimes();
+    EasyMock.replay(coreSiteConfig);
+
+    AmbariCluster.ServiceConfiguration hdfsSiteConfig = EasyMock.createNiceMock(AmbariCluster.ServiceConfiguration.class);
+    Map<String, String> configProps = new HashMap<>();
+    configProps.put(HDFSURLCreatorBase.HTTP_POLICY_PROPERTY, HDFSURLCreatorBase.HTTPS_ONLY_POLICY);
+    configProps.put(HDFSURLCreatorBase.HTTP_ADDRESS_PROPERTY, HTTP_ADDRESS);
+    configProps.put(HDFSURLCreatorBase.HTTPS_ADDRESS_PROPERTY, HTTPS_ADDRESS);
+    configProps.put("dfs.nameservices", "X,Y");
+    configProps.put("dfs.ha.namenodes.X", "nn2,nn3");
+    configProps.put("dfs.ha.namenodes.Y", "nn7,nn8");
+    configProps.put(HDFSURLCreatorBase.HTTPS_ADDRESS_PROPERTY + ".X.nn2", "host3:20470");
+    configProps.put(HDFSURLCreatorBase.HTTPS_ADDRESS_PROPERTY + ".X.nn3", "host4:20470");
+    configProps.put(HDFSURLCreatorBase.HTTPS_ADDRESS_PROPERTY + ".Y.nn7", "host5:20470");
+    configProps.put(HDFSURLCreatorBase.HTTPS_ADDRESS_PROPERTY + ".Y.nn8", "host6:20470");
+
+    EasyMock.expect(hdfsSiteConfig.getProperties()).andReturn(configProps).anyTimes();
+    EasyMock.replay(hdfsSiteConfig);
+
+    AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class);
+    EasyMock.expect(cluster.getServiceConfiguration("HDFS", "hdfs-site"))
+            .andReturn(hdfsSiteConfig)
+            .anyTimes();
+    EasyMock.expect(cluster.getServiceConfiguration("HDFS", "core-site"))
+            .andReturn(coreSiteConfig)
+            .anyTimes();
+    EasyMock.replay(cluster);
+
+    Map<String, String> serviceParams = new HashMap<>();
+    serviceParams.put("discovery-nameservice", "Z");
+
+    WebHdfsUrlCreator c = new WebHdfsUrlCreator();
+    c.init(cluster);
+    List<String> urls = c.create("WEBHDFS", serviceParams);
+    assertNotNull(urls);
+    assertTrue(urls.isEmpty());
+  }
+
+
+  @Test
+  public void testFederatedHttpsNamenodeEndpointAddressWithValidDeclaredNS() {
+    final String HTTP_ADDRESS  = "host1:20070";
+    final String HTTPS_ADDRESS = "host2:20470";
+
     AmbariComponent nnComp = EasyMock.createNiceMock(AmbariComponent.class);
     EasyMock.expect(nnComp.getConfigProperty("dfs.nameservices")).andReturn("X,Y").anyTimes();
     EasyMock.replay(nnComp);
@@ -163,7 +298,10 @@ public class WebHdfsUrlCreatorTest {
     configProps.put(HDFSURLCreatorBase.HTTP_POLICY_PROPERTY, HDFSURLCreatorBase.HTTPS_ONLY_POLICY);
     configProps.put(HDFSURLCreatorBase.HTTP_ADDRESS_PROPERTY, HTTP_ADDRESS);
     configProps.put(HDFSURLCreatorBase.HTTPS_ADDRESS_PROPERTY, HTTPS_ADDRESS);
+    configProps.put("dfs.ha.namenodes.X", "nn2,nn3");
     configProps.put("dfs.ha.namenodes.Y", "nn7,nn8");
+    configProps.put(HDFSURLCreatorBase.HTTPS_ADDRESS_PROPERTY + ".X.nn2", "host3:20470");
+    configProps.put(HDFSURLCreatorBase.HTTPS_ADDRESS_PROPERTY + ".X.nn3", "host4:20470");
     configProps.put(HDFSURLCreatorBase.HTTPS_ADDRESS_PROPERTY + ".Y.nn7", "host5:20470");
     configProps.put(HDFSURLCreatorBase.HTTPS_ADDRESS_PROPERTY + ".Y.nn8", "host6:20470");
 
@@ -182,14 +320,68 @@ public class WebHdfsUrlCreatorTest {
             .anyTimes();
     EasyMock.replay(cluster);
 
-    WebHdfsUrlCreator c = new WebHdfsUrlCreator();
+    NameNodeUrlCreator c = new NameNodeUrlCreator();
     c.init(cluster);
-    List<String> urls = c.create("WEBHDFS", null);
+
+    Map<String, String> serviceParams = new HashMap<>();
+    serviceParams.put("discovery-nameservice", "X");
+
+    List<String> urls = c.create("NAMENODE", serviceParams);
     assertNotNull(urls);
     assertFalse(urls.isEmpty());
-    assertEquals(2, urls.size());
-    assertTrue(urls.contains("https://" + "host5:20470" + "/webhdfs"));
-    assertTrue(urls.contains("https://" + "host6:20470" + "/webhdfs"));
+    assertEquals(1, urls.size());
+    assertTrue(urls.contains("hdfs://X"));
+  }
+
+
+  @Test
+  public void testFederatedHttpsNamenodeEndpointAddressWithInvalidDeclaredNS() {
+    final String HTTP_ADDRESS  = "host1:20070";
+    final String HTTPS_ADDRESS = "host2:20470";
+
+    AmbariComponent nnComp = EasyMock.createNiceMock(AmbariComponent.class);
+    EasyMock.expect(nnComp.getConfigProperty("dfs.nameservices")).andReturn("X,Y").anyTimes();
+    EasyMock.replay(nnComp);
+
+    AmbariCluster.ServiceConfiguration coreSiteConfig = EasyMock.createNiceMock(AmbariCluster.ServiceConfiguration.class);
+    Map<String, String> corSiteProps = new HashMap<>();
+    corSiteProps.put("fs.defaultFS", "hdfs://Y");
+    EasyMock.expect(coreSiteConfig.getProperties()).andReturn(corSiteProps).anyTimes();
+    EasyMock.replay(coreSiteConfig);
+
+    AmbariCluster.ServiceConfiguration hdfsSiteConfig = EasyMock.createNiceMock(AmbariCluster.ServiceConfiguration.class);
+    Map<String, String> configProps = new HashMap<>();
+    configProps.put(HDFSURLCreatorBase.HTTP_POLICY_PROPERTY, HDFSURLCreatorBase.HTTPS_ONLY_POLICY);
+    configProps.put(HDFSURLCreatorBase.HTTP_ADDRESS_PROPERTY, HTTP_ADDRESS);
+    configProps.put(HDFSURLCreatorBase.HTTPS_ADDRESS_PROPERTY, HTTPS_ADDRESS);
+    configProps.put("dfs.ha.namenodes.Y", "nn7,nn8");
+    configProps.put(HDFSURLCreatorBase.HTTPS_ADDRESS_PROPERTY + ".Y.nn7", "host5:20470");
+    configProps.put(HDFSURLCreatorBase.HTTPS_ADDRESS_PROPERTY + ".Y.nn8", "host6:20470");
+
+    EasyMock.expect(hdfsSiteConfig.getProperties()).andReturn(configProps).anyTimes();
+    EasyMock.replay(hdfsSiteConfig);
+
+    AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class);
+    EasyMock.expect(cluster.getComponent("NAMENODE"))
+            .andReturn(nnComp)
+            .anyTimes();
+    EasyMock.expect(cluster.getServiceConfiguration("HDFS", "hdfs-site"))
+            .andReturn(hdfsSiteConfig)
+            .anyTimes();
+    EasyMock.expect(cluster.getServiceConfiguration("HDFS", "core-site"))
+            .andReturn(coreSiteConfig)
+            .anyTimes();
+    EasyMock.replay(cluster);
+
+    NameNodeUrlCreator c = new NameNodeUrlCreator();
+    c.init(cluster);
+
+    Map<String, String> serviceParams = new HashMap<>();
+    serviceParams.put("discovery-nameservice", "Z");
+
+    List<String> urls = c.create("NAMENODE", serviceParams);
+    assertNotNull(urls);
+    assertTrue(urls.isEmpty());
   }
 
 }

http://git-wip-us.apache.org/repos/asf/knox/blob/8a1f03b1/gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorHandler.java
----------------------------------------------------------------------
diff --git a/gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorHandler.java
b/gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorHandler.java
index 28a8094..89020d8 100644
--- a/gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorHandler.java
+++ b/gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorHandler.java
@@ -158,10 +158,22 @@ public class SimpleDescriptorHandler {
             }
 
             // Service params
-            if (descService.getParams() != null) {
-                serviceParams.put(serviceName, descService.getParams());
-                if (!validServiceNames.contains(serviceName)) {
-                    validServiceNames.add(serviceName);
+            Map<String, String> descriptorServiceParams = descService.getParams();
+            if (descriptorServiceParams != null && !descriptorServiceParams.isEmpty())
{
+                boolean hasNonDiscoveryParams = false;
+                // Determine if there are any params which are not discovery-only
+                for (String paramName : descriptorServiceParams.keySet()) {
+                    if (!paramName.startsWith(DISCOVERY_PARAM_PREFIX)) {
+                        hasNonDiscoveryParams = true;
+                        break;
+                    }
+                }
+                // Don't add the service if the only params are discovery-only params
+                if (hasNonDiscoveryParams) {
+                    serviceParams.put(serviceName, descService.getParams());
+                    if (!validServiceNames.contains(serviceName)) {
+                        validServiceNames.add(serviceName);
+                    }
                 }
             }
         }


Mime
View raw message