knox-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lmc...@apache.org
Subject [24/37] knox git commit: KNOX-1064 - Externalize Hadoop Service Configuration Details and Service URL Creation (Phil Zampino via Sandeep More)
Date Thu, 26 Oct 2017 14:23:00 GMT
http://git-wip-us.apache.org/repos/asf/knox/blob/7b401def/gateway-discovery-ambari/src/test/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariDynamicServiceURLCreatorTest.java
----------------------------------------------------------------------
diff --git a/gateway-discovery-ambari/src/test/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariDynamicServiceURLCreatorTest.java b/gateway-discovery-ambari/src/test/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariDynamicServiceURLCreatorTest.java
new file mode 100644
index 0000000..dd35dbb
--- /dev/null
+++ b/gateway-discovery-ambari/src/test/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariDynamicServiceURLCreatorTest.java
@@ -0,0 +1,876 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.gateway.topology.discovery.ambari;
+
+import org.apache.commons.io.FileUtils;
+import org.easymock.EasyMock;
+import org.junit.Test;
+
+import java.io.File;
+import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import static junit.framework.TestCase.assertTrue;
+import static junit.framework.TestCase.fail;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+
+public class AmbariDynamicServiceURLCreatorTest {
+
+    @Test
+    public void testHiveURLFromInternalMapping() throws Exception {
+        testHiveURL(null);
+    }
+
+    @Test
+    public void testHiveURLFromExternalMapping() throws Exception {
+        testHiveURL(TEST_MAPPING_CONFIG);
+    }
+
+    private void testHiveURL(Object mappingConfiguration) throws Exception {
+
+        final String   SERVICE_NAME = "HIVE";
+        final String[] HOSTNAMES    = {"host3", "host2", "host4"};
+        final String   HTTP_PATH    = "cliservice";
+        final String   HTTP_PORT    = "10001";
+        final String   BINARY_PORT  = "10000";
+
+        String expectedScheme = "http";
+
+        final List<String> hiveServerHosts = Arrays.asList(HOSTNAMES);
+
+        AmbariComponent hiveServer = EasyMock.createNiceMock(AmbariComponent.class);
+
+        AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class);
+        EasyMock.expect(cluster.getComponent("HIVE_SERVER")).andReturn(hiveServer).anyTimes();
+        EasyMock.replay(cluster);
+
+        // Configure HTTP Transport
+        EasyMock.expect(hiveServer.getHostNames()).andReturn(hiveServerHosts).anyTimes();
+        EasyMock.expect(hiveServer.getConfigProperty("hive.server2.use.SSL")).andReturn("false").anyTimes();
+        EasyMock.expect(hiveServer.getConfigProperty("hive.server2.thrift.http.path")).andReturn(HTTP_PATH).anyTimes();
+        EasyMock.expect(hiveServer.getConfigProperty("hive.server2.thrift.http.port")).andReturn(HTTP_PORT).anyTimes();
+        EasyMock.expect(hiveServer.getConfigProperty("hive.server2.transport.mode")).andReturn("http").anyTimes();
+        EasyMock.replay(hiveServer);
+
+        // Run the test
+        AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, mappingConfiguration);
+        List<String> urls = builder.create(SERVICE_NAME);
+        assertEquals(HOSTNAMES.length, urls.size());
+        validateServiceURLs(urls, HOSTNAMES, expectedScheme, HTTP_PORT, HTTP_PATH);
+
+        // Configure BINARY Transport
+        EasyMock.reset(hiveServer);
+        EasyMock.expect(hiveServer.getHostNames()).andReturn(hiveServerHosts).anyTimes();
+        EasyMock.expect(hiveServer.getConfigProperty("hive.server2.use.SSL")).andReturn("false").anyTimes();
+        EasyMock.expect(hiveServer.getConfigProperty("hive.server2.thrift.http.path")).andReturn("").anyTimes();
+        EasyMock.expect(hiveServer.getConfigProperty("hive.server2.thrift.http.port")).andReturn(HTTP_PORT).anyTimes();
+        EasyMock.expect(hiveServer.getConfigProperty("hive.server2.thrift.port")).andReturn(BINARY_PORT).anyTimes();
+        EasyMock.expect(hiveServer.getConfigProperty("hive.server2.transport.mode")).andReturn("binary").anyTimes();
+        EasyMock.replay(hiveServer);
+
+        // Run the test
+        urls = builder.create(SERVICE_NAME);
+        assertEquals(HOSTNAMES.length, urls.size());
+        validateServiceURLs(urls, HOSTNAMES, expectedScheme, HTTP_PORT, "");
+
+        // Configure HTTPS Transport
+        EasyMock.reset(hiveServer);
+        EasyMock.expect(hiveServer.getHostNames()).andReturn(hiveServerHosts).anyTimes();
+        EasyMock.expect(hiveServer.getConfigProperty("hive.server2.use.SSL")).andReturn("true").anyTimes();
+        EasyMock.expect(hiveServer.getConfigProperty("hive.server2.thrift.http.path")).andReturn(HTTP_PATH).anyTimes();
+        EasyMock.expect(hiveServer.getConfigProperty("hive.server2.thrift.http.port")).andReturn(HTTP_PORT).anyTimes();
+        EasyMock.expect(hiveServer.getConfigProperty("hive.server2.transport.mode")).andReturn("http").anyTimes();
+        EasyMock.replay(hiveServer);
+
+        // Run the test
+        expectedScheme = "https";
+        urls = builder.create(SERVICE_NAME);
+        assertEquals(HOSTNAMES.length, urls.size());
+        validateServiceURLs(urls, HOSTNAMES, expectedScheme, HTTP_PORT, HTTP_PATH);
+    }
+
+    @Test
+    public void testResourceManagerURLFromInternalMapping() throws Exception {
+        testResourceManagerURL(null);
+    }
+
+    @Test
+    public void testResourceManagerURLFromExternalMapping() throws Exception {
+        testResourceManagerURL(TEST_MAPPING_CONFIG);
+    }
+
+    private void testResourceManagerURL(Object mappingConfiguration) throws Exception {
+
+        final String HTTP_ADDRESS  = "host2:1111";
+        final String HTTPS_ADDRESS = "host2:22222";
+
+        // HTTP
+        AmbariComponent resman = EasyMock.createNiceMock(AmbariComponent.class);
+        setResourceManagerComponentExpectations(resman, HTTP_ADDRESS, HTTPS_ADDRESS, "HTTP");
+
+        AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class);
+        EasyMock.expect(cluster.getComponent("RESOURCEMANAGER")).andReturn(resman).anyTimes();
+        EasyMock.replay(cluster);
+
+        // Run the test
+        AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, mappingConfiguration);
+        String url = builder.create("RESOURCEMANAGER").get(0);
+        assertEquals("http://" + HTTP_ADDRESS + "/ws", url);
+
+        // HTTPS
+        EasyMock.reset(resman);
+        setResourceManagerComponentExpectations(resman, HTTP_ADDRESS, HTTPS_ADDRESS, "HTTPS_ONLY");
+
+        // Run the test
+        url = builder.create("RESOURCEMANAGER").get(0);
+        assertEquals("https://" + HTTPS_ADDRESS + "/ws", url);
+    }
+
+    private void setResourceManagerComponentExpectations(final AmbariComponent resmanMock,
+                                                         final String          httpAddress,
+                                                         final String          httpsAddress,
+                                                         final String          httpPolicy) {
+        EasyMock.expect(resmanMock.getConfigProperty("yarn.resourcemanager.webapp.address")).andReturn(httpAddress).anyTimes();
+        EasyMock.expect(resmanMock.getConfigProperty("yarn.resourcemanager.webapp.https.address")).andReturn(httpsAddress).anyTimes();
+        EasyMock.expect(resmanMock.getConfigProperty("yarn.http.policy")).andReturn(httpPolicy).anyTimes();
+        EasyMock.replay(resmanMock);
+    }
+
+    @Test
+    public void testJobTrackerURLFromInternalMapping() throws Exception {
+        testJobTrackerURL(null);
+    }
+
+    @Test
+    public void testJobTrackerURLFromExternalMapping() throws Exception {
+        testJobTrackerURL(TEST_MAPPING_CONFIG);
+    }
+
+    private void testJobTrackerURL(Object mappingConfiguration) throws Exception {
+        final String ADDRESS = "host2:5678";
+
+        AmbariComponent resman = EasyMock.createNiceMock(AmbariComponent.class);
+        EasyMock.expect(resman.getConfigProperty("yarn.resourcemanager.address")).andReturn(ADDRESS).anyTimes();
+        EasyMock.replay(resman);
+
+        AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class);
+        EasyMock.expect(cluster.getComponent("RESOURCEMANAGER")).andReturn(resman).anyTimes();
+        EasyMock.replay(cluster);
+
+        // Run the test
+        AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, mappingConfiguration);
+        String url = builder.create("JOBTRACKER").get(0);
+        assertEquals("rpc://" + ADDRESS, url);
+    }
+
+    @Test
+    public void testNameNodeURLFromInternalMapping() throws Exception {
+        testNameNodeURL(null);
+    }
+
+    @Test
+    public void testNameNodeURLFromExternalMapping() throws Exception {
+        testNameNodeURL(TEST_MAPPING_CONFIG);
+    }
+
+    private void testNameNodeURL(Object mappingConfiguration) throws Exception {
+        final String ADDRESS = "host1:1234";
+
+        AmbariComponent namenode = EasyMock.createNiceMock(AmbariComponent.class);
+        EasyMock.expect(namenode.getConfigProperty("dfs.namenode.rpc-address")).andReturn(ADDRESS).anyTimes();
+        EasyMock.replay(namenode);
+
+        AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class);
+        EasyMock.expect(cluster.getComponent("NAMENODE")).andReturn(namenode).anyTimes();
+        EasyMock.replay(cluster);
+
+        // Run the test
+        AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, mappingConfiguration);
+        String url = builder.create("NAMENODE").get(0);
+        assertEquals("hdfs://" + ADDRESS, url);
+    }
+
+    @Test
+    public void testWebHCatURLFromInternalMapping() throws Exception {
+        testWebHCatURL(null);
+    }
+
+    @Test
+    public void testWebHCatURLFromExternalMapping() throws Exception {
+        testWebHCatURL(TEST_MAPPING_CONFIG);
+    }
+
+    private void testWebHCatURL(Object mappingConfiguration) throws Exception {
+
+        final String HOSTNAME = "host3";
+        final String PORT     = "1919";
+
+        AmbariComponent webhcatServer = EasyMock.createNiceMock(AmbariComponent.class);
+        EasyMock.expect(webhcatServer.getConfigProperty("templeton.port")).andReturn(PORT).anyTimes();
+        List<String> webHcatServerHosts = Collections.singletonList(HOSTNAME);
+        EasyMock.expect(webhcatServer.getHostNames()).andReturn(webHcatServerHosts).anyTimes();
+        EasyMock.replay(webhcatServer);
+
+        AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class);
+        EasyMock.expect(cluster.getComponent("WEBHCAT_SERVER")).andReturn(webhcatServer).anyTimes();
+        EasyMock.replay(cluster);
+
+        // Run the test
+        AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, mappingConfiguration);
+        String url = builder.create("WEBHCAT").get(0);
+        assertEquals("http://" + HOSTNAME + ":" + PORT + "/templeton", url);
+    }
+
+    @Test
+    public void testOozieURLFromInternalMapping() throws Exception {
+        testOozieURL(null);
+    }
+
+    @Test
+    public void testOozieURLFromExternalMapping() throws Exception {
+        testOozieURL(TEST_MAPPING_CONFIG);
+    }
+
+    private void testOozieURL(Object mappingConfiguration) throws Exception {
+        final String URL = "http://host3:2222";
+
+        AmbariComponent oozieServer = EasyMock.createNiceMock(AmbariComponent.class);
+        EasyMock.expect(oozieServer.getConfigProperty("oozie.base.url")).andReturn(URL).anyTimes();
+        EasyMock.replay(oozieServer);
+
+        AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class);
+        EasyMock.expect(cluster.getComponent("OOZIE_SERVER")).andReturn(oozieServer).anyTimes();
+        EasyMock.replay(cluster);
+
+        // Run the test
+        AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, mappingConfiguration);
+        String url = builder.create("OOZIE").get(0);
+        assertEquals(URL, url);
+    }
+
+    @Test
+    public void testWebHBaseURLFromInternalMapping() throws Exception {
+        testWebHBaseURL(null);
+    }
+
+    @Test
+    public void testWebHBaseURLFromExternalMapping() throws Exception {
+        testWebHBaseURL(TEST_MAPPING_CONFIG);
+    }
+
+    private void testWebHBaseURL(Object mappingConfiguration) throws Exception {
+        final String[] HOSTNAMES = {"host2", "host4"};
+
+        AmbariComponent hbaseMaster = EasyMock.createNiceMock(AmbariComponent.class);
+        List<String> hbaseMasterHosts = Arrays.asList(HOSTNAMES);
+        EasyMock.expect(hbaseMaster.getHostNames()).andReturn(hbaseMasterHosts).anyTimes();
+        EasyMock.replay(hbaseMaster);
+
+        AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class);
+        EasyMock.expect(cluster.getComponent("HBASE_MASTER")).andReturn(hbaseMaster).anyTimes();
+        EasyMock.replay(cluster);
+
+        // Run the test
+        AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, mappingConfiguration);
+        List<String> urls = builder.create("WEBHBASE");
+        validateServiceURLs(urls, HOSTNAMES, "http", "60080", null);
+    }
+
+    @Test
+    public void testWebHdfsURLFromInternalMapping() throws Exception {
+        testWebHdfsURL(null);
+    }
+
+    @Test
+    public void testWebHdfsURLFromExternalMapping() throws Exception {
+        testWebHdfsURL(TEST_MAPPING_CONFIG);
+    }
+
+    @Test
+    public void testWebHdfsURLFromSystemPropertyOverride() throws Exception {
+        // Write the test mapping configuration to a temp file
+        File mappingFile = File.createTempFile("mapping-config", "xml");
+        FileUtils.write(mappingFile, OVERRIDE_MAPPING_FILE_CONTENTS, "utf-8");
+
+        // Set the system property to point to the temp file
+        System.setProperty(AmbariDynamicServiceURLCreator.MAPPING_CONFIG_OVERRIDE_PROPERTY,
+                           mappingFile.getAbsolutePath());
+        try {
+            final String ADDRESS = "host3:1357";
+            // The URL creator should apply the file contents, and create the URL accordingly
+            String url = getTestWebHdfsURL(ADDRESS, null);
+
+            // Verify the URL matches the pattern from the file
+            assertEquals("http://" + ADDRESS + "/webhdfs/OVERRIDE", url);
+        } finally {
+            // Reset the system property, and delete the temp file
+            System.clearProperty(AmbariDynamicServiceURLCreator.MAPPING_CONFIG_OVERRIDE_PROPERTY);
+            mappingFile.delete();
+        }
+    }
+
+    private void testWebHdfsURL(Object mappingConfiguration) throws Exception {
+        final String ADDRESS = "host3:1357";
+        assertEquals("http://" + ADDRESS + "/webhdfs", getTestWebHdfsURL(ADDRESS, mappingConfiguration));
+    }
+
+
+    private String getTestWebHdfsURL(String address, Object mappingConfiguration) throws Exception {
+        AmbariCluster.ServiceConfiguration hdfsSC = EasyMock.createNiceMock(AmbariCluster.ServiceConfiguration.class);
+        Map<String, String> hdfsProps = new HashMap<>();
+        hdfsProps.put("dfs.namenode.http-address", address);
+        EasyMock.expect(hdfsSC.getProperties()).andReturn(hdfsProps).anyTimes();
+        EasyMock.replay(hdfsSC);
+
+        AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class);
+        EasyMock.expect(cluster.getServiceConfiguration("HDFS", "hdfs-site")).andReturn(hdfsSC).anyTimes();
+        EasyMock.replay(cluster);
+
+        // Create the URL
+        AmbariDynamicServiceURLCreator creator = newURLCreator(cluster, mappingConfiguration);
+        return creator.create("WEBHDFS").get(0);
+    }
+
+
+    @Test
+    public void testAtlasApiURL() throws Exception {
+        final String ATLAS_REST_ADDRESS = "http://host2:21000";
+
+        AmbariComponent atlasServer = EasyMock.createNiceMock(AmbariComponent.class);
+        EasyMock.expect(atlasServer.getConfigProperty("atlas.rest.address")).andReturn(ATLAS_REST_ADDRESS).anyTimes();
+        EasyMock.replay(atlasServer);
+
+        AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class);
+        EasyMock.expect(cluster.getComponent("ATLAS_SERVER")).andReturn(atlasServer).anyTimes();
+        EasyMock.replay(cluster);
+
+        // Run the test
+        AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, null);
+        List<String> urls = builder.create("ATLAS-API");
+        assertEquals(1, urls.size());
+        assertEquals(ATLAS_REST_ADDRESS, urls.get(0));
+    }
+
+
+    @Test
+    public void testAtlasURL() throws Exception {
+        final String HTTP_PORT = "8787";
+        final String HTTPS_PORT = "8989";
+
+        final String[] HOSTNAMES = {"host1", "host4"};
+        final List<String> atlastServerHosts = Arrays.asList(HOSTNAMES);
+
+        AmbariComponent atlasServer = EasyMock.createNiceMock(AmbariComponent.class);
+        EasyMock.expect(atlasServer.getHostNames()).andReturn(atlastServerHosts).anyTimes();
+        EasyMock.expect(atlasServer.getConfigProperty("atlas.enableTLS")).andReturn("false").anyTimes();
+        EasyMock.expect(atlasServer.getConfigProperty("atlas.server.http.port")).andReturn(HTTP_PORT).anyTimes();
+        EasyMock.expect(atlasServer.getConfigProperty("atlas.server.https.port")).andReturn(HTTPS_PORT).anyTimes();
+        EasyMock.replay(atlasServer);
+
+        AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class);
+        EasyMock.expect(cluster.getComponent("ATLAS_SERVER")).andReturn(atlasServer).anyTimes();
+        EasyMock.replay(cluster);
+
+        // Run the test
+        AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, null);
+        List<String> urls = builder.create("ATLAS");
+        validateServiceURLs(urls, HOSTNAMES, "http", HTTP_PORT, null);
+
+        EasyMock.reset(atlasServer);
+        EasyMock.expect(atlasServer.getHostNames()).andReturn(atlastServerHosts).anyTimes();
+        EasyMock.expect(atlasServer.getConfigProperty("atlas.enableTLS")).andReturn("true").anyTimes();
+        EasyMock.expect(atlasServer.getConfigProperty("atlas.server.http.port")).andReturn(HTTP_PORT).anyTimes();
+        EasyMock.expect(atlasServer.getConfigProperty("atlas.server.https.port")).andReturn(HTTPS_PORT).anyTimes();
+        EasyMock.replay(atlasServer);
+
+        // Run the test
+        urls = builder.create("ATLAS");
+        validateServiceURLs(urls, HOSTNAMES, "https", HTTPS_PORT, null);
+    }
+
+
+    @Test
+    public void testZeppelinURL() throws Exception {
+        final String HTTP_PORT = "8787";
+        final String HTTPS_PORT = "8989";
+
+        final String[] HOSTNAMES = {"host1", "host4"};
+        final List<String> atlastServerHosts = Arrays.asList(HOSTNAMES);
+
+        AmbariComponent zeppelinMaster = EasyMock.createNiceMock(AmbariComponent.class);
+        EasyMock.expect(zeppelinMaster.getHostNames()).andReturn(atlastServerHosts).anyTimes();
+        EasyMock.expect(zeppelinMaster.getConfigProperty("zeppelin.ssl")).andReturn("false").anyTimes();
+        EasyMock.expect(zeppelinMaster.getConfigProperty("zeppelin.server.port")).andReturn(HTTP_PORT).anyTimes();
+        EasyMock.expect(zeppelinMaster.getConfigProperty("zeppelin.server.ssl.port")).andReturn(HTTPS_PORT).anyTimes();
+        EasyMock.replay(zeppelinMaster);
+
+        AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class);
+        EasyMock.expect(cluster.getComponent("ZEPPELIN_MASTER")).andReturn(zeppelinMaster).anyTimes();
+        EasyMock.replay(cluster);
+
+        AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, null);
+
+        // Run the test
+        validateServiceURLs(builder.create("ZEPPELIN"), HOSTNAMES, "http", HTTP_PORT, null);
+
+        EasyMock.reset(zeppelinMaster);
+        EasyMock.expect(zeppelinMaster.getHostNames()).andReturn(atlastServerHosts).anyTimes();
+        EasyMock.expect(zeppelinMaster.getConfigProperty("zeppelin.ssl")).andReturn("true").anyTimes();
+        EasyMock.expect(zeppelinMaster.getConfigProperty("zeppelin.server.port")).andReturn(HTTP_PORT).anyTimes();
+        EasyMock.expect(zeppelinMaster.getConfigProperty("zeppelin.server.ssl.port")).andReturn(HTTPS_PORT).anyTimes();
+        EasyMock.replay(zeppelinMaster);
+
+        // Run the test
+        validateServiceURLs(builder.create("ZEPPELIN"), HOSTNAMES, "https", HTTPS_PORT, null);
+    }
+
+
+    @Test
+    public void testZeppelinUiURL() throws Exception {
+        final String HTTP_PORT = "8787";
+        final String HTTPS_PORT = "8989";
+
+        final String[] HOSTNAMES = {"host1", "host4"};
+        final List<String> atlastServerHosts = Arrays.asList(HOSTNAMES);
+
+        AmbariComponent zeppelinMaster = EasyMock.createNiceMock(AmbariComponent.class);
+        EasyMock.expect(zeppelinMaster.getHostNames()).andReturn(atlastServerHosts).anyTimes();
+        EasyMock.expect(zeppelinMaster.getConfigProperty("zeppelin.ssl")).andReturn("false").anyTimes();
+        EasyMock.expect(zeppelinMaster.getConfigProperty("zeppelin.server.port")).andReturn(HTTP_PORT).anyTimes();
+        EasyMock.expect(zeppelinMaster.getConfigProperty("zeppelin.server.ssl.port")).andReturn(HTTPS_PORT).anyTimes();
+        EasyMock.replay(zeppelinMaster);
+
+        AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class);
+        EasyMock.expect(cluster.getComponent("ZEPPELIN_MASTER")).andReturn(zeppelinMaster).anyTimes();
+        EasyMock.replay(cluster);
+
+        AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, null);
+
+        // Run the test
+        validateServiceURLs(builder.create("ZEPPELINUI"), HOSTNAMES, "http", HTTP_PORT, null);
+
+        EasyMock.reset(zeppelinMaster);
+        EasyMock.expect(zeppelinMaster.getHostNames()).andReturn(atlastServerHosts).anyTimes();
+        EasyMock.expect(zeppelinMaster.getConfigProperty("zeppelin.ssl")).andReturn("true").anyTimes();
+        EasyMock.expect(zeppelinMaster.getConfigProperty("zeppelin.server.port")).andReturn(HTTP_PORT).anyTimes();
+        EasyMock.expect(zeppelinMaster.getConfigProperty("zeppelin.server.ssl.port")).andReturn(HTTPS_PORT).anyTimes();
+        EasyMock.replay(zeppelinMaster);
+
+        // Run the test
+        validateServiceURLs(builder.create("ZEPPELINUI"), HOSTNAMES, "https", HTTPS_PORT, null);
+    }
+
+
+    @Test
+    public void testZeppelinWsURL() throws Exception {
+        final String HTTP_PORT = "8787";
+        final String HTTPS_PORT = "8989";
+
+        final String[] HOSTNAMES = {"host1", "host4"};
+        final List<String> atlastServerHosts = Arrays.asList(HOSTNAMES);
+
+        AmbariComponent zeppelinMaster = EasyMock.createNiceMock(AmbariComponent.class);
+        EasyMock.expect(zeppelinMaster.getHostNames()).andReturn(atlastServerHosts).anyTimes();
+        EasyMock.expect(zeppelinMaster.getConfigProperty("zeppelin.ssl")).andReturn("false").anyTimes();
+        EasyMock.expect(zeppelinMaster.getConfigProperty("zeppelin.server.port")).andReturn(HTTP_PORT).anyTimes();
+        EasyMock.expect(zeppelinMaster.getConfigProperty("zeppelin.server.ssl.port")).andReturn(HTTPS_PORT).anyTimes();
+        EasyMock.replay(zeppelinMaster);
+
+        AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class);
+        EasyMock.expect(cluster.getComponent("ZEPPELIN_MASTER")).andReturn(zeppelinMaster).anyTimes();
+        EasyMock.replay(cluster);
+
+        AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, null);
+
+        // Run the test
+        validateServiceURLs(builder.create("ZEPPELINWS"), HOSTNAMES, "ws", HTTP_PORT, null);
+
+        EasyMock.reset(zeppelinMaster);
+        EasyMock.expect(zeppelinMaster.getHostNames()).andReturn(atlastServerHosts).anyTimes();
+        EasyMock.expect(zeppelinMaster.getConfigProperty("zeppelin.ssl")).andReturn("true").anyTimes();
+        EasyMock.expect(zeppelinMaster.getConfigProperty("zeppelin.server.port")).andReturn(HTTP_PORT).anyTimes();
+        EasyMock.expect(zeppelinMaster.getConfigProperty("zeppelin.server.ssl.port")).andReturn(HTTPS_PORT).anyTimes();
+        EasyMock.replay(zeppelinMaster);
+
+        // Run the test
+        validateServiceURLs(builder.create("ZEPPELINWS"), HOSTNAMES, "wss", HTTPS_PORT, null);
+    }
+
+
+    @Test
+    public void testDruidCoordinatorURL() throws Exception {
+        final String PORT = "8787";
+
+        final String[] HOSTNAMES = {"host3", "host2"};
+        final List<String> druidCoordinatorHosts = Arrays.asList(HOSTNAMES);
+
+        AmbariComponent druidCoordinator = EasyMock.createNiceMock(AmbariComponent.class);
+        EasyMock.expect(druidCoordinator.getHostNames()).andReturn(druidCoordinatorHosts).anyTimes();
+        EasyMock.expect(druidCoordinator.getConfigProperty("druid.port")).andReturn(PORT).anyTimes();
+        EasyMock.replay(druidCoordinator);
+
+        AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class);
+        EasyMock.expect(cluster.getComponent("DRUID_COORDINATOR")).andReturn(druidCoordinator).anyTimes();
+        EasyMock.replay(cluster);
+
+        // Run the test
+        AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, null);
+        List<String> urls = builder.create("DRUID-COORDINATOR");
+        validateServiceURLs(urls, HOSTNAMES, "http", PORT, null);
+    }
+
+
+    @Test
+    public void testDruidBrokerURL() throws Exception {
+        final String PORT = "8181";
+
+        final String[] HOSTNAMES = {"host4", "host3"};
+        final List<String> druidHosts = Arrays.asList(HOSTNAMES);
+
+        AmbariComponent druidBroker = EasyMock.createNiceMock(AmbariComponent.class);
+        EasyMock.expect(druidBroker.getHostNames()).andReturn(druidHosts).anyTimes();
+        EasyMock.expect(druidBroker.getConfigProperty("druid.port")).andReturn(PORT).anyTimes();
+        EasyMock.replay(druidBroker);
+
+        AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class);
+        EasyMock.expect(cluster.getComponent("DRUID_BROKER")).andReturn(druidBroker).anyTimes();
+        EasyMock.replay(cluster);
+
+        // Run the test
+        AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, null);
+        List<String> urls = builder.create("DRUID-BROKER");
+        validateServiceURLs(urls, HOSTNAMES, "http", PORT, null);
+    }
+
+
+    @Test
+    public void testDruidRouterURL() throws Exception {
+        final String PORT = "8282";
+
+        final String[] HOSTNAMES = {"host5", "host7"};
+        final List<String> druidHosts = Arrays.asList(HOSTNAMES);
+
+        AmbariComponent druidRouter = EasyMock.createNiceMock(AmbariComponent.class);
+        EasyMock.expect(druidRouter.getHostNames()).andReturn(druidHosts).anyTimes();
+        EasyMock.expect(druidRouter.getConfigProperty("druid.port")).andReturn(PORT).anyTimes();
+        EasyMock.replay(druidRouter);
+
+        AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class);
+        EasyMock.expect(cluster.getComponent("DRUID_ROUTER")).andReturn(druidRouter).anyTimes();
+        EasyMock.replay(cluster);
+
+        // Run the test
+        AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, null);
+        List<String> urls = builder.create("DRUID-ROUTER");
+        validateServiceURLs(urls, HOSTNAMES, "http", PORT, null);
+    }
+
+
+    @Test
+    public void testDruidOverlordURL() throws Exception {
+        final String PORT = "8383";
+
+        final String[] HOSTNAMES = {"host4", "host1"};
+        final List<String> druidHosts = Arrays.asList(HOSTNAMES);
+
+        AmbariComponent druidOverlord = EasyMock.createNiceMock(AmbariComponent.class);
+        EasyMock.expect(druidOverlord.getHostNames()).andReturn(druidHosts).anyTimes();
+        EasyMock.expect(druidOverlord.getConfigProperty("druid.port")).andReturn(PORT).anyTimes();
+        EasyMock.replay(druidOverlord);
+
+        AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class);
+        EasyMock.expect(cluster.getComponent("DRUID_OVERLORD")).andReturn(druidOverlord).anyTimes();
+        EasyMock.replay(cluster);
+
+        // Run the test
+        AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, null);
+        List<String> urls = builder.create("DRUID-OVERLORD");
+        validateServiceURLs(urls, HOSTNAMES, "http", PORT, null);
+    }
+
+
+    @Test
+    public void testDruidSupersetURL() throws Exception {
+        final String PORT = "8484";
+
+        final String[] HOSTNAMES = {"host4", "host1"};
+        final List<String> druidHosts = Arrays.asList(HOSTNAMES);
+
+        AmbariComponent druidSuperset = EasyMock.createNiceMock(AmbariComponent.class);
+        EasyMock.expect(druidSuperset.getHostNames()).andReturn(druidHosts).anyTimes();
+        EasyMock.expect(druidSuperset.getConfigProperty("SUPERSET_WEBSERVER_PORT")).andReturn(PORT).anyTimes();
+        EasyMock.replay(druidSuperset);
+
+        AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class);
+        EasyMock.expect(cluster.getComponent("DRUID_SUPERSET")).andReturn(druidSuperset).anyTimes();
+        EasyMock.replay(cluster);
+
+        // Run the test
+        AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, null);
+        List<String> urls = builder.create("SUPERSET");
+        validateServiceURLs(urls, HOSTNAMES, "http", PORT, null);
+    }
+
+
+    @Test
+    public void testMissingServiceComponentURL() throws Exception {
+        AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class);
+        EasyMock.expect(cluster.getComponent("DRUID_BROKER")).andReturn(null).anyTimes();
+        EasyMock.expect(cluster.getComponent("HIVE_SERVER")).andReturn(null).anyTimes();
+        EasyMock.replay(cluster);
+
+        // Run the test
+        AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, null);
+        List<String> urls = builder.create("DRUID-BROKER");
+        assertNotNull(urls);
+        assertEquals(1, urls.size());
+        assertEquals("http://{HOST}:{PORT}", urls.get(0));
+
+        urls = builder.create("HIVE");
+        assertNotNull(urls);
+        assertEquals(1, urls.size());
+        assertEquals("http://{HOST}:{PORT}/{PATH}", urls.get(0));
+    }
+
+
+    /**
+     * Convenience method for creating AmbariDynamicServiceURLCreator instances from different mapping configuration
+     * input sources.
+     *
+     * @param cluster       The Ambari ServiceDiscovery Cluster model
+     * @param mappingConfig The mapping configuration, or null if the internal config should be used.
+     *
+     * @return An AmbariDynamicServiceURLCreator instance, capable of creating service URLs based on the specified
+     *         cluster's configuration details.
+     */
+    private static AmbariDynamicServiceURLCreator newURLCreator(AmbariCluster cluster, Object mappingConfig) throws Exception {
+        AmbariDynamicServiceURLCreator result = null;
+
+        if (mappingConfig == null) {
+            result = new AmbariDynamicServiceURLCreator(cluster);
+        } else {
+            if (mappingConfig instanceof String) {
+                result = new AmbariDynamicServiceURLCreator(cluster, (String) mappingConfig);
+            } else if (mappingConfig instanceof File) {
+                result = new AmbariDynamicServiceURLCreator(cluster, (File) mappingConfig);
+            }
+        }
+
+        return result;
+    }
+
+
+    /**
+     * Validate the specifed HIVE URLs.
+     *
+     * @param urlsToValidate The URLs to validate
+     * @param hostNames      The host names expected in the test URLs
+     * @param scheme         The expected scheme for the URLs
+     * @param port           The expected port for the URLs
+     * @param path           The expected path for the URLs
+     */
+    private static void validateServiceURLs(List<String> urlsToValidate,
+                                            String[]     hostNames,
+                                            String       scheme,
+                                            String       port,
+                                            String       path) throws MalformedURLException {
+
+        List<String> hostNamesToTest = new LinkedList<>(Arrays.asList(hostNames));
+        for (String url : urlsToValidate) {
+            URI test = null;
+            try {
+                // Make sure it's a valid URL
+                test = new URI(url);
+            } catch (URISyntaxException e) {
+                fail(e.getMessage());
+            }
+
+            // Validate the scheme
+            assertEquals(scheme, test.getScheme());
+
+            // Validate the port
+            assertEquals(port, String.valueOf(test.getPort()));
+
+            // If the expected path is not specified, don't validate it
+            if (path != null) {
+                assertEquals("/" + path, test.getPath());
+            }
+
+            // Validate the host name
+            assertTrue(hostNamesToTest.contains(test.getHost()));
+            hostNamesToTest.remove(test.getHost());
+        }
+        assertTrue(hostNamesToTest.isEmpty());
+    }
+
+
+    private static final String TEST_MAPPING_CONFIG =
+            "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n" +
+            "<service-discovery-url-mappings>\n" +
+            "  <service name=\"NAMENODE\">\n" +
+            "    <url-pattern>hdfs://{DFS_NAMENODE_RPC_ADDRESS}</url-pattern>\n" +
+            "    <properties>\n" +
+            "      <property name=\"DFS_NAMENODE_RPC_ADDRESS\">\n" +
+            "        <component>NAMENODE</component>\n" +
+            "        <config-property>dfs.namenode.rpc-address</config-property>\n" +
+            "      </property>\n" +
+            "    </properties>\n" +
+            "  </service>\n" +
+            "\n" +
+            "  <service name=\"JOBTRACKER\">\n" +
+            "    <url-pattern>rpc://{YARN_RM_ADDRESS}</url-pattern>\n" +
+            "    <properties>\n" +
+            "      <property name=\"YARN_RM_ADDRESS\">\n" +
+            "        <component>RESOURCEMANAGER</component>\n" +
+            "        <config-property>yarn.resourcemanager.address</config-property>\n" +
+            "      </property>\n" +
+            "    </properties>\n" +
+            "  </service>\n" +
+            "\n" +
+            "  <service name=\"WEBHDFS\">\n" +
+            "    <url-pattern>http://{WEBHDFS_ADDRESS}/webhdfs</url-pattern>\n" +
+            "    <properties>\n" +
+            "      <property name=\"WEBHDFS_ADDRESS\">\n" +
+            "        <service-config name=\"HDFS\">hdfs-site</service-config>\n" +
+            "        <config-property>dfs.namenode.http-address</config-property>\n" +
+            "      </property>\n" +
+            "    </properties>\n" +
+            "  </service>\n" +
+            "\n" +
+            "  <service name=\"WEBHCAT\">\n" +
+            "    <url-pattern>http://{HOST}:{PORT}/templeton</url-pattern>\n" +
+            "    <properties>\n" +
+            "      <property name=\"HOST\">\n" +
+            "        <component>WEBHCAT_SERVER</component>\n" +
+            "        <hostname/>\n" +
+            "      </property>\n" +
+            "      <property name=\"PORT\">\n" +
+            "        <component>WEBHCAT_SERVER</component>\n" +
+            "        <config-property>templeton.port</config-property>\n" +
+            "      </property>\n" +
+            "    </properties>\n" +
+            "  </service>\n" +
+            "\n" +
+            "  <service name=\"OOZIE\">\n" +
+            "    <url-pattern>{OOZIE_ADDRESS}</url-pattern>\n" +
+            "    <properties>\n" +
+            "      <property name=\"OOZIE_ADDRESS\">\n" +
+            "        <component>OOZIE_SERVER</component>\n" +
+            "        <config-property>oozie.base.url</config-property>\n" +
+            "      </property>\n" +
+            "    </properties>\n" +
+            "  </service>\n" +
+            "\n" +
+            "  <service name=\"WEBHBASE\">\n" +
+            "    <url-pattern>http://{HOST}:60080</url-pattern>\n" +
+            "    <properties>\n" +
+            "      <property name=\"HOST\">\n" +
+            "        <component>HBASE_MASTER</component>\n" +
+            "        <hostname/>\n" +
+            "      </property>\n" +
+            "    </properties>\n" +
+            "  </service>\n" +
+            "  <service name=\"RESOURCEMANAGER\">\n" +
+            "    <url-pattern>{SCHEME}://{WEBAPP_ADDRESS}/ws</url-pattern>\n" +
+            "    <properties>\n" +
+            "      <property name=\"WEBAPP_HTTP_ADDRESS\">\n" +
+            "        <component>RESOURCEMANAGER</component>\n" +
+            "        <config-property>yarn.resourcemanager.webapp.address</config-property>\n" +
+            "      </property>\n" +
+            "      <property name=\"WEBAPP_HTTPS_ADDRESS\">\n" +
+            "        <component>RESOURCEMANAGER</component>\n" +
+            "        <config-property>yarn.resourcemanager.webapp.https.address</config-property>\n" +
+            "      </property>\n" +
+            "      <property name=\"HTTP_POLICY\">\n" +
+            "        <component>RESOURCEMANAGER</component>\n" +
+            "        <config-property>yarn.http.policy</config-property>\n" +
+            "      </property>\n" +
+            "      <property name=\"SCHEME\">\n" +
+            "        <config-property>\n" +
+            "          <if property=\"HTTP_POLICY\" value=\"HTTPS_ONLY\">\n" +
+            "            <then>https</then>\n" +
+            "            <else>http</else>\n" +
+            "          </if>\n" +
+            "        </config-property>\n" +
+            "      </property>\n" +
+            "      <property name=\"WEBAPP_ADDRESS\">\n" +
+            "        <component>RESOURCEMANAGER</component>\n" +
+            "        <config-property>\n" +
+            "          <if property=\"HTTP_POLICY\" value=\"HTTPS_ONLY\">\n" +
+            "            <then>WEBAPP_HTTPS_ADDRESS</then>\n" +
+            "            <else>WEBAPP_HTTP_ADDRESS</else>\n" +
+            "          </if>\n" +
+            "        </config-property>\n" +
+            "      </property>\n" +
+            "    </properties>\n" +
+            "  </service>\n" +
+            "  <service name=\"HIVE\">\n" +
+            "    <url-pattern>{SCHEME}://{HOST}:{PORT}/{PATH}</url-pattern>\n" +
+            "    <properties>\n" +
+            "      <property name=\"HOST\">\n" +
+            "        <component>HIVE_SERVER</component>\n" +
+            "        <hostname/>\n" +
+            "      </property>\n" +
+            "      <property name=\"USE_SSL\">\n" +
+            "        <component>HIVE_SERVER</component>\n" +
+            "        <config-property>hive.server2.use.SSL</config-property>\n" +
+            "      </property>\n" +
+            "      <property name=\"PATH\">\n" +
+            "        <component>HIVE_SERVER</component>\n" +
+            "        <config-property>hive.server2.thrift.http.path</config-property>\n" +
+            "      </property>\n" +
+            "      <property name=\"PORT\">\n" +
+            "        <component>HIVE_SERVER</component>\n" +
+            "        <config-property>hive.server2.thrift.http.port</config-property>\n" +
+            "      </property>\n" +
+            "      <property name=\"SCHEME\">\n" +
+            "        <config-property>\n" +
+            "            <if property=\"USE_SSL\" value=\"true\">\n" +
+            "                <then>https</then>\n" +
+            "                <else>http</else>\n" +
+            "            </if>\n" +
+            "        </config-property>\n" +
+            "      </property>\n" +
+            "    </properties>\n" +
+            "  </service>\n" +
+            "</service-discovery-url-mappings>\n";
+
+
+    private static final String OVERRIDE_MAPPING_FILE_CONTENTS =
+            "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n" +
+            "<service-discovery-url-mappings>\n" +
+            "  <service name=\"WEBHDFS\">\n" +
+            "    <url-pattern>http://{WEBHDFS_ADDRESS}/webhdfs/OVERRIDE</url-pattern>\n" +
+            "    <properties>\n" +
+            "      <property name=\"WEBHDFS_ADDRESS\">\n" +
+            "        <service-config name=\"HDFS\">hdfs-site</service-config>\n" +
+            "        <config-property>dfs.namenode.http-address</config-property>\n" +
+            "      </property>\n" +
+            "    </properties>\n" +
+            "  </service>\n" +
+            "</service-discovery-url-mappings>\n";
+
+}

http://git-wip-us.apache.org/repos/asf/knox/blob/7b401def/gateway-discovery-ambari/src/test/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscoveryTest.java
----------------------------------------------------------------------
diff --git a/gateway-discovery-ambari/src/test/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscoveryTest.java b/gateway-discovery-ambari/src/test/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscoveryTest.java
index 1e5e7b2..f7f0553 100644
--- a/gateway-discovery-ambari/src/test/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscoveryTest.java
+++ b/gateway-discovery-ambari/src/test/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscoveryTest.java
@@ -27,7 +27,9 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 
 
 /**

http://git-wip-us.apache.org/repos/asf/knox/blob/7b401def/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorHandler.java
----------------------------------------------------------------------
diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorHandler.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorHandler.java
index fb563fa..521b5b4 100644
--- a/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorHandler.java
+++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorHandler.java
@@ -16,15 +16,28 @@
  */
 package org.apache.hadoop.gateway.topology.simple;
 
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileWriter;
+import java.io.InputStreamReader;
+import java.io.IOException;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
 import org.apache.hadoop.gateway.i18n.messages.MessagesFactory;
 import org.apache.hadoop.gateway.services.Service;
 import org.apache.hadoop.gateway.topology.discovery.DefaultServiceDiscoveryConfig;
 import org.apache.hadoop.gateway.topology.discovery.ServiceDiscovery;
 import org.apache.hadoop.gateway.topology.discovery.ServiceDiscoveryFactory;
 
-import java.io.*;
-import java.util.*;
-
 
 /**
  * Processes simple topology descriptors, producing full topology files, which can subsequently be deployed to the
@@ -78,19 +91,29 @@ public class SimpleDescriptorHandler {
                     descServiceURLs = cluster.getServiceURLs(serviceName);
                 }
 
-                // If there is at least one URL associated with the service, then add it to the map
+                // Validate the discovered service URLs
+                List<String> validURLs = new ArrayList<>();
                 if (descServiceURLs != null && !descServiceURLs.isEmpty()) {
-                    serviceURLs.put(serviceName, descServiceURLs);
+                    // Validate the URL(s)
+                    for (String descServiceURL : descServiceURLs) {
+                        if (validateURL(serviceName, descServiceURL)) {
+                            validURLs.add(descServiceURL);
+                        }
+                    }
+                }
+
+                // If there is at least one valid URL associated with the service, then add it to the map
+                if (!validURLs.isEmpty()) {
+                    serviceURLs.put(serviceName, validURLs);
                 } else {
                     log.failedToDiscoverClusterServiceURLs(serviceName, cluster.getName());
-                    throw new IllegalStateException("ServiceDiscovery failed to resolve any URLs for " + serviceName +
-                                                    ". Topology update aborted!");
                 }
             }
         } else {
             log.failedToDiscoverClusterServices(desc.getClusterName());
         }
 
+        BufferedWriter fw = null;
         topologyDescriptor = null;
         File providerConfig = null;
         try {
@@ -110,7 +133,7 @@ public class SimpleDescriptorHandler {
                 topologyFilename = desc.getClusterName();
             }
             topologyDescriptor = new File(destDirectory, topologyFilename + ".xml");
-            FileWriter fw = new FileWriter(topologyDescriptor);
+            fw = new BufferedWriter(new FileWriter(topologyDescriptor));
 
             fw.write("<topology>\n");
 
@@ -123,8 +146,12 @@ public class SimpleDescriptorHandler {
             }
             policyReader.close();
 
+            // Sort the service names to write the services alphabetically
+            List<String> serviceNames = new ArrayList<>(serviceURLs.keySet());
+            Collections.sort(serviceNames);
+
             // Write the service declarations
-            for (String serviceName : serviceURLs.keySet()) {
+            for (String serviceName : serviceNames) {
                 fw.write("    <service>\n");
                 fw.write("        <role>" + serviceName + "</role>\n");
                 for (String url : serviceURLs.get(serviceName)) {
@@ -136,16 +163,37 @@ public class SimpleDescriptorHandler {
             fw.write("</topology>\n");
 
             fw.flush();
-            fw.close();
         } catch (IOException e) {
             log.failedToGenerateTopologyFromSimpleDescriptor(topologyDescriptor.getName(), e);
             topologyDescriptor.delete();
+        } finally {
+            if (fw != null) {
+                try {
+                    fw.close();
+                } catch (IOException e) {
+                    // ignore
+                }
+            }
         }
 
         result.put("topology", topologyDescriptor);
         return result;
     }
 
+    private static boolean validateURL(String serviceName, String url) {
+        boolean result = false;
+
+        if (url != null && !url.isEmpty()) {
+            try {
+                new URI(url);
+                result = true;
+            } catch (URISyntaxException e) {
+                log.serviceURLValidationFailed(serviceName, url, e);
+            }
+        }
+
+        return result;
+    }
 
     private static File resolveProviderConfigurationReference(String reference, File srcDirectory) {
         File providerConfig;

http://git-wip-us.apache.org/repos/asf/knox/blob/7b401def/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorMessages.java
----------------------------------------------------------------------
diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorMessages.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorMessages.java
index cf9aa28..2a2c4c1 100644
--- a/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorMessages.java
+++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorMessages.java
@@ -29,7 +29,7 @@ public interface SimpleDescriptorMessages {
     void failedToDiscoverClusterServices(final String cluster);
 
     @Message(level = MessageLevel.ERROR,
-            text = "No URLs were discovered for {0} in the {1} cluster.")
+            text = "No valid URLs were discovered for {0} in the {1} cluster.")
     void failedToDiscoverClusterServiceURLs(final String serviceName, final String clusterName);
 
     @Message(level = MessageLevel.ERROR,
@@ -37,6 +37,12 @@ public interface SimpleDescriptorMessages {
     void failedToResolveProviderConfigRef(final String providerConfigRef);
 
     @Message(level = MessageLevel.ERROR,
+            text = "URL validation failed for {0} URL {1} : {2}")
+    void serviceURLValidationFailed(final String serviceName,
+                                    final String url,
+                                    @StackTrace( level = MessageLevel.DEBUG ) Exception e );
+
+    @Message(level = MessageLevel.ERROR,
             text = "Error generating topology {0} from simple descriptor: {1}")
     void failedToGenerateTopologyFromSimpleDescriptor(final String topologyFile,
                                                       @StackTrace( level = MessageLevel.DEBUG ) Exception e );

http://git-wip-us.apache.org/repos/asf/knox/blob/7b401def/gateway-server/src/test/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorHandlerTest.java
----------------------------------------------------------------------
diff --git a/gateway-server/src/test/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorHandlerTest.java b/gateway-server/src/test/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorHandlerTest.java
index 90c7146..f79ef23 100644
--- a/gateway-server/src/test/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorHandlerTest.java
+++ b/gateway-server/src/test/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorHandlerTest.java
@@ -17,6 +17,23 @@
  */
 package org.apache.hadoop.gateway.topology.simple;
 
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import javax.xml.xpath.XPath;
+import javax.xml.xpath.XPathConstants;
+import javax.xml.xpath.XPathFactory;
+
+import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.gateway.topology.validation.TopologyValidator;
 import org.apache.hadoop.gateway.util.XmlUtils;
 import org.easymock.EasyMock;
@@ -26,91 +43,89 @@ import org.w3c.dom.Node;
 import org.w3c.dom.NodeList;
 import org.xml.sax.SAXException;
 
-import javax.xml.xpath.XPath;
-import javax.xml.xpath.XPathConstants;
-import javax.xml.xpath.XPathFactory;
-import java.io.*;
-import java.util.*;
-
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
 
 public class SimpleDescriptorHandlerTest {
 
     private static final String TEST_PROVIDER_CONFIG =
-            "    <gateway>\n" +
-                    "        <provider>\n" +
-                    "            <role>authentication</role>\n" +
-                    "            <name>ShiroProvider</name>\n" +
-                    "            <enabled>true</enabled>\n" +
-                    "            <param>\n" +
-                    "                <!-- \n" +
-                    "                session timeout in minutes,  this is really idle timeout,\n" +
-                    "                defaults to 30mins, if the property value is not defined,, \n" +
-                    "                current client authentication would expire if client idles contiuosly for more than this value\n" +
-                    "                -->\n" +
-                    "                <name>sessionTimeout</name>\n" +
-                    "                <value>30</value>\n" +
-                    "            </param>\n" +
-                    "            <param>\n" +
-                    "                <name>main.ldapRealm</name>\n" +
-                    "                <value>org.apache.hadoop.gateway.shirorealm.KnoxLdapRealm</value>\n" +
-                    "            </param>\n" +
-                    "            <param>\n" +
-                    "                <name>main.ldapContextFactory</name>\n" +
-                    "                <value>org.apache.hadoop.gateway.shirorealm.KnoxLdapContextFactory</value>\n" +
-                    "            </param>\n" +
-                    "            <param>\n" +
-                    "                <name>main.ldapRealm.contextFactory</name>\n" +
-                    "                <value>$ldapContextFactory</value>\n" +
-                    "            </param>\n" +
-                    "            <param>\n" +
-                    "                <name>main.ldapRealm.userDnTemplate</name>\n" +
-                    "                <value>uid={0},ou=people,dc=hadoop,dc=apache,dc=org</value>\n" +
-                    "            </param>\n" +
-                    "            <param>\n" +
-                    "                <name>main.ldapRealm.contextFactory.url</name>\n" +
-                    "                <value>ldap://localhost:33389</value>\n" +
-                    "            </param>\n" +
-                    "            <param>\n" +
-                    "                <name>main.ldapRealm.contextFactory.authenticationMechanism</name>\n" +
-                    "                <value>simple</value>\n" +
-                    "            </param>\n" +
-                    "            <param>\n" +
-                    "                <name>urls./**</name>\n" +
-                    "                <value>authcBasic</value>\n" +
-                    "            </param>\n" +
-                    "        </provider>\n" +
-                    "\n" +
-                    "        <provider>\n" +
-                    "            <role>identity-assertion</role>\n" +
-                    "            <name>Default</name>\n" +
-                    "            <enabled>true</enabled>\n" +
-                    "        </provider>\n" +
-                    "\n" +
-                    "        <!--\n" +
-                    "        Defines rules for mapping host names internal to a Hadoop cluster to externally accessible host names.\n" +
-                    "        For example, a hadoop service running in AWS may return a response that includes URLs containing the\n" +
-                    "        some AWS internal host name.  If the client needs to make a subsequent request to the host identified\n" +
-                    "        in those URLs they need to be mapped to external host names that the client Knox can use to connect.\n" +
-                    "\n" +
-                    "        If the external hostname and internal host names are same turn of this provider by setting the value of\n" +
-                    "        enabled parameter as false.\n" +
-                    "\n" +
-                    "        The name parameter specifies the external host names in a comma separated list.\n" +
-                    "        The value parameter specifies corresponding internal host names in a comma separated list.\n" +
-                    "\n" +
-                    "        Note that when you are using Sandbox, the external hostname needs to be localhost, as seen in out\n" +
-                    "        of box sandbox.xml.  This is because Sandbox uses port mapping to allow clients to connect to the\n" +
-                    "        Hadoop services using localhost.  In real clusters, external host names would almost never be localhost.\n" +
-                    "        -->\n" +
-                    "        <provider>\n" +
-                    "            <role>hostmap</role>\n" +
-                    "            <name>static</name>\n" +
-                    "            <enabled>true</enabled>\n" +
-                    "            <param><name>localhost</name><value>sandbox,sandbox.hortonworks.com</value></param>\n" +
-                    "        </provider>\n" +
-                    "    </gateway>\n";
+        "    <gateway>\n" +
+        "        <provider>\n" +
+        "            <role>authentication</role>\n" +
+        "            <name>ShiroProvider</name>\n" +
+        "            <enabled>true</enabled>\n" +
+        "            <param>\n" +
+        "                <!-- \n" +
+        "                session timeout in minutes,  this is really idle timeout,\n" +
+        "                defaults to 30mins, if the property value is not defined,, \n" +
+        "                current client authentication would expire if client idles contiuosly for more than this value\n" +
+        "                -->\n" +
+        "                <name>sessionTimeout</name>\n" +
+        "                <value>30</value>\n" +
+        "            </param>\n" +
+        "            <param>\n" +
+        "                <name>main.ldapRealm</name>\n" +
+        "                <value>org.apache.hadoop.gateway.shirorealm.KnoxLdapRealm</value>\n" +
+        "            </param>\n" +
+        "            <param>\n" +
+        "                <name>main.ldapContextFactory</name>\n" +
+        "                <value>org.apache.hadoop.gateway.shirorealm.KnoxLdapContextFactory</value>\n" +
+        "            </param>\n" +
+        "            <param>\n" +
+        "                <name>main.ldapRealm.contextFactory</name>\n" +
+        "                <value>$ldapContextFactory</value>\n" +
+        "            </param>\n" +
+        "            <param>\n" +
+        "                <name>main.ldapRealm.userDnTemplate</name>\n" +
+        "                <value>uid={0},ou=people,dc=hadoop,dc=apache,dc=org</value>\n" +
+        "            </param>\n" +
+        "            <param>\n" +
+        "                <name>main.ldapRealm.contextFactory.url</name>\n" +
+        "                <value>ldap://localhost:33389</value>\n" +
+        "            </param>\n" +
+        "            <param>\n" +
+        "                <name>main.ldapRealm.contextFactory.authenticationMechanism</name>\n" +
+        "                <value>simple</value>\n" +
+        "            </param>\n" +
+        "            <param>\n" +
+        "                <name>urls./**</name>\n" +
+        "                <value>authcBasic</value>\n" +
+        "            </param>\n" +
+        "        </provider>\n" +
+        "\n" +
+        "        <provider>\n" +
+        "            <role>identity-assertion</role>\n" +
+        "            <name>Default</name>\n" +
+        "            <enabled>true</enabled>\n" +
+        "        </provider>\n" +
+        "\n" +
+        "        <!--\n" +
+        "        Defines rules for mapping host names internal to a Hadoop cluster to externally accessible host names.\n" +
+        "        For example, a hadoop service running in AWS may return a response that includes URLs containing the\n" +
+        "        some AWS internal host name.  If the client needs to make a subsequent request to the host identified\n" +
+        "        in those URLs they need to be mapped to external host names that the client Knox can use to connect.\n" +
+        "\n" +
+        "        If the external hostname and internal host names are same turn of this provider by setting the value of\n" +
+        "        enabled parameter as false.\n" +
+        "\n" +
+        "        The name parameter specifies the external host names in a comma separated list.\n" +
+        "        The value parameter specifies corresponding internal host names in a comma separated list.\n" +
+        "\n" +
+        "        Note that when you are using Sandbox, the external hostname needs to be localhost, as seen in out\n" +
+        "        of box sandbox.xml.  This is because Sandbox uses port mapping to allow clients to connect to the\n" +
+        "        Hadoop services using localhost.  In real clusters, external host names would almost never be localhost.\n" +
+        "        -->\n" +
+        "        <provider>\n" +
+        "            <role>hostmap</role>\n" +
+        "            <name>static</name>\n" +
+        "            <enabled>true</enabled>\n" +
+        "            <param><name>localhost</name><value>sandbox,sandbox.hortonworks.com</value></param>\n" +
+        "        </provider>\n" +
+        "    </gateway>\n";
 
 
     /**
@@ -134,7 +149,7 @@ public class SimpleDescriptorHandlerTest {
         serviceURLs.put("WEBHBASE", null);
         serviceURLs.put("HIVE", null);
         serviceURLs.put("RESOURCEMANAGER", null);
-        serviceURLs.put("AMBARIUI", Arrays.asList("http://c6401.ambari.apache.org:8080"));
+        serviceURLs.put("AMBARIUI", Collections.singletonList("http://c6401.ambari.apache.org:8080"));
 
         // Write the externalized provider config to a temp file
         File providerConfig = writeProviderConfig("ambari-cluster-policy.xml", TEST_PROVIDER_CONFIG);
@@ -225,14 +240,152 @@ public class SimpleDescriptorHandlerTest {
     }
 
 
-    private File writeProviderConfig(String path, String content) throws IOException {
-        File f = new File(path);
+    /**
+     * KNOX-1006
+     *
+     * Verify the behavior of the SimpleDescriptorHandler when service discovery fails to produce a valid URL for
+     * a service.
+     *
+     * N.B. This test depends on the PropertiesFileServiceDiscovery extension being configured:
+     *             org.apache.hadoop.gateway.topology.discovery.test.extension.PropertiesFileServiceDiscovery
+     */
+    @Test
+    public void testInvalidServiceURLFromDiscovery() throws Exception {
+        final String CLUSTER_NAME = "myproperties";
+
+        // Configure the PropertiesFile Service Discovery implementation for this test
+        final String DEFAULT_VALID_SERVICE_URL = "http://localhost:9999/thiswillwork";
+        Properties serviceDiscoverySourceProps = new Properties();
+        serviceDiscoverySourceProps.setProperty(CLUSTER_NAME + ".NAMENODE",
+                                                DEFAULT_VALID_SERVICE_URL.replace("http", "hdfs"));
+        serviceDiscoverySourceProps.setProperty(CLUSTER_NAME + ".JOBTRACKER",
+                                                DEFAULT_VALID_SERVICE_URL.replace("http", "rpc"));
+        serviceDiscoverySourceProps.setProperty(CLUSTER_NAME + ".WEBHDFS",         DEFAULT_VALID_SERVICE_URL);
+        serviceDiscoverySourceProps.setProperty(CLUSTER_NAME + ".WEBHCAT",         DEFAULT_VALID_SERVICE_URL);
+        serviceDiscoverySourceProps.setProperty(CLUSTER_NAME + ".OOZIE",           DEFAULT_VALID_SERVICE_URL);
+        serviceDiscoverySourceProps.setProperty(CLUSTER_NAME + ".WEBHBASE",        DEFAULT_VALID_SERVICE_URL);
+        serviceDiscoverySourceProps.setProperty(CLUSTER_NAME + ".HIVE",            "{SCHEME}://localhost:10000/");
+        serviceDiscoverySourceProps.setProperty(CLUSTER_NAME + ".RESOURCEMANAGER", DEFAULT_VALID_SERVICE_URL);
+        serviceDiscoverySourceProps.setProperty(CLUSTER_NAME + ".AMBARIUI",        DEFAULT_VALID_SERVICE_URL);
+        File serviceDiscoverySource = File.createTempFile("service-discovery", ".properties");
+        serviceDiscoverySourceProps.store(new FileOutputStream(serviceDiscoverySource),
+                                          "Test Service Discovery Source");
+
+        // Prepare a mock SimpleDescriptor
+        final String type = "PROPERTIES_FILE";
+        final String address = serviceDiscoverySource.getAbsolutePath();
+        final Map<String, List<String>> serviceURLs = new HashMap<>();
+        serviceURLs.put("NAMENODE", null);
+        serviceURLs.put("JOBTRACKER", null);
+        serviceURLs.put("WEBHDFS", null);
+        serviceURLs.put("WEBHCAT", null);
+        serviceURLs.put("OOZIE", null);
+        serviceURLs.put("WEBHBASE", null);
+        serviceURLs.put("HIVE", null);
+        serviceURLs.put("RESOURCEMANAGER", null);
+        serviceURLs.put("AMBARIUI", Collections.singletonList("http://c6401.ambari.apache.org:8080"));
 
-        Writer fw = new FileWriter(f);
-        fw.write(content);
-        fw.flush();
-        fw.close();
+        // Write the externalized provider config to a temp file
+        File providerConfig = writeProviderConfig("ambari-cluster-policy.xml", TEST_PROVIDER_CONFIG);
+
+        File topologyFile = null;
+        try {
+            File destDir = (new File(".")).getCanonicalFile();
+
+            // Mock out the simple descriptor
+            SimpleDescriptor testDescriptor = EasyMock.createNiceMock(SimpleDescriptor.class);
+            EasyMock.expect(testDescriptor.getName()).andReturn("mysimpledescriptor").anyTimes();
+            EasyMock.expect(testDescriptor.getDiscoveryAddress()).andReturn(address).anyTimes();
+            EasyMock.expect(testDescriptor.getDiscoveryType()).andReturn(type).anyTimes();
+            EasyMock.expect(testDescriptor.getDiscoveryUser()).andReturn(null).anyTimes();
+            EasyMock.expect(testDescriptor.getProviderConfig()).andReturn(providerConfig.getAbsolutePath()).anyTimes();
+            EasyMock.expect(testDescriptor.getClusterName()).andReturn(CLUSTER_NAME).anyTimes();
+            List<SimpleDescriptor.Service> serviceMocks = new ArrayList<>();
+            for (String serviceName : serviceURLs.keySet()) {
+                SimpleDescriptor.Service svc = EasyMock.createNiceMock(SimpleDescriptor.Service.class);
+                EasyMock.expect(svc.getName()).andReturn(serviceName).anyTimes();
+                EasyMock.expect(svc.getURLs()).andReturn(serviceURLs.get(serviceName)).anyTimes();
+                EasyMock.replay(svc);
+                serviceMocks.add(svc);
+            }
+            EasyMock.expect(testDescriptor.getServices()).andReturn(serviceMocks).anyTimes();
+            EasyMock.replay(testDescriptor);
+
+            // Invoke the simple descriptor handler
+            Map<String, File> files =
+                    SimpleDescriptorHandler.handle(testDescriptor,
+                                                   providerConfig.getParentFile(), // simple desc co-located with provider config
+                                                   destDir);
+
+            topologyFile = files.get("topology");
 
+            // Validate the resulting topology descriptor
+            assertTrue(topologyFile.exists());
+
+            // Validate the topology descriptor's correctness
+            TopologyValidator validator = new TopologyValidator( topologyFile.getAbsolutePath() );
+            if( !validator.validateTopology() ){
+                throw new SAXException( validator.getErrorString() );
+            }
+
+            XPathFactory xPathfactory = XPathFactory.newInstance();
+            XPath xpath = xPathfactory.newXPath();
+
+            // Parse the topology descriptor
+            Document topologyXml = XmlUtils.readXml(topologyFile);
+
+            // Validate the provider configuration
+            Document extProviderConf = XmlUtils.readXml(new ByteArrayInputStream(TEST_PROVIDER_CONFIG.getBytes()));
+            Node gatewayNode = (Node) xpath.compile("/topology/gateway").evaluate(topologyXml, XPathConstants.NODE);
+            assertTrue("Resulting provider config should be identical to the referenced content.",
+                    extProviderConf.getDocumentElement().isEqualNode(gatewayNode));
+
+            // Validate the service declarations
+            List<String> topologyServices = new ArrayList<>();
+            Map<String, List<String>> topologyServiceURLs = new HashMap<>();
+            NodeList serviceNodes =
+                    (NodeList) xpath.compile("/topology/service").evaluate(topologyXml, XPathConstants.NODESET);
+            for (int serviceNodeIndex=0; serviceNodeIndex < serviceNodes.getLength(); serviceNodeIndex++) {
+                Node serviceNode = serviceNodes.item(serviceNodeIndex);
+                Node roleNode = (Node) xpath.compile("role/text()").evaluate(serviceNode, XPathConstants.NODE);
+                assertNotNull(roleNode);
+                String role = roleNode.getNodeValue();
+                topologyServices.add(role);
+                NodeList urlNodes = (NodeList) xpath.compile("url/text()").evaluate(serviceNode, XPathConstants.NODESET);
+                for(int urlNodeIndex = 0 ; urlNodeIndex < urlNodes.getLength(); urlNodeIndex++) {
+                    Node urlNode = urlNodes.item(urlNodeIndex);
+                    assertNotNull(urlNode);
+                    String url = urlNode.getNodeValue();
+                    assertNotNull("Every declared service should have a URL.", url);
+                    if (!topologyServiceURLs.containsKey(role)) {
+                        topologyServiceURLs.put(role, new ArrayList<String>());
+                    }
+                    topologyServiceURLs.get(role).add(url);
+                }
+            }
+
+            // There should not be a service element for HIVE, since it had no valid URLs
+            assertEquals("Unexpected number of service declarations.", serviceURLs.size() - 1, topologyServices.size());
+            assertFalse("The HIVE service should have been omitted from the generated topology.", topologyServices.contains("HIVE"));
+
+            assertEquals("Unexpected number of service URLs.", serviceURLs.size() - 1, topologyServiceURLs.size());
+
+        } catch (Exception e) {
+            e.printStackTrace();
+            fail(e.getMessage());
+        } finally {
+            serviceDiscoverySource.delete();
+            providerConfig.delete();
+            if (topologyFile != null) {
+                topologyFile.delete();
+            }
+        }
+    }
+
+
+    private File writeProviderConfig(String path, String content) throws IOException {
+        File f = new File(path);
+        FileUtils.write(f, content);
         return f;
     }
 


Mime
View raw message