knox-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lmc...@apache.org
Subject [09/37] knox git commit: KNOX-1014 - Service Discovery and Topology Generation Framework (Phil Zampino via lmccay)
Date Thu, 26 Oct 2017 14:22:45 GMT
http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/gateway-discovery-ambari/src/test/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscoveryTest.java
----------------------------------------------------------------------
diff --git a/gateway-discovery-ambari/src/test/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscoveryTest.java b/gateway-discovery-ambari/src/test/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscoveryTest.java
new file mode 100644
index 0000000..1e5e7b2
--- /dev/null
+++ b/gateway-discovery-ambari/src/test/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscoveryTest.java
@@ -0,0 +1,856 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.gateway.topology.discovery.ambari;
+
+import net.minidev.json.JSONObject;
+import net.minidev.json.JSONValue;
+import org.apache.hadoop.gateway.topology.discovery.ServiceDiscovery;
+import org.apache.hadoop.gateway.topology.discovery.ServiceDiscoveryConfig;
+import org.easymock.EasyMock;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.*;
+
+
+/**
+ * Test the Ambari ServiceDiscovery implementation.
+ *
+ * N.B. These tests do NOT verify Ambari API responses. They DO validate the Ambari ServiceDiscovery implementation's
+ *      treatment of the responses as they were observed at the time the tests are developed.
+ */
+public class AmbariServiceDiscoveryTest {
+
+    @Test
+    public void testSingleClusterDiscovery() throws Exception {
+        final String discoveryAddress = "http://ambarihost:8080";
+        final String clusterName = "testCluster";
+        ServiceDiscovery sd = new TestAmbariServiceDiscovery(clusterName);
+
+        ServiceDiscoveryConfig sdc = EasyMock.createNiceMock(ServiceDiscoveryConfig.class);
+        EasyMock.expect(sdc.getAddress()).andReturn(discoveryAddress).anyTimes();
+        EasyMock.expect(sdc.getUser()).andReturn(null).anyTimes();
+        EasyMock.replay(sdc);
+
+        ServiceDiscovery.Cluster cluster = sd.discover(sdc, clusterName);
+        assertNotNull(cluster);
+        assertEquals(clusterName, cluster.getName());
+        assertTrue(AmbariCluster.class.isAssignableFrom(cluster.getClass()));
+        assertEquals(6, ((AmbariCluster) cluster).getComponents().size());
+
+//        printServiceURLs(cluster);
+    }
+
+
+    @Test
+    public void testBulkClusterDiscovery() throws Exception {
+        final String discoveryAddress = "http://ambarihost:8080";
+        final String clusterName = "anotherCluster";
+        ServiceDiscovery sd = new TestAmbariServiceDiscovery(clusterName);
+
+        ServiceDiscoveryConfig sdc = EasyMock.createNiceMock(ServiceDiscoveryConfig.class);
+        EasyMock.expect(sdc.getAddress()).andReturn(discoveryAddress).anyTimes();
+        EasyMock.expect(sdc.getUser()).andReturn(null).anyTimes();
+        EasyMock.replay(sdc);
+
+        Map<String, ServiceDiscovery.Cluster> clusters = sd.discover(sdc);
+        assertNotNull(clusters);
+        assertEquals(1, clusters.size());
+        ServiceDiscovery.Cluster cluster = clusters.get(clusterName);
+        assertNotNull(cluster);
+        assertEquals(clusterName, cluster.getName());
+        assertTrue(AmbariCluster.class.isAssignableFrom(cluster.getClass()));
+        assertEquals(6, ((AmbariCluster) cluster).getComponents().size());
+
+//        printServiceURLs(cluster, "NAMENODE", "WEBHCAT", "OOZIE", "RESOURCEMANAGER");
+    }
+
+
+    private static void printServiceURLs(ServiceDiscovery.Cluster cluster) {
+        final String[] services = new String[]{"NAMENODE",
+                                               "JOBTRACKER",
+                                               "WEBHDFS",
+                                               "WEBHCAT",
+                                               "OOZIE",
+                                               "WEBHBASE",
+                                               "HIVE",
+                                               "RESOURCEMANAGER"};
+        printServiceURLs(cluster, services);
+    }
+
+
+    private static void printServiceURLs(ServiceDiscovery.Cluster cluster, String...services) {
+        for (String name : services) {
+            StringBuilder sb = new StringBuilder();
+            List<String> urls = cluster.getServiceURLs(name);
+            if (urls != null && !urls.isEmpty()) {
+                for (String url : urls) {
+                    sb.append(url);
+                    sb.append(" ");
+                }
+            }
+            System.out.println(String.format("%18s: %s", name, sb.toString()));
+        }
+    }
+
+
+    /**
+     * ServiceDiscovery implementation derived from AmbariServiceDiscovery, so the invokeREST method can be overridden
+     * to eliminate the need to perform actual HTTP interactions with a real Ambari endpoint.
+     */
+    private static final class TestAmbariServiceDiscovery extends AmbariServiceDiscovery {
+
+        final static String CLUSTER_PLACEHOLDER = "CLUSTER_NAME";
+
+        private Map<String, JSONObject> cannedResponses = new HashMap<>();
+
+        TestAmbariServiceDiscovery(String clusterName) {
+            cannedResponses.put(AmbariServiceDiscovery.AMBARI_CLUSTERS_URI,
+                                (JSONObject) JSONValue.parse(CLUSTERS_JSON_TEMPLATE.replaceAll(CLUSTER_PLACEHOLDER,
+                                                                                               clusterName)));
+
+            cannedResponses.put(String.format(AmbariServiceDiscovery.AMBARI_HOSTROLES_URI, clusterName),
+                                (JSONObject) JSONValue.parse(HOSTROLES_JSON_TEMPLATE.replaceAll(CLUSTER_PLACEHOLDER,
+                                                                                                clusterName)));
+
+            cannedResponses.put(String.format(AmbariServiceDiscovery.AMBARI_SERVICECONFIGS_URI, clusterName),
+                                (JSONObject) JSONValue.parse(SERVICECONFIGS_JSON_TEMPLATE.replaceAll(CLUSTER_PLACEHOLDER,
+                                                                                                     clusterName)));
+        }
+
+        @Override
+        protected JSONObject invokeREST(String url, String username, String passwordAlias) {
+            return cannedResponses.get(url.substring(url.indexOf("/api")));
+        }
+    }
+
+
+    ////////////////////////////////////////////////////////////////////////
+    //  JSON response templates, based on actual response content excerpts
+    ////////////////////////////////////////////////////////////////////////
+
+    private static final String CLUSTERS_JSON_TEMPLATE =
+    "{\n" +
+    "  \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters\",\n" +
+    "  \"items\" : [\n" +
+    "    {\n" +
+    "      \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "      \"Clusters\" : {\n" +
+    "        \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "        \"version\" : \"HDP-2.6\"\n" +
+    "      }\n" +
+    "    }\n" +
+    "  ]" +
+    "}";
+
+
+    private static final String HOSTROLES_JSON_TEMPLATE =
+    "{\n" +
+    "  \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services?fields=components/host_components/HostRoles\",\n" +
+    "  \"items\" : [\n" +
+    "    {\n" +
+    "      \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/AMBARI_METRICS\",\n" +
+    "      \"ServiceInfo\" : {\n" +
+    "        \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "        \"service_name\" : \"AMBARI_METRICS\"\n" +
+    "      },\n" +
+    "      \"components\" : [\n" +
+    "        {\n" +
+    "          \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/AMBARI_METRICS/components/METRICS_COLLECTOR\",\n" +
+    "          \"ServiceComponentInfo\" : {\n" +
+    "            \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "            \"component_name\" : \"METRICS_COLLECTOR\",\n" +
+    "            \"service_name\" : \"AMBARI_METRICS\"\n" +
+    "          },\n" +
+    "          \"host_components\" : [\n" +
+    "            {\n" +
+    "              \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6403.ambari.apache.org/host_components/METRICS_COLLECTOR\",\n" +
+    "              \"HostRoles\" : {\n" +
+    "                \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "                \"component_name\" : \"METRICS_COLLECTOR\",\n" +
+    "                \"host_name\" : \"c6403.ambari.apache.org\",\n" +
+    "                \"public_host_name\" : \"c6403.ambari.apache.org\",\n" +
+    "                \"service_name\" : \"AMBARI_METRICS\",\n" +
+    "                \"stack_id\" : \"HDP-2.6\",\n" +
+    "              }\n" +
+    "            }\n" +
+    "          ]\n" +
+    "        },\n" +
+    "        {\n" +
+    "          \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/HBASE/components/HBASE_MASTER\",\n" +
+    "          \"ServiceComponentInfo\" : {\n" +
+    "            \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "            \"component_name\" : \"HBASE_MASTER\",\n" +
+    "            \"service_name\" : \"HBASE\"\n" +
+    "          },\n" +
+    "          \"host_components\" : [\n" +
+    "            {\n" +
+    "              \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6401.ambari.apache.org/host_components/HBASE_MASTER\",\n" +
+    "              \"HostRoles\" : {\n" +
+    "                \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "                \"component_name\" : \"HBASE_MASTER\",\n" +
+    "                \"host_name\" : \"c6401.ambari.apache.org\",\n" +
+    "                \"public_host_name\" : \"c6401.ambari.apache.org\",\n" +
+    "                \"service_name\" : \"HBASE\",\n" +
+    "                \"stack_id\" : \"HDP-2.6\",\n" +
+    "              }\n" +
+    "            }\n" +
+    "          ]\n" +
+    "        }\n" +
+    "      ]\n" +
+    "    },\n" +
+    "    {\n" +
+    "      \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/HDFS\",\n" +
+    "      \"ServiceInfo\" : {\n" +
+    "        \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "        \"service_name\" : \"HDFS\"\n" +
+    "      },\n" +
+    "      \"components\" : [\n" +
+    "        {\n" +
+    "          \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/HDFS/components/NAMENODE\",\n" +
+    "          \"ServiceComponentInfo\" : {\n" +
+    "            \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "            \"component_name\" : \"NAMENODE\",\n" +
+    "            \"service_name\" : \"HDFS\"\n" +
+    "          },\n" +
+    "          \"host_components\" : [\n" +
+    "            {\n" +
+    "              \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6401.ambari.apache.org/host_components/NAMENODE\",\n" +
+    "              \"HostRoles\" : {\n" +
+    "                \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "                \"component_name\" : \"NAMENODE\",\n" +
+    "                \"host_name\" : \"c6401.ambari.apache.org\",\n" +
+    "                \"public_host_name\" : \"c6401.ambari.apache.org\",\n" +
+    "                \"service_name\" : \"HDFS\",\n" +
+    "                \"stack_id\" : \"HDP-2.6\",\n" +
+    "              }\n" +
+    "            }\n" +
+    "          ]\n" +
+    "        },\n" +
+    "        {\n" +
+    "          \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/HDFS/components/SECONDARY_NAMENODE\",\n" +
+    "          \"ServiceComponentInfo\" : {\n" +
+    "            \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "            \"component_name\" : \"SECONDARY_NAMENODE\",\n" +
+    "            \"service_name\" : \"HDFS\"\n" +
+    "          },\n" +
+    "          \"host_components\" : [\n" +
+    "            {\n" +
+    "              \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6402.ambari.apache.org/host_components/SECONDARY_NAMENODE\",\n" +
+    "              \"HostRoles\" : {\n" +
+    "                \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "                \"component_name\" : \"SECONDARY_NAMENODE\",\n" +
+    "                \"host_name\" : \"c6402.ambari.apache.org\",\n" +
+    "                \"public_host_name\" : \"c6402.ambari.apache.org\",\n" +
+    "                \"service_name\" : \"HDFS\",\n" +
+    "                \"stack_id\" : \"HDP-2.6\",\n" +
+    "              }\n" +
+    "            }\n" +
+    "          ]\n" +
+    "        }\n" +
+    "      ]\n" +
+    "    },\n" +
+    "    {\n" +
+    "      \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/HIVE\",\n" +
+    "      \"ServiceInfo\" : {\n" +
+    "        \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "        \"service_name\" : \"HIVE\"\n" +
+    "      },\n" +
+    "      \"components\" : [\n" +
+    "        {\n" +
+    "          \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/HIVE/components/HCAT\",\n" +
+    "          \"ServiceComponentInfo\" : {\n" +
+    "            \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "            \"component_name\" : \"HCAT\",\n" +
+    "            \"service_name\" : \"HIVE\"\n" +
+    "          },\n" +
+    "          \"host_components\" : [\n" +
+    "            {\n" +
+    "              \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6403.ambari.apache.org/host_components/HCAT\",\n" +
+    "              \"HostRoles\" : {\n" +
+    "                \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "                \"component_name\" : \"HCAT\",\n" +
+    "                \"host_name\" : \"c6403.ambari.apache.org\",\n" +
+    "                \"public_host_name\" : \"c6403.ambari.apache.org\",\n" +
+    "                \"service_name\" : \"HIVE\",\n" +
+    "                \"stack_id\" : \"HDP-2.6\",\n" +
+    "              }\n" +
+    "            }\n" +
+    "          ]\n" +
+    "        }\n" +
+    "        {\n" +
+    "          \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/HIVE/components/HIVE_METASTORE\",\n" +
+    "          \"ServiceComponentInfo\" : {\n" +
+    "            \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "            \"component_name\" : \"HIVE_METASTORE\",\n" +
+    "            \"service_name\" : \"HIVE\"\n" +
+    "          },\n" +
+    "          \"host_components\" : [\n" +
+    "            {\n" +
+    "              \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6402.ambari.apache.org/host_components/HIVE_METASTORE\",\n" +
+    "              \"HostRoles\" : {\n" +
+    "                \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "                \"component_name\" : \"HIVE_METASTORE\",\n" +
+    "                \"host_name\" : \"c6402.ambari.apache.org\",\n" +
+    "                \"public_host_name\" : \"c6402.ambari.apache.org\",\n" +
+    "                \"service_name\" : \"HIVE\",\n" +
+    "                \"stack_id\" : \"HDP-2.6\",\n" +
+    "              }\n" +
+    "            }\n" +
+    "          ]\n" +
+    "        },\n" +
+    "        {\n" +
+    "          \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/HIVE/components/HIVE_SERVER\",\n" +
+    "          \"ServiceComponentInfo\" : {\n" +
+    "            \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "            \"component_name\" : \"HIVE_SERVER\",\n" +
+    "            \"service_name\" : \"HIVE\"\n" +
+    "          },\n" +
+    "          \"host_components\" : [\n" +
+    "            {\n" +
+    "              \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6402.ambari.apache.org/host_components/HIVE_SERVER\",\n" +
+    "              \"HostRoles\" : {\n" +
+    "                \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "                \"component_name\" : \"HIVE_SERVER\",\n" +
+    "                \"host_name\" : \"c6402.ambari.apache.org\",\n" +
+    "                \"public_host_name\" : \"c6402.ambari.apache.org\",\n" +
+    "                \"service_name\" : \"HIVE\",\n" +
+    "                \"stack_id\" : \"HDP-2.6\",\n" +
+    "              }\n" +
+    "            }\n" +
+    "          ]\n" +
+    "        },\n" +
+    "        {\n" +
+    "          \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/HIVE/components/WEBHCAT_SERVER\",\n" +
+    "          \"ServiceComponentInfo\" : {\n" +
+    "            \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "            \"component_name\" : \"WEBHCAT_SERVER\",\n" +
+    "            \"service_name\" : \"HIVE\"\n" +
+    "          },\n" +
+    "          \"host_components\" : [\n" +
+    "            {\n" +
+    "              \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6402.ambari.apache.org/host_components/WEBHCAT_SERVER\",\n" +
+    "              \"HostRoles\" : {\n" +
+    "                \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "                \"component_name\" : \"WEBHCAT_SERVER\",\n" +
+    "                \"host_name\" : \"c6402.ambari.apache.org\",\n" +
+    "                \"public_host_name\" : \"c6402.ambari.apache.org\",\n" +
+    "                \"service_name\" : \"HIVE\",\n" +
+    "                \"stack_id\" : \"HDP-2.6\",\n" +
+    "              }\n" +
+    "            }\n" +
+    "          ]\n" +
+    "        }\n" +
+    "      ]\n" +
+    "    },\n" +
+    "    {\n" +
+    "      \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/OOZIE\",\n" +
+    "      \"ServiceInfo\" : {\n" +
+    "        \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "        \"service_name\" : \"OOZIE\"\n" +
+    "      },\n" +
+    "      \"components\" : [\n" +
+    "        {\n" +
+    "          \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/OOZIE/components/OOZIE_SERVER\",\n" +
+    "          \"ServiceComponentInfo\" : {\n" +
+    "            \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "            \"component_name\" : \"OOZIE_SERVER\",\n" +
+    "            \"service_name\" : \"OOZIE\"\n" +
+    "          },\n" +
+    "          \"host_components\" : [\n" +
+    "            {\n" +
+    "              \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6402.ambari.apache.org/host_components/OOZIE_SERVER\",\n" +
+    "              \"HostRoles\" : {\n" +
+    "                \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "                \"component_name\" : \"OOZIE_SERVER\",\n" +
+    "                \"host_name\" : \"c6402.ambari.apache.org\",\n" +
+    "                \"public_host_name\" : \"c6402.ambari.apache.org\",\n" +
+    "                \"service_name\" : \"OOZIE\",\n" +
+    "                \"stack_id\" : \"HDP-2.6\"\n" +
+    "              }\n" +
+    "            }\n" +
+    "          ]\n" +
+    "        }\n" +
+    "      ]\n" +
+    "    },\n" +
+    "    {\n" +
+    "      \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/YARN\",\n" +
+    "      \"ServiceInfo\" : {\n" +
+    "        \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "        \"service_name\" : \"YARN\"\n" +
+    "      },\n" +
+    "      \"components\" : [\n" +
+    "        {\n" +
+    "          \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/YARN/components/APP_TIMELINE_SERVER\",\n" +
+    "          \"ServiceComponentInfo\" : {\n" +
+    "            \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "            \"component_name\" : \"APP_TIMELINE_SERVER\",\n" +
+    "            \"service_name\" : \"YARN\"\n" +
+    "          },\n" +
+    "          \"host_components\" : [\n" +
+    "            {\n" +
+    "              \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6402.ambari.apache.org/host_components/APP_TIMELINE_SERVER\",\n" +
+    "              \"HostRoles\" : {\n" +
+    "                \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "                \"component_name\" : \"APP_TIMELINE_SERVER\",\n" +
+    "                \"host_name\" : \"c6402.ambari.apache.org\",\n" +
+    "                \"public_host_name\" : \"c6402.ambari.apache.org\",\n" +
+    "                \"service_name\" : \"YARN\",\n" +
+    "                \"stack_id\" : \"HDP-2.6\"\n" +
+    "              }\n" +
+    "            }\n" +
+    "          ]\n" +
+    "        },\n" +
+    "        {\n" +
+    "          \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/YARN/components/NODEMANAGER\",\n" +
+    "          \"ServiceComponentInfo\" : {\n" +
+    "            \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "            \"component_name\" : \"NODEMANAGER\",\n" +
+    "            \"service_name\" : \"YARN\"\n" +
+    "          },\n" +
+    "          \"host_components\" : [\n" +
+    "            {\n" +
+    "              \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6403.ambari.apache.org/host_components/NODEMANAGER\",\n" +
+    "              \"HostRoles\" : {\n" +
+    "                \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "                \"component_name\" : \"NODEMANAGER\",\n" +
+    "                \"host_name\" : \"c6403.ambari.apache.org\",\n" +
+    "                \"public_host_name\" : \"c6403.ambari.apache.org\",\n" +
+    "                \"service_name\" : \"YARN\",\n" +
+    "                \"stack_id\" : \"HDP-2.6\"\n" +
+    "              }\n" +
+    "            }\n" +
+    "          ]\n" +
+    "        },\n" +
+    "        {\n" +
+    "          \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/YARN/components/RESOURCEMANAGER\",\n" +
+    "          \"ServiceComponentInfo\" : {\n" +
+    "            \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "            \"component_name\" : \"RESOURCEMANAGER\",\n" +
+    "            \"service_name\" : \"YARN\"\n" +
+    "          },\n" +
+    "          \"host_components\" : [\n" +
+    "            {\n" +
+    "              \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6402.ambari.apache.org/host_components/RESOURCEMANAGER\",\n" +
+    "              \"HostRoles\" : {\n" +
+    "                \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "                \"component_name\" : \"RESOURCEMANAGER\",\n" +
+    "                \"ha_state\" : \"ACTIVE\",\n" +
+    "                \"host_name\" : \"c6402.ambari.apache.org\",\n" +
+    "                \"public_host_name\" : \"c6402.ambari.apache.org\",\n" +
+    "                \"service_name\" : \"YARN\",\n" +
+    "                \"stack_id\" : \"HDP-2.6\"\n" +
+    "              }\n" +
+    "            }\n" +
+    "          ]\n" +
+    "        }\n" +
+    "      ]\n" +
+    "    },\n" +
+    "    {\n" +
+    "      \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/ZOOKEEPER\",\n" +
+    "      \"ServiceInfo\" : {\n" +
+    "        \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "        \"service_name\" : \"ZOOKEEPER\"\n" +
+    "      },\n" +
+    "      \"components\" : [\n" +
+    "        {\n" +
+    "          \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/ZOOKEEPER/components/ZOOKEEPER_SERVER\",\n" +
+    "          \"ServiceComponentInfo\" : {\n" +
+    "            \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "            \"component_name\" : \"ZOOKEEPER_SERVER\",\n" +
+    "            \"service_name\" : \"ZOOKEEPER\"\n" +
+    "          },\n" +
+    "          \"host_components\" : [\n" +
+    "            {\n" +
+    "              \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6401.ambari.apache.org/host_components/ZOOKEEPER_SERVER\",\n" +
+    "              \"HostRoles\" : {\n" +
+    "                \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "                \"component_name\" : \"ZOOKEEPER_SERVER\",\n" +
+    "                \"host_name\" : \"c6401.ambari.apache.org\",\n" +
+    "                \"public_host_name\" : \"c6401.ambari.apache.org\",\n" +
+    "                \"service_name\" : \"ZOOKEEPER\",\n" +
+    "                \"stack_id\" : \"HDP-2.6\"\n" +
+    "              }\n" +
+    "            },\n" +
+    "            {\n" +
+    "              \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6402.ambari.apache.org/host_components/ZOOKEEPER_SERVER\",\n" +
+    "              \"HostRoles\" : {\n" +
+    "                \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "                \"component_name\" : \"ZOOKEEPER_SERVER\",\n" +
+    "                \"host_name\" : \"c6402.ambari.apache.org\",\n" +
+    "                \"public_host_name\" : \"c6402.ambari.apache.org\",\n" +
+    "                \"service_name\" : \"ZOOKEEPER\",\n" +
+    "                \"stack_id\" : \"HDP-2.6\"\n" +
+    "              }\n" +
+    "            },\n" +
+    "            {\n" +
+    "              \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6403.ambari.apache.org/host_components/ZOOKEEPER_SERVER\",\n" +
+    "              \"HostRoles\" : {\n" +
+    "                \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "                \"component_name\" : \"ZOOKEEPER_SERVER\",\n" +
+    "                \"host_name\" : \"c6403.ambari.apache.org\",\n" +
+    "                \"public_host_name\" : \"c6403.ambari.apache.org\",\n" +
+    "                \"service_name\" : \"ZOOKEEPER\",\n" +
+    "                \"stack_id\" : \"HDP-2.6\"\n" +
+    "              }\n" +
+    "            }\n" +
+    "          ]\n" +
+    "        }\n" +
+    "      ]\n" +
+    "    }\n" +
+    "  ]\n" +
+    "}\n";
+
+
+    private static final String SERVICECONFIGS_JSON_TEMPLATE =
+    "{\n" +
+    "  \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/configurations/service_config_versions?is_current=true\",\n" +
+    "  \"items\" : [\n" +
+    "    {\n" +
+    "      \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/configurations/service_config_versions?service_name=HBASE&service_config_version=1\",\n" +
+    "      \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "      \"configurations\" : [\n" +
+    "        {\n" +
+    "          \"Config\" : {\n" +
+    "            \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "            \"stack_id\" : \"HDP-2.6\"\n" +
+    "          },\n" +
+    "          \"type\" : \"hbase-site\",\n" +
+    "          \"tag\" : \"version1503410563715\",\n" +
+    "          \"version\" : 1,\n" +
+    "          \"properties\" : {\n" +
+    "            \"hbase.master.info.bindAddress\" : \"0.0.0.0\",\n" +
+    "            \"hbase.master.info.port\" : \"16010\",\n" +
+    "            \"hbase.master.port\" : \"16000\",\n" +
+    "            \"hbase.regionserver.info.port\" : \"16030\",\n" +
+    "            \"hbase.regionserver.port\" : \"16020\",\n" +
+    "            \"hbase.zookeeper.property.clientPort\" : \"2181\",\n" +
+    "            \"hbase.zookeeper.quorum\" : \"c6403.ambari.apache.org,c6402.ambari.apache.org,c6401.ambari.apache.org\",\n" +
+    "            \"hbase.zookeeper.useMulti\" : \"true\",\n" +
+    "            \"zookeeper.znode.parent\" : \"/hbase-unsecure\"\n" +
+    "          },\n" +
+    "          \"properties_attributes\" : { }\n" +
+    "        },\n" +
+    "      ],\n" +
+    "      \"is_current\" : true,\n" +
+    "      \"service_config_version\" : 1,\n" +
+    "      \"service_config_version_note\" : \"Initial configurations for HBase\",\n" +
+    "      \"service_name\" : \"HBASE\",\n" +
+    "      \"stack_id\" : \"HDP-2.6\",\n" +
+    "      \"user\" : \"admin\"\n" +
+    "    },\n" +
+    "    {\n" +
+    "      \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/configurations/service_config_versions?service_name=HDFS&service_config_version=2\",\n" +
+    "      \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "      \"configurations\" : [\n" +
+    "        {\n" +
+    "          \"Config\" : {\n" +
+    "            \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "            \"stack_id\" : \"HDP-2.6\"\n" +
+    "          },\n" +
+    "          \"type\" : \"hdfs-site\",\n" +
+    "          \"tag\" : \"version1\",\n" +
+    "          \"version\" : 1,\n" +
+    "          \"properties\" : {\n" +
+    "            \"dfs.cluster.administrators\" : \" hdfs\",\n" +
+    "            \"dfs.datanode.address\" : \"0.0.0.0:50010\",\n" +
+    "            \"dfs.datanode.http.address\" : \"0.0.0.0:50075\",\n" +
+    "            \"dfs.datanode.https.address\" : \"0.0.0.0:50475\",\n" +
+    "            \"dfs.datanode.ipc.address\" : \"0.0.0.0:8010\",\n" +
+    "            \"dfs.http.policy\" : \"HTTP_ONLY\",\n" +
+    "            \"dfs.https.port\" : \"50470\",\n" +
+    "            \"dfs.journalnode.http-address\" : \"0.0.0.0:8480\",\n" +
+    "            \"dfs.journalnode.https-address\" : \"0.0.0.0:8481\",\n" +
+    "            \"dfs.namenode.http-address\" : \"c6401.ambari.apache.org:50070\",\n" +
+    "            \"dfs.namenode.https-address\" : \"c6401.ambari.apache.org:50470\",\n" +
+    "            \"dfs.namenode.rpc-address\" : \"c6401.ambari.apache.org:8020\",\n" +
+    "            \"dfs.namenode.secondary.http-address\" : \"c6402.ambari.apache.org:50090\",\n" +
+    "            \"dfs.webhdfs.enabled\" : \"true\"\n" +
+    "          },\n" +
+    "          \"properties_attributes\" : {\n" +
+    "            \"final\" : {\n" +
+    "              \"dfs.webhdfs.enabled\" : \"true\",\n" +
+    "              \"dfs.namenode.http-address\" : \"true\",\n" +
+    "              \"dfs.support.append\" : \"true\",\n" +
+    "              \"dfs.namenode.name.dir\" : \"true\",\n" +
+    "              \"dfs.datanode.failed.volumes.tolerated\" : \"true\",\n" +
+    "              \"dfs.datanode.data.dir\" : \"true\"\n" +
+    "            }\n" +
+    "          }\n" +
+    "        },\n" +
+    "        {\n" +
+    "          \"Config\" : {\n" +
+    "            \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "            \"stack_id\" : \"HDP-2.6\"\n" +
+    "          },\n" +
+    "          \"type\" : \"core-site\",\n" +
+    "          \"tag\" : \"version1502131215159\",\n" +
+    "          \"version\" : 2,\n" +
+    "          \"properties\" : {\n" +
+    "            \"hadoop.http.authentication.simple.anonymous.allowed\" : \"true\",\n" +
+    "            \"net.topology.script.file.name\" : \"/etc/hadoop/conf/topology_script.py\"\n" +
+    "          },\n" +
+    "          \"properties_attributes\" : {\n" +
+    "            \"final\" : {\n" +
+    "              \"fs.defaultFS\" : \"true\"\n" +
+    "            }\n" +
+    "          }\n" +
+    "        }\n" +
+    "      ],\n" +
+    "      \"is_current\" : true,\n" +
+    "      \"service_config_version\" : 2,\n" +
+    "      \"service_config_version_note\" : \"knox trusted proxy support\",\n" +
+    "      \"service_name\" : \"HDFS\",\n" +
+    "      \"stack_id\" : \"HDP-2.6\",\n" +
+    "      \"user\" : \"admin\"\n" +
+    "    },\n" +
+    "    {\n" +
+    "      \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/configurations/service_config_versions?service_name=HIVE&service_config_version=3\",\n" +
+    "      \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "      \"configurations\" : [\n" +
+    "        {\n" +
+    "          \"Config\" : {\n" +
+    "            \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "            \"stack_id\" : \"HDP-2.6\"\n" +
+    "          },\n" +
+    "          \"type\" : \"hive-env\",\n" +
+    "          \"tag\" : \"version1\",\n" +
+    "          \"version\" : 1,\n" +
+    "          \"properties\" : {\n" +
+    "            \"hive_security_authorization\" : \"None\",\n" +
+    "            \"webhcat_user\" : \"hcat\"\n" +
+    "          },\n" +
+    "          \"properties_attributes\" : { }\n" +
+    "        },\n" +
+    "        {\n" +
+    "          \"Config\" : {\n" +
+    "            \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "            \"stack_id\" : \"HDP-2.6\"\n" +
+    "          },\n" +
+    "          \"type\" : \"hiveserver2-site\",\n" +
+    "          \"tag\" : \"version1\",\n" +
+    "          \"version\" : 1,\n" +
+    "          \"properties\" : {\n" +
+    "            \"hive.metastore.metrics.enabled\" : \"true\",\n" +
+    "            \"hive.security.authorization.enabled\" : \"false\",\n" +
+    "            \"hive.service.metrics.hadoop2.component\" : \"hiveserver2\",\n" +
+    "            \"hive.service.metrics.reporter\" : \"HADOOP2\"\n" +
+    "          },\n" +
+    "          \"properties_attributes\" : { }\n" +
+    "        },\n" +
+    "        {\n" +
+    "          \"Config\" : {\n" +
+    "            \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "            \"stack_id\" : \"HDP-2.6\"\n" +
+    "          },\n" +
+    "          \"type\" : \"hive-interactive-site\",\n" +
+    "          \"tag\" : \"version1\",\n" +
+    "          \"version\" : 1,\n" +
+    "          \"properties\" : {\n" +
+    "            \"hive.server2.enable.doAs\" : \"false\",\n" +
+    "            \"hive.server2.tez.default.queues\" : \"default\",\n" +
+    "            \"hive.server2.tez.initialize.default.sessions\" : \"true\",\n" +
+    "            \"hive.server2.tez.sessions.custom.queue.allowed\" : \"ignore\",\n" +
+    "            \"hive.server2.tez.sessions.per.default.queue\" : \"1\",\n" +
+    "            \"hive.server2.tez.sessions.restricted.configs\" : \"hive.execution.mode,hive.execution.engine\",\n" +
+    "            \"hive.server2.thrift.http.port\" : \"10501\",\n" +
+    "            \"hive.server2.thrift.port\" : \"10500\",\n" +
+    "            \"hive.server2.webui.port\" : \"10502\",\n" +
+    "            \"hive.server2.webui.use.ssl\" : \"false\",\n" +
+    "            \"hive.server2.zookeeper.namespace\" : \"hiveserver2-hive2\"\n" +
+    "          },\n" +
+    "          \"properties_attributes\" : { }\n" +
+    "        },\n" +
+    "        {\n" +
+    "          \"Config\" : {\n" +
+    "            \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "            \"stack_id\" : \"HDP-2.6\"\n" +
+    "          },\n" +
+    "          \"type\" : \"tez-interactive-site\",\n" +
+    "          \"tag\" : \"version1\",\n" +
+    "          \"version\" : 1,\n" +
+    "          \"properties\" : {\n" +
+    "            \"tez.am.am-rm.heartbeat.interval-ms.max\" : \"10000\",\n" +
+    "            \"tez.am.client.heartbeat.poll.interval.millis\" : \"6000\",\n" +
+    "            \"tez.am.client.heartbeat.timeout.secs\" : \"90\"\n" +
+    "          },\n" +
+    "          \"properties_attributes\" : { }\n" +
+    "        },\n" +
+    "        {\n" +
+    "          \"Config\" : {\n" +
+    "            \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "            \"stack_id\" : \"HDP-2.6\"\n" +
+    "          },\n" +
+    "          \"type\" : \"hive-site\",\n" +
+    "          \"tag\" : \"version1502130841736\",\n" +
+    "          \"version\" : 2,\n" +
+    "          \"properties\" : {\n" +
+    "            \"hive.metastore.sasl.enabled\" : \"false\",\n" +
+    "            \"hive.metastore.server.max.threads\" : \"100000\",\n" +
+    "            \"hive.metastore.uris\" : \"thrift://c6402.ambari.apache.org:9083\",\n" +
+    "            \"hive.server2.allow.user.substitution\" : \"true\",\n" +
+    "            \"hive.server2.authentication\" : \"NONE\",\n" +
+    "            \"hive.server2.authentication.spnego.keytab\" : \"HTTP/_HOST@EXAMPLE.COM\",\n" +
+    "            \"hive.server2.authentication.spnego.principal\" : \"/etc/security/keytabs/spnego.service.keytab\",\n" +
+    "            \"hive.server2.enable.doAs\" : \"true\",\n" +
+    "            \"hive.server2.support.dynamic.service.discovery\" : \"true\",\n" +
+    "            \"hive.server2.thrift.http.path\" : \"cliservice\",\n" +
+    "            \"hive.server2.thrift.http.port\" : \"10001\",\n" +
+    "            \"hive.server2.thrift.max.worker.threads\" : \"500\",\n" +
+    "            \"hive.server2.thrift.port\" : \"10000\",\n" +
+    "            \"hive.server2.thrift.sasl.qop\" : \"auth\",\n" +
+    "            \"hive.server2.transport.mode\" : \"http\",\n" +
+    "            \"hive.server2.use.SSL\" : \"false\",\n" +
+    "            \"hive.server2.zookeeper.namespace\" : \"hiveserver2\"\n" +
+    "          },\n" +
+    "          \"properties_attributes\" : {\n" +
+    "            \"hidden\" : {\n" +
+    "              \"javax.jdo.option.ConnectionPassword\" : \"HIVE_CLIENT,WEBHCAT_SERVER,HCAT,CONFIG_DOWNLOAD\"\n" +
+    "            }\n" +
+    "          }\n" +
+    "        },\n" +
+    "        {\n" +
+    "          \"Config\" : {\n" +
+    "            \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "            \"stack_id\" : \"HDP-2.6\"\n" +
+    "          },\n" +
+    "          \"type\" : \"webhcat-site\",\n" +
+    "          \"tag\" : \"version1502131111746\",\n" +
+    "          \"version\" : 2,\n" +
+    "          \"properties\" : {\n" +
+    "            \"templeton.port\" : \"50111\",\n" +
+    "            \"templeton.zookeeper.hosts\" : \"c6403.ambari.apache.org:2181,c6401.ambari.apache.org:2181,c6402.ambari.apache.org:2181\",\n" +
+    "            \"webhcat.proxyuser.knox.groups\" : \"users\",\n" +
+    "            \"webhcat.proxyuser.knox.hosts\" : \"*\",\n" +
+    "            \"webhcat.proxyuser.root.groups\" : \"*\",\n" +
+    "            \"webhcat.proxyuser.root.hosts\" : \"c6401.ambari.apache.org\"\n" +
+    "          },\n" +
+    "          \"properties_attributes\" : { }\n" +
+    "        }\n" +
+    "      ],\n" +
+    "      \"createtime\" : 1502131110745,\n" +
+    "      \"group_id\" : -1,\n" +
+    "      \"group_name\" : \"Default\",\n" +
+    "      \"hosts\" : [ ],\n" +
+    "      \"is_cluster_compatible\" : true,\n" +
+    "      \"is_current\" : true,\n" +
+    "      \"service_config_version\" : 3,\n" +
+    "      \"service_config_version_note\" : \"knox trusted proxy support\",\n" +
+    "      \"service_name\" : \"HIVE\",\n" +
+    "      \"stack_id\" : \"HDP-2.6\",\n" +
+    "      \"user\" : \"admin\"\n" +
+    "    },\n" +
+    "    {\n" +
+    "      \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/configurations/service_config_versions?service_name=OOZIE&service_config_version=3\",\n" +
+    "      \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "      \"configurations\" : [\n" +
+    "        {\n" +
+    "          \"Config\" : {\n" +
+    "            \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "            \"stack_id\" : \"HDP-2.6\"\n" +
+    "          },\n" +
+    "          \"type\" : \"oozie-site\",\n" +
+    "          \"tag\" : \"version1502131137103\",\n" +
+    "          \"version\" : 3,\n" +
+    "          \"properties\" : {\n" +
+    "            \"oozie.base.url\" : \"http://c6402.ambari.apache.org:11000/oozie\",\n" +
+    "          },\n" +
+    "          \"properties_attributes\" : { }\n" +
+    "        }\n" +
+    "      ],\n" +
+    "      \"is_current\" : true,\n" +
+    "      \"service_config_version\" : 3,\n" +
+    "      \"service_name\" : \"OOZIE\",\n" +
+    "      \"stack_id\" : \"HDP-2.6\",\n" +
+    "      \"user\" : \"admin\"\n" +
+    "    },\n" +
+    "    {\n" +
+    "      \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/configurations/service_config_versions?service_name=TEZ&service_config_version=1\",\n" +
+    "      \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "      \"configurations\" : [\n" +
+    "        {\n" +
+    "          \"Config\" : {\n" +
+    "            \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "            \"stack_id\" : \"HDP-2.6\"\n" +
+    "          },\n" +
+    "          \"type\" : \"tez-site\",\n" +
+    "          \"tag\" : \"version1\",\n" +
+    "          \"version\" : 1,\n" +
+    "          \"properties\" : {\n" +
+    "            \"tez.use.cluster.hadoop-libs\" : \"false\"\n" +
+    "          },\n" +
+    "          \"properties_attributes\" : { }\n" +
+    "        }\n" +
+    "      ],\n" +
+    "      \"createtime\" : 1502122253525,\n" +
+    "      \"group_id\" : -1,\n" +
+    "      \"group_name\" : \"Default\",\n" +
+    "      \"hosts\" : [ ],\n" +
+    "      \"is_cluster_compatible\" : true,\n" +
+    "      \"is_current\" : true,\n" +
+    "      \"service_config_version\" : 1,\n" +
+    "      \"service_config_version_note\" : \"Initial configurations for Tez\",\n" +
+    "      \"service_name\" : \"TEZ\",\n" +
+    "      \"stack_id\" : \"HDP-2.6\",\n" +
+    "      \"user\" : \"admin\"\n" +
+    "    },\n" +
+    "    {\n" +
+    "      \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/configurations/service_config_versions?service_name=YARN&service_config_version=1\",\n" +
+    "      \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "      \"configurations\" : [\n" +
+    "        {\n" +
+    "          \"Config\" : {\n" +
+    "            \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+    "            \"stack_id\" : \"HDP-2.6\"\n" +
+    "          },\n" +
+    "          \"type\" : \"yarn-site\",\n" +
+    "          \"tag\" : \"version1\",\n" +
+    "          \"version\" : 1,\n" +
+    "          \"properties\" : {\n" +
+    "            \"hadoop.registry.rm.enabled\" : \"true\",\n" +
+    "            \"hadoop.registry.zk.quorum\" : \"c6403.ambari.apache.org:2181,c6401.ambari.apache.org:2181,c6402.ambari.apache.org:2181\",\n" +
+    "            \"yarn.acl.enable\" : \"false\",\n" +
+    "            \"yarn.http.policy\" : \"HTTP_ONLY\",\n" +
+    "            \"yarn.nodemanager.address\" : \"0.0.0.0:45454\",\n" +
+    "            \"yarn.nodemanager.bind-host\" : \"0.0.0.0\",\n" +
+    "            \"yarn.resourcemanager.address\" : \"c6402.ambari.apache.org:8050\",\n" +
+    "            \"yarn.resourcemanager.admin.address\" : \"c6402.ambari.apache.org:8141\",\n" +
+    "            \"yarn.resourcemanager.ha.enabled\" : \"false\",\n" +
+    "            \"yarn.resourcemanager.hostname\" : \"c6402.ambari.apache.org\",\n" +
+    "            \"yarn.resourcemanager.resource-tracker.address\" : \"c6402.ambari.apache.org:8025\",\n" +
+    "            \"yarn.resourcemanager.scheduler.address\" : \"c6402.ambari.apache.org:8030\",\n" +
+    "            \"yarn.resourcemanager.webapp.address\" : \"c6402.ambari.apache.org:8088\",\n" +
+    "            \"yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled\" : \"false\",\n" +
+    "            \"yarn.resourcemanager.webapp.https.address\" : \"c6402.ambari.apache.org:8090\",\n" +
+    "            \"yarn.resourcemanager.zk-address\" : \"c6403.ambari.apache.org:2181,c6401.ambari.apache.org:2181,c6402.ambari.apache.org:2181\"\n" +
+    "          },\n" +
+    "          \"properties_attributes\" : { }\n" +
+    "        }\n" +
+    "      ],\n" +
+    "      \"is_current\" : true,\n" +
+    "      \"service_config_version\" : 1,\n" +
+    "      \"service_name\" : \"YARN\",\n" +
+    "      \"stack_id\" : \"HDP-2.6\",\n" +
+    "      \"user\" : \"admin\"\n" +
+    "    }\n" +
+    "  ]\n" +
+    "}";
+
+}

http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/gateway-release/home/conf/descriptors/README
----------------------------------------------------------------------
diff --git a/gateway-release/home/conf/descriptors/README b/gateway-release/home/conf/descriptors/README
new file mode 100644
index 0000000..a2e5226
--- /dev/null
+++ b/gateway-release/home/conf/descriptors/README
@@ -0,0 +1 @@
+THIS DIRECTORY IS WHERE SIMPLE TOPOLOGY DESCRIPTORS CAN BE PLACED
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/gateway-release/home/conf/shared-providers/README
----------------------------------------------------------------------
diff --git a/gateway-release/home/conf/shared-providers/README b/gateway-release/home/conf/shared-providers/README
new file mode 100644
index 0000000..44d12a3
--- /dev/null
+++ b/gateway-release/home/conf/shared-providers/README
@@ -0,0 +1 @@
+THIS DIRECTORY IS WHERE SHARED PROVIDER CONFIGURATIONS CAN BE PLACED
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/gateway-release/pom.xml
----------------------------------------------------------------------
diff --git a/gateway-release/pom.xml b/gateway-release/pom.xml
index 83824cd..cbff307 100644
--- a/gateway-release/pom.xml
+++ b/gateway-release/pom.xml
@@ -334,6 +334,10 @@
             <groupId>${gateway-group}</groupId>
             <artifactId>gateway-shell-samples</artifactId>
         </dependency>
+        <dependency>
+            <groupId>${gateway-group}</groupId>
+            <artifactId>gateway-discovery-ambari</artifactId>
+        </dependency>
 
         <dependency>
             <groupId>junit</groupId>

http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/gateway-server/pom.xml
----------------------------------------------------------------------
diff --git a/gateway-server/pom.xml b/gateway-server/pom.xml
index 437d22d..5ebf793 100644
--- a/gateway-server/pom.xml
+++ b/gateway-server/pom.xml
@@ -248,6 +248,11 @@
            <artifactId>javax-websocket-client-impl</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>com.fasterxml.jackson.dataformat</groupId>
+            <artifactId>jackson-dataformat-yaml</artifactId>
+            <version>2.3.0</version>
+        </dependency>
 
         <!-- ********** ********** ********** ********** ********** ********** -->
         <!-- ********** Test Dependencies                           ********** -->

http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/gateway-server/src/main/java/org/apache/hadoop/gateway/GatewayMessages.java
----------------------------------------------------------------------
diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/GatewayMessages.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/GatewayMessages.java
index 1f94584..6f73c1e 100644
--- a/gateway-server/src/main/java/org/apache/hadoop/gateway/GatewayMessages.java
+++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/GatewayMessages.java
@@ -511,6 +511,11 @@ public interface GatewayMessages {
                " Could not find topology \"{0}\" mapped to port \"{1}\" configured in gateway-config.xml. "
                    + "This invalid topology mapping will be ignored by the gateway. "
                    + "Gateway restart will be required if in the future \"{0}\" topology is added.")
-  void topologyPortMappingCannotFindTopology(final String topology,
-      final int port);
+  void topologyPortMappingCannotFindTopology(final String topology, final int port);
+
+
+  @Message( level = MessageLevel.ERROR, text = "An error occurred while processing {0} : {1}" )
+  void simpleDescriptorHandlingError(final String simpleDesc,
+                                     @StackTrace( level = MessageLevel.DEBUG ) Exception e );
+
 }

http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/gateway-server/src/main/java/org/apache/hadoop/gateway/services/DefaultGatewayServices.java
----------------------------------------------------------------------
diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/services/DefaultGatewayServices.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/services/DefaultGatewayServices.java
index cefada1..02ac154 100644
--- a/gateway-server/src/main/java/org/apache/hadoop/gateway/services/DefaultGatewayServices.java
+++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/services/DefaultGatewayServices.java
@@ -105,14 +105,13 @@ public class DefaultGatewayServices implements GatewayServices {
     services.put( SERVER_INFO_SERVICE, sis );
 
     DefaultTopologyService tops = new DefaultTopologyService();
+    tops.setAliasService(alias);
     tops.init(  config, options  );
     services.put(  TOPOLOGY_SERVICE, tops  );
 
     DefaultServiceDefinitionRegistry sdr = new DefaultServiceDefinitionRegistry();
     sdr.init( config, options );
     services.put( SERVICE_DEFINITION_REGISTRY, sdr );
-    tops.init( config, options );
-    services.put( TOPOLOGY_SERVICE, tops );
 
     DefaultMetricsService metricsService = new DefaultMetricsService();
     metricsService.init( config, options );

http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/gateway-server/src/main/java/org/apache/hadoop/gateway/services/topology/impl/DefaultTopologyService.java
----------------------------------------------------------------------
diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/services/topology/impl/DefaultTopologyService.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/services/topology/impl/DefaultTopologyService.java
index 3321f3d..a493bc4 100644
--- a/gateway-server/src/main/java/org/apache/hadoop/gateway/services/topology/impl/DefaultTopologyService.java
+++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/services/topology/impl/DefaultTopologyService.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.gateway.config.GatewayConfig;
 import org.apache.hadoop.gateway.i18n.messages.MessagesFactory;
 import org.apache.hadoop.gateway.service.definition.ServiceDefinition;
 import org.apache.hadoop.gateway.services.ServiceLifecycleException;
+import org.apache.hadoop.gateway.services.security.AliasService;
 import org.apache.hadoop.gateway.services.topology.TopologyService;
 import org.apache.hadoop.gateway.topology.Topology;
 import org.apache.hadoop.gateway.topology.TopologyEvent;
@@ -45,6 +46,7 @@ import org.apache.hadoop.gateway.topology.TopologyListener;
 import org.apache.hadoop.gateway.topology.TopologyMonitor;
 import org.apache.hadoop.gateway.topology.TopologyProvider;
 import org.apache.hadoop.gateway.topology.builder.TopologyBuilder;
+import org.apache.hadoop.gateway.topology.simple.SimpleDescriptorHandler;
 import org.apache.hadoop.gateway.topology.validation.TopologyValidator;
 import org.apache.hadoop.gateway.topology.xml.AmbariFormatXmlTopologyRules;
 import org.apache.hadoop.gateway.topology.xml.KnoxFormatXmlTopologyRules;
@@ -74,20 +76,27 @@ import static org.apache.commons.digester3.binder.DigesterLoader.newLoader;
 public class DefaultTopologyService
     extends FileAlterationListenerAdaptor
     implements TopologyService, TopologyMonitor, TopologyProvider, FileFilter, FileAlterationListener {
+
   private static Auditor auditor = AuditServiceFactory.getAuditService().getAuditor(
     AuditConstants.DEFAULT_AUDITOR_NAME, AuditConstants.KNOX_SERVICE_NAME,
     AuditConstants.KNOX_COMPONENT_NAME);
+
   private static final List<String> SUPPORTED_TOPOLOGY_FILE_EXTENSIONS = new ArrayList<String>();
   static {
     SUPPORTED_TOPOLOGY_FILE_EXTENSIONS.add("xml");
     SUPPORTED_TOPOLOGY_FILE_EXTENSIONS.add("conf");
   }
+
   private static GatewayMessages log = MessagesFactory.get(GatewayMessages.class);
   private static DigesterLoader digesterLoader = newLoader(new KnoxFormatXmlTopologyRules(), new AmbariFormatXmlTopologyRules());
-  private FileAlterationMonitor monitor;
-  private File directory;
+  private List<FileAlterationMonitor> monitors = new ArrayList<>();
+  private File topologiesDirectory;
+  private File descriptorsDirectory;
+
   private Set<TopologyListener> listeners;
   private volatile Map<File, Topology> topologies;
+  private AliasService aliasService;
+
 
   private Topology loadTopology(File file) throws IOException, SAXException, URISyntaxException, InterruptedException {
     final long TIMEOUT = 250; //ms
@@ -202,29 +211,40 @@ public class DefaultTopologyService
   }
 
   private File calculateAbsoluteTopologiesDir(GatewayConfig config) {
-
-    File topoDir = new File(config.getGatewayTopologyDir());
+    String normalizedTopologyDir = FilenameUtils.normalize(config.getGatewayTopologyDir());
+    File topoDir = new File(normalizedTopologyDir);
     topoDir = topoDir.getAbsoluteFile();
     return topoDir;
   }
 
-  private void initListener(FileAlterationMonitor monitor, File directory) {
-    this.directory = directory;
-    this.monitor = monitor;
+  private File calculateAbsoluteConfigDir(GatewayConfig config) {
+    File configDir = null;
 
+    String path = FilenameUtils.normalize(config.getGatewayConfDir());
+    if (path != null) {
+      configDir = new File(config.getGatewayConfDir());
+    } else {
+      configDir = (new File(config.getGatewayTopologyDir())).getParentFile();
+    }
+    configDir = configDir.getAbsoluteFile();
 
-    FileAlterationObserver observer = new FileAlterationObserver(this.directory, this);
-    observer.addListener(this);
-    monitor.addObserver(observer);
+    return configDir;
+  }
 
-    this.listeners = new HashSet<>();
-    this.topologies = new HashMap<>(); //loadTopologies( this.directory );
+  private void  initListener(FileAlterationMonitor  monitor,
+                            File                   directory,
+                            FileFilter             filter,
+                            FileAlterationListener listener) {
+    monitors.add(monitor);
+    FileAlterationObserver observer = new FileAlterationObserver(directory, filter);
+    observer.addListener(listener);
+    monitor.addObserver(observer);
   }
 
-  private void initListener(File directory) throws IOException, SAXException {
+  private void initListener(File directory, FileFilter filter, FileAlterationListener listener) throws IOException, SAXException {
     // Increasing the monitoring interval to 5 seconds as profiling has shown
     // this is rather expensive in terms of generated garbage objects.
-    initListener(new FileAlterationMonitor(5000L), directory);
+    initListener(new FileAlterationMonitor(5000L), directory, filter, listener);
   }
 
   private Map<File, Topology> loadTopologies(File directory) {
@@ -261,10 +281,14 @@ public class DefaultTopologyService
     return map;
   }
 
+  public void setAliasService(AliasService as) {
+    this.aliasService = as;
+  }
+
   public void deployTopology(Topology t){
 
     try {
-      File temp = new File(directory.getAbsolutePath() + "/" + t.getName() + ".xml.temp");
+      File temp = new File(topologiesDirectory.getAbsolutePath() + "/" + t.getName() + ".xml.temp");
       Package topologyPkg = Topology.class.getPackage();
       String pkgName = topologyPkg.getName();
       String bindingFile = pkgName.replace(".", "/") + "/topology_binding-xml.xml";
@@ -277,7 +301,7 @@ public class DefaultTopologyService
       mr.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true);
       mr.marshal(t, temp);
 
-      File topology = new File(directory.getAbsolutePath() + "/" + t.getName() + ".xml");
+      File topology = new File(topologiesDirectory.getAbsolutePath() + "/" + t.getName() + ".xml");
       if(!temp.renameTo(topology)) {
         FileUtils.forceDelete(temp);
         throw new IOException("Could not rename temp file");
@@ -317,7 +341,7 @@ public class DefaultTopologyService
     try {
       synchronized (this) {
         Map<File, Topology> oldTopologies = topologies;
-        Map<File, Topology> newTopologies = loadTopologies(directory);
+        Map<File, Topology> newTopologies = loadTopologies(topologiesDirectory);
         List<TopologyEvent> events = createChangeEvents(oldTopologies, newTopologies);
         topologies = newTopologies;
         notifyChangeListeners(events);
@@ -329,7 +353,7 @@ public class DefaultTopologyService
   }
 
   public void deleteTopology(Topology t) {
-    File topoDir = directory;
+    File topoDir = topologiesDirectory;
 
     if(topoDir.isDirectory() && topoDir.canRead()) {
       File[] results = topoDir.listFiles();
@@ -357,8 +381,8 @@ public class DefaultTopologyService
   public Map<String, List<String>> getServiceTestURLs(Topology t, GatewayConfig config) {
     File tFile = null;
     Map<String, List<String>> urls = new HashMap<>();
-    if(directory.isDirectory() && directory.canRead()) {
-      for(File f : directory.listFiles()){
+    if(topologiesDirectory.isDirectory() && topologiesDirectory.canRead()) {
+      for(File f : topologiesDirectory.listFiles()){
         if(FilenameUtils.removeExtension(f.getName()).equals(t.getName())){
           tFile = f;
         }
@@ -387,12 +411,16 @@ public class DefaultTopologyService
 
   @Override
   public void startMonitor() throws Exception {
-    monitor.start();
+    for (FileAlterationMonitor monitor : monitors) {
+      monitor.start();
+    }
   }
 
   @Override
   public void stopMonitor() throws Exception {
-    monitor.stop();
+    for (FileAlterationMonitor monitor : monitors) {
+      monitor.stop();
+    }
   }
 
   @Override
@@ -414,6 +442,16 @@ public class DefaultTopologyService
 
   @Override
   public void onFileDelete(java.io.File file) {
+    // For full topology descriptors, we need to make sure to delete any corresponding simple descriptors to prevent
+    // unintended subsequent generation of the topology descriptor
+    for (String ext : DescriptorsMonitor.SUPPORTED_EXTENSIONS) {
+      File simpleDesc =
+              new File(descriptorsDirectory, FilenameUtils.getBaseName(file.getName()) + "." + ext);
+      if (simpleDesc.exists()) {
+        simpleDesc.delete();
+      }
+    }
+
     onFileChange(file);
   }
 
@@ -436,12 +474,200 @@ public class DefaultTopologyService
   public void init(GatewayConfig config, Map<String, String> options) throws ServiceLifecycleException {
 
     try {
-      initListener(calculateAbsoluteTopologiesDir(config));
-    } catch (IOException io) {
+      listeners = new HashSet<>();
+      topologies = new HashMap<>();
+
+      topologiesDirectory = calculateAbsoluteTopologiesDir(config);
+
+      File configDirectory = calculateAbsoluteConfigDir(config);
+      descriptorsDirectory = new File(configDirectory, "descriptors");
+      File sharedProvidersDirectory = new File(configDirectory, "shared-providers");
+
+      // Add support for conf/topologies
+      initListener(topologiesDirectory, this, this);
+
+      // Add support for conf/descriptors
+      DescriptorsMonitor dm = new DescriptorsMonitor(topologiesDirectory, aliasService);
+      initListener(descriptorsDirectory,
+                   dm,
+                   dm);
+
+      // Add support for conf/shared-providers
+      SharedProviderConfigMonitor spm = new SharedProviderConfigMonitor(dm, descriptorsDirectory);
+      initListener(sharedProvidersDirectory, spm, spm);
+
+    } catch (IOException | SAXException io) {
       throw new ServiceLifecycleException(io.getMessage());
-    } catch (SAXException sax) {
-      throw new ServiceLifecycleException(sax.getMessage());
+    }
+  }
+
+
+  /**
+   * Change handler for simple descriptors
+   */
+  public static class DescriptorsMonitor extends FileAlterationListenerAdaptor
+                                          implements FileFilter {
+
+    static final List<String> SUPPORTED_EXTENSIONS = new ArrayList<String>();
+    static {
+      SUPPORTED_EXTENSIONS.add("json");
+      SUPPORTED_EXTENSIONS.add("yml");
+    }
+
+    private File topologiesDir;
+
+    private AliasService aliasService;
+
+    private Map<String, List<String>> providerConfigReferences = new HashMap<>();
+
+
+    public DescriptorsMonitor(File topologiesDir, AliasService aliasService) {
+      this.topologiesDir  = topologiesDir;
+      this.aliasService   = aliasService;
+    }
+
+    List<String> getReferencingDescriptors(String providerConfigPath) {
+      List<String> result = providerConfigReferences.get(providerConfigPath);
+      if (result == null) {
+        result = Collections.emptyList();
+      }
+      return result;
+    }
+
+    @Override
+    public void onFileCreate(File file) {
+      onFileChange(file);
+    }
+
+    @Override
+    public void onFileDelete(File file) {
+      // For simple descriptors, we need to make sure to delete any corresponding full topology descriptors to trigger undeployment
+      for (String ext : DefaultTopologyService.SUPPORTED_TOPOLOGY_FILE_EXTENSIONS) {
+        File topologyFile =
+                new File(topologiesDir, FilenameUtils.getBaseName(file.getName()) + "." + ext);
+        if (topologyFile.exists()) {
+          topologyFile.delete();
+        }
+      }
+
+      String normalizedFilePath = FilenameUtils.normalize(file.getAbsolutePath());
+      String reference = null;
+      for (Map.Entry<String, List<String>> entry : providerConfigReferences.entrySet()) {
+        if (entry.getValue().contains(normalizedFilePath)) {
+          reference = entry.getKey();
+          break;
+        }
+      }
+      if (reference != null) {
+        providerConfigReferences.get(reference).remove(normalizedFilePath);
+      }
+    }
+
+    @Override
+    public void onFileChange(File file) {
+      try {
+        // When a simple descriptor has been created or modified, generate the new topology descriptor
+        Map<String, File> result = SimpleDescriptorHandler.handle(file, topologiesDir, aliasService);
+
+        // Add the provider config reference relationship for handling updates to the provider config
+        String providerConfig = FilenameUtils.normalize(result.get("reference").getAbsolutePath());
+        if (!providerConfigReferences.containsKey(providerConfig)) {
+          providerConfigReferences.put(providerConfig, new ArrayList<String>());
+        }
+        List<String> refs = providerConfigReferences.get(providerConfig);
+        String descriptorName = FilenameUtils.normalize(file.getAbsolutePath());
+        if (!refs.contains(descriptorName)) {
+          // Need to check if descriptor had previously referenced another provider config, so it can be removed
+          for (List<String> descs : providerConfigReferences.values()) {
+            if (descs.contains(descriptorName)) {
+              descs.remove(descriptorName);
+            }
+          }
+
+          // Add the current reference relationship
+          refs.add(descriptorName);
+        }
+      } catch (Exception e) {
+        log.simpleDescriptorHandlingError(file.getName(), e);
+      }
+    }
+
+    @Override
+    public boolean accept(File file) {
+      boolean accept = false;
+      if (!file.isDirectory() && file.canRead()) {
+        String extension = FilenameUtils.getExtension(file.getName());
+        if (SUPPORTED_EXTENSIONS.contains(extension)) {
+          accept = true;
+        }
+      }
+      return accept;
+    }
+  }
+
+  /**
+   * Change handler for shared provider configurations
+   */
+  public static class SharedProviderConfigMonitor extends FileAlterationListenerAdaptor
+          implements FileFilter {
+
+    static final List<String> SUPPORTED_EXTENSIONS = new ArrayList<>();
+    static {
+      SUPPORTED_EXTENSIONS.add("xml");
     }
 
+    private DescriptorsMonitor descriptorsMonitor;
+    private File descriptorsDir;
+
+
+    SharedProviderConfigMonitor(DescriptorsMonitor descMonitor, File descriptorsDir) {
+      this.descriptorsMonitor = descMonitor;
+      this.descriptorsDir     = descriptorsDir;
+    }
+
+    @Override
+    public void onFileCreate(File file) {
+      onFileChange(file);
+    }
+
+    @Override
+    public void onFileDelete(File file) {
+      onFileChange(file);
+    }
+
+    @Override
+    public void onFileChange(File file) {
+      // For shared provider configuration, we need to update any simple descriptors that reference it
+      for (File descriptor : getReferencingDescriptors(file)) {
+        descriptor.setLastModified(System.currentTimeMillis());
+      }
+    }
+
+    private List<File> getReferencingDescriptors(File sharedProviderConfig) {
+      List<File> references = new ArrayList<>();
+
+      for (File descriptor : descriptorsDir.listFiles()) {
+        if (DescriptorsMonitor.SUPPORTED_EXTENSIONS.contains(FilenameUtils.getExtension(descriptor.getName()))) {
+          for (String reference : descriptorsMonitor.getReferencingDescriptors(FilenameUtils.normalize(sharedProviderConfig.getAbsolutePath()))) {
+            references.add(new File(reference));
+          }
+        }
+      }
+
+      return references;
+    }
+
+    @Override
+    public boolean accept(File file) {
+      boolean accept = false;
+      if (!file.isDirectory() && file.canRead()) {
+        String extension = FilenameUtils.getExtension(file.getName());
+        if (SUPPORTED_EXTENSIONS.contains(extension)) {
+          accept = true;
+        }
+      }
+      return accept;
+    }
   }
+
 }

http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/builder/BeanPropertyTopologyBuilder.java
----------------------------------------------------------------------
diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/builder/BeanPropertyTopologyBuilder.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/builder/BeanPropertyTopologyBuilder.java
index b5e80d2..a30cf13 100644
--- a/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/builder/BeanPropertyTopologyBuilder.java
+++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/builder/BeanPropertyTopologyBuilder.java
@@ -77,7 +77,7 @@ public class BeanPropertyTopologyBuilder implements TopologyBuilder {
         Topology topology = new Topology();
         topology.setName(name);
 
-          for (Provider provider : providers) {
+        for (Provider provider : providers) {
             topology.addProvider(provider);
         }
 

http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/discovery/DefaultServiceDiscoveryConfig.java
----------------------------------------------------------------------
diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/discovery/DefaultServiceDiscoveryConfig.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/discovery/DefaultServiceDiscoveryConfig.java
new file mode 100644
index 0000000..6534b5e
--- /dev/null
+++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/discovery/DefaultServiceDiscoveryConfig.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.gateway.topology.discovery;
+
+public class DefaultServiceDiscoveryConfig implements ServiceDiscoveryConfig {
+    private String address  = null;
+    private String user     = null;
+    private String pwdAlias = null;
+
+    public DefaultServiceDiscoveryConfig(String address) {
+        this.address = address;
+    }
+
+    public void setUser(String username) {
+        this.user = username;
+    }
+
+    public void setPasswordAlias(String alias) {
+        this.pwdAlias = alias;
+    }
+
+    public String getAddress() {
+        return address;
+    }
+
+    public String getUser() {
+        return user;
+    }
+
+    public String getPasswordAlias() {
+        return pwdAlias;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/discovery/ServiceDiscoveryFactory.java
----------------------------------------------------------------------
diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/discovery/ServiceDiscoveryFactory.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/discovery/ServiceDiscoveryFactory.java
new file mode 100644
index 0000000..70d5f61
--- /dev/null
+++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/discovery/ServiceDiscoveryFactory.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.gateway.topology.discovery;
+
+import org.apache.hadoop.gateway.services.Service;
+
+import java.lang.reflect.Field;
+import java.util.ServiceLoader;
+
+/**
+ * Creates instances of ServiceDiscovery implementations.
+ *
+ * This factory uses the ServiceLoader mechanism to load ServiceDiscovery implementations as extensions.
+ *
+ */
+public abstract class ServiceDiscoveryFactory {
+
+    private static final Service[] NO_GATEWAY_SERVICS = new Service[]{};
+
+
+    public static ServiceDiscovery get(String type) {
+        return get(type, NO_GATEWAY_SERVICS);
+    }
+
+
+    public static ServiceDiscovery get(String type, Service...gatewayServices) {
+        ServiceDiscovery sd  = null;
+
+        // Look up the available ServiceDiscovery types
+        ServiceLoader<ServiceDiscoveryType> loader = ServiceLoader.load(ServiceDiscoveryType.class);
+        for (ServiceDiscoveryType sdt : loader) {
+            if (sdt.getType().equalsIgnoreCase(type)) {
+                try {
+                    ServiceDiscovery instance = sdt.newInstance();
+                    // Make sure the type reported by the instance matches the type declared by the factory
+                    // (is this necessary?)
+                    if (instance.getType().equalsIgnoreCase(type)) {
+                        sd = instance;
+
+                        // Inject any gateway services that were specified, and which are referenced in the impl
+                        if (gatewayServices != null && gatewayServices.length > 0) {
+                            for (Field field : sd.getClass().getDeclaredFields()) {
+                                if (field.getDeclaredAnnotation(GatewayService.class) != null) {
+                                    for (Service s : gatewayServices) {
+                                        if (s != null) {
+                                            if (field.getType().isAssignableFrom(s.getClass())) {
+                                                field.setAccessible(true);
+                                                field.set(sd, s);
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                        break;
+                    }
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
+            }
+        }
+
+        return sd;
+    }
+
+
+}

http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptor.java
----------------------------------------------------------------------
diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptor.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptor.java
new file mode 100644
index 0000000..aa28469
--- /dev/null
+++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptor.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.gateway.topology.simple;
+
+import java.util.List;
+
+public interface SimpleDescriptor {
+
+    String getName();
+
+    String getDiscoveryType();
+
+    String getDiscoveryAddress();
+
+    String getDiscoveryUser();
+
+    String getDiscoveryPasswordAlias();
+
+    String getClusterName();
+
+    String getProviderConfig();
+
+    List<Service> getServices();
+
+
+    interface Service {
+        String getName();
+
+        List<String> getURLs();
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorFactory.java
----------------------------------------------------------------------
diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorFactory.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorFactory.java
new file mode 100644
index 0000000..3df6d2f
--- /dev/null
+++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorFactory.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.gateway.topology.simple;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+import org.apache.commons.io.FilenameUtils;
+
+import java.io.File;
+import java.io.IOException;
+
+
+public class SimpleDescriptorFactory {
+
+    /**
+     * Create a SimpleDescriptor from the specified file.
+     *
+     * @param path The path to the file.
+     * @return A SimpleDescriptor based on the contents of the file.
+     *
+     * @throws IOException
+     */
+    public static SimpleDescriptor parse(String path) throws IOException {
+        SimpleDescriptor sd;
+
+        if (path.endsWith(".json")) {
+            sd = parseJSON(path);
+        } else if (path.endsWith(".yml")) {
+            sd = parseYAML(path);
+        } else {
+           throw new IllegalArgumentException("Unsupported simple descriptor format: " + path.substring(path.lastIndexOf('.')));
+        }
+
+        return sd;
+    }
+
+
+    static SimpleDescriptor parseJSON(String path) throws IOException {
+        final ObjectMapper mapper = new ObjectMapper();
+        SimpleDescriptorImpl sd = mapper.readValue(new File(path), SimpleDescriptorImpl.class);
+        if (sd != null) {
+            sd.setName(FilenameUtils.getBaseName(path));
+        }
+        return sd;
+    }
+
+
+    static SimpleDescriptor parseYAML(String path) throws IOException {
+        final ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
+        SimpleDescriptorImpl sd = mapper.readValue(new File(path), SimpleDescriptorImpl.class);
+        if (sd != null) {
+            sd.setName(FilenameUtils.getBaseName(path));
+        }
+        return sd;
+    }
+
+}


Mime
View raw message