knox-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lmc...@apache.org
Subject [11/37] knox git commit: KNOX-1014 - Service Discovery and Topology Generation Framework (Phil Zampino via lmccay)
Date Thu, 26 Oct 2017 14:22:47 GMT
KNOX-1014 - Service Discovery and Topology Generation Framework (Phil Zampino via lmccay)

Project: http://git-wip-us.apache.org/repos/asf/knox/repo
Commit: http://git-wip-us.apache.org/repos/asf/knox/commit/c2ca4432
Tree: http://git-wip-us.apache.org/repos/asf/knox/tree/c2ca4432
Diff: http://git-wip-us.apache.org/repos/asf/knox/diff/c2ca4432

Branch: refs/heads/KNOX-1049
Commit: c2ca443262a848ae0d56e03e92ecba32fbf149f2
Parents: 3a0119b
Author: Larry McCay <lmccay@hortonworks.com>
Authored: Sat Sep 23 11:04:39 2017 -0400
Committer: Larry McCay <lmccay@hortonworks.com>
Committed: Sat Sep 23 11:04:39 2017 -0400

----------------------------------------------------------------------
 b/gateway-discovery-ambari/pom.xml              |  66 ++
 .../discovery/ambari/AmbariCluster.java         | 114 +++
 .../discovery/ambari/AmbariComponent.java       |  76 ++
 .../ambari/AmbariServiceDiscovery.java          | 291 +++++++
 .../ambari/AmbariServiceDiscoveryMessages.java  |  81 ++
 .../ambari/AmbariServiceDiscoveryType.java      |  35 +
 .../ambari/AmbariServiceURLCreator.java         | 184 ++++
 ...eway.topology.discovery.ServiceDiscoveryType |  19 +
 .../ambari/AmbariServiceDiscoveryTest.java      | 856 +++++++++++++++++++
 b/gateway-release/home/conf/descriptors/README  |   1 +
 .../home/conf/shared-providers/README           |   1 +
 gateway-discovery-ambari/pom.xml                |  66 ++
 .../discovery/ambari/AmbariCluster.java         | 114 +++
 .../discovery/ambari/AmbariComponent.java       |  76 ++
 .../ambari/AmbariServiceDiscovery.java          | 291 +++++++
 .../ambari/AmbariServiceDiscoveryMessages.java  |  81 ++
 .../ambari/AmbariServiceDiscoveryType.java      |  35 +
 .../ambari/AmbariServiceURLCreator.java         | 184 ++++
 ...eway.topology.discovery.ServiceDiscoveryType |  19 +
 .../ambari/AmbariServiceDiscoveryTest.java      | 856 +++++++++++++++++++
 gateway-release/home/conf/descriptors/README    |   1 +
 .../home/conf/shared-providers/README           |   1 +
 gateway-release/pom.xml                         |   4 +
 gateway-server/pom.xml                          |   5 +
 .../apache/hadoop/gateway/GatewayMessages.java  |   9 +-
 .../services/DefaultGatewayServices.java        |   3 +-
 .../topology/impl/DefaultTopologyService.java   | 278 +++++-
 .../builder/BeanPropertyTopologyBuilder.java    |   2 +-
 .../DefaultServiceDiscoveryConfig.java          |  48 ++
 .../discovery/ServiceDiscoveryFactory.java      |  81 ++
 .../topology/simple/SimpleDescriptor.java       |  46 +
 .../simple/SimpleDescriptorFactory.java         |  71 ++
 .../simple/SimpleDescriptorHandler.java         | 186 ++++
 .../topology/simple/SimpleDescriptorImpl.java   | 111 +++
 .../simple/SimpleDescriptorMessages.java        |  44 +
 .../topology/DefaultTopologyServiceTest.java    |  70 +-
 .../PropertiesFileServiceDiscoveryTest.java     |  90 ++
 .../discovery/ServiceDiscoveryFactoryTest.java  |  81 ++
 .../test/extension/DummyServiceDiscovery.java   |  66 ++
 .../extension/DummyServiceDiscoveryType.java    |  32 +
 .../PropertiesFileServiceDiscovery.java         | 108 +++
 .../PropertiesFileServiceDiscoveryType.java     |  35 +
 .../extension/SneakyServiceDiscoveryImpl.java   |  40 +
 .../extension/SneakyServiceDiscoveryType.java   |  33 +
 .../simple/SimpleDescriptorFactoryTest.java     | 218 +++++
 .../simple/SimpleDescriptorHandlerTest.java     | 239 ++++++
 ...eway.topology.discovery.ServiceDiscoveryType |  21 +
 .../topology/file/ambari-cluster-policy.xml     |  74 ++
 .../topology/file/simple-topology-four.json     |  18 +
 .../topology/discovery/GatewayService.java      |  29 +
 .../topology/discovery/ServiceDiscovery.java    |  76 ++
 .../discovery/ServiceDiscoveryConfig.java       |  42 +
 .../discovery/ServiceDiscoveryType.java         |  40 +
 pom.xml                                         |  27 +-
 54 files changed, 5626 insertions(+), 49 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/b/gateway-discovery-ambari/pom.xml
----------------------------------------------------------------------
diff --git a/b/gateway-discovery-ambari/pom.xml b/b/gateway-discovery-ambari/pom.xml
new file mode 100644
index 0000000..924e89c
--- /dev/null
+++ b/b/gateway-discovery-ambari/pom.xml
@@ -0,0 +1,66 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.apache.knox</groupId>
+        <artifactId>gateway</artifactId>
+        <version>0.14.0-SNAPSHOT</version>
+    </parent>
+    <artifactId>gateway-discovery-ambari</artifactId>
+
+    <name>gateway-discovery-ambari</name>
+    <description>The extension to the gateway for service discovery using Apache Ambari.</description>
+
+    <licenses>
+        <license>
+            <name>The Apache Software License, Version 2.0</name>
+            <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+            <distribution>repo</distribution>
+        </license>
+    </licenses>
+
+    <dependencies>
+        <dependency>
+            <groupId>${gateway-group}</groupId>
+            <artifactId>gateway-spi</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>${gateway-group}</groupId>
+            <artifactId>gateway-test-utils</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.easymock</groupId>
+            <artifactId>easymock</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+    </dependencies>
+
+</project>

http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariCluster.java
----------------------------------------------------------------------
diff --git a/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariCluster.java
b/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariCluster.java
new file mode 100644
index 0000000..6eaabd3
--- /dev/null
+++ b/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariCluster.java
@@ -0,0 +1,114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.gateway.topology.discovery.ambari;
+
+import org.apache.hadoop.gateway.topology.discovery.ServiceDiscovery;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+class AmbariCluster implements ServiceDiscovery.Cluster {
+
+    private String name = null;
+
+    private AmbariServiceURLCreator urlCreator = new AmbariServiceURLCreator();
+
+    private Map<String, Map<String, ServiceConfiguration>> serviceConfigurations
= new HashMap<>();
+
+    private Map<String, AmbariComponent> components = null;
+
+
+    AmbariCluster(String name) {
+        this.name = name;
+        components = new HashMap<String, AmbariComponent>();
+    }
+
+    void addServiceConfiguration(String serviceName, String configurationType, ServiceConfiguration
serviceConfig) {
+        if (!serviceConfigurations.keySet().contains(serviceName)) {
+            serviceConfigurations.put(serviceName, new HashMap<String, ServiceConfiguration>());
+        }
+        serviceConfigurations.get(serviceName).put(configurationType, serviceConfig);
+    }
+
+
+    void addComponent(AmbariComponent component) {
+        components.put(component.getName(), component);
+    }
+
+
+    ServiceConfiguration getServiceConfiguration(String serviceName, String configurationType)
{
+        ServiceConfiguration sc = null;
+        Map<String, ServiceConfiguration> configs = serviceConfigurations.get(serviceName);
+        if (configs != null) {
+            sc = configs.get(configurationType);
+        }
+        return sc;
+    }
+
+
+    Map<String, AmbariComponent> getComponents() {
+        return components;
+    }
+
+
+    AmbariComponent getComponent(String name) {
+        return components.get(name);
+    }
+
+
+    @Override
+    public String getName() {
+        return name;
+    }
+
+
+    @Override
+    public List<String> getServiceURLs(String serviceName) {
+        List<String> urls = new ArrayList<>();
+        urls.addAll(urlCreator.create(this, serviceName));
+        return urls;
+    }
+
+
+    static class ServiceConfiguration {
+
+        private String type;
+        private String version;
+        private Map<String, String> props;
+
+        ServiceConfiguration(String type, String version, Map<String, String> properties)
{
+            this.type = type;
+            this.version = version;
+            this.props = properties;
+        }
+
+        public String getVersion() {
+            return version;
+        }
+
+        public String getType() {
+            return type;
+        }
+
+        public Map<String, String> getProperties() {
+            return props;
+        }
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariComponent.java
----------------------------------------------------------------------
diff --git a/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariComponent.java
b/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariComponent.java
new file mode 100644
index 0000000..55257fb
--- /dev/null
+++ b/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariComponent.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.gateway.topology.discovery.ambari;
+
+import java.util.List;
+import java.util.Map;
+
+class AmbariComponent {
+
+    private String clusterName = null;
+    private String serviceName = null;
+    private String name        = null;
+    private String version     = null;
+
+    private List<String> hostNames = null;
+
+    private Map<String, String> properties = null;
+
+    AmbariComponent(String              name,
+                    String              version,
+                    String              cluster,
+                    String              service,
+                    List<String>        hostNames,
+                    Map<String, String> properties) {
+        this.name = name;
+        this.serviceName = service;
+        this.clusterName = cluster;
+        this.version = version;
+        this.hostNames = hostNames;
+        this.properties = properties;
+    }
+
+    public String getVersion() {
+        return version;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public String getServiceName() {
+        return serviceName;
+    }
+
+    public String getClusterName() {
+        return clusterName;
+    }
+
+    public List<String> getHostNames() {
+        return hostNames;
+    }
+
+    public Map<String, String> getConfigProperties() {
+        return properties;
+    }
+
+    public String getConfigProperty(String propertyName) {
+        return properties.get(propertyName);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscovery.java
----------------------------------------------------------------------
diff --git a/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscovery.java
b/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscovery.java
new file mode 100644
index 0000000..34f20a7
--- /dev/null
+++ b/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscovery.java
@@ -0,0 +1,291 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.gateway.topology.discovery.ambari;
+
+import net.minidev.json.JSONArray;
+import net.minidev.json.JSONObject;
+import net.minidev.json.JSONValue;
+import org.apache.hadoop.gateway.config.ConfigurationException;
+import org.apache.hadoop.gateway.i18n.messages.MessagesFactory;
+import org.apache.hadoop.gateway.services.security.AliasService;
+import org.apache.hadoop.gateway.services.security.AliasServiceException;
+import org.apache.hadoop.gateway.topology.discovery.GatewayService;
+import org.apache.hadoop.gateway.topology.discovery.ServiceDiscovery;
+import org.apache.hadoop.gateway.topology.discovery.ServiceDiscoveryConfig;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpStatus;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.message.BasicHeader;
+import org.apache.http.util.EntityUtils;
+
+import java.io.IOException;
+import java.util.*;
+
+
+class AmbariServiceDiscovery implements ServiceDiscovery {
+
+    static final String TYPE = "AMBARI";
+
+    static final String AMBARI_CLUSTERS_URI = "/api/v1/clusters";
+
+    static final String AMBARI_HOSTROLES_URI =
+                                       AMBARI_CLUSTERS_URI + "/%s/services?fields=components/host_components/HostRoles";
+
+    static final String AMBARI_SERVICECONFIGS_URI =
+            AMBARI_CLUSTERS_URI + "/%s/configurations/service_config_versions?is_current=true";
+
+    // Map of component names to service configuration types
+    private static Map<String, String> componentServiceConfigs = new HashMap<>();
+    static {
+        componentServiceConfigs.put("NAMENODE", "hdfs-site");
+        componentServiceConfigs.put("RESOURCEMANAGER", "yarn-site");
+        componentServiceConfigs.put("OOZIE_SERVER", "oozie-site");
+        componentServiceConfigs.put("HIVE_SERVER", "hive-site");
+        componentServiceConfigs.put("WEBHCAT_SERVER", "webhcat-site");
+        componentServiceConfigs.put("HBASE_MASTER", "hbase-site");
+    } // TODO: Are there other service components, for which the endpoints can be discovered
via Ambari?
+
+    private static final String DEFAULT_USER_ALIAS = "ambari.discovery.user";
+    private static final String DEFAULT_PWD_ALIAS  = "ambari.discovery.password";
+
+    private static AmbariServiceURLCreator urlCreator = new AmbariServiceURLCreator();
+
+    private AmbariServiceDiscoveryMessages log = MessagesFactory.get(AmbariServiceDiscoveryMessages.class);
+
+    @GatewayService
+    private AliasService aliasService;
+
+    private CloseableHttpClient httpClient = null;
+
+    private Map<String, Map<String, String>> serviceConfiguration = new HashMap<>();
+
+
+    AmbariServiceDiscovery() {
+        httpClient = org.apache.http.impl.client.HttpClients.createDefault();
+    }
+
+
+    @Override
+    public String getType() {
+        return TYPE;
+    }
+
+
+    @Override
+    public Map<String, Cluster> discover(ServiceDiscoveryConfig config) {
+        Map<String, Cluster> clusters = new HashMap<String, Cluster>();
+
+        String discoveryAddress = config.getAddress();
+
+        // Invoke Ambari REST API to discover the available clusters
+        String clustersDiscoveryURL = String.format("%s" + AMBARI_CLUSTERS_URI, discoveryAddress);
+
+        JSONObject json = invokeREST(clustersDiscoveryURL, config.getUser(), config.getPasswordAlias());
+
+        // Parse the cluster names from the response, and perform the cluster discovery
+        JSONArray clusterItems = (JSONArray) json.get("items");
+        for (Object clusterItem : clusterItems) {
+            String clusterName = (String) ((JSONObject)((JSONObject) clusterItem).get("Clusters")).get("cluster_name");
+            try {
+                Cluster c = discover(config, clusterName);
+                clusters.put(clusterName, c);
+            } catch (Exception e) {
+                log.clusterDiscoveryError(clusterName, e);
+            }
+        }
+
+        return clusters;
+    }
+
+
+    @Override
+    public Cluster discover(ServiceDiscoveryConfig config, String clusterName) {
+        AmbariCluster cluster = new AmbariCluster(clusterName);
+
+        Map<String, String> serviceComponents = new HashMap<>();
+
+        String discoveryAddress = config.getAddress();
+        String discoveryUser = config.getUser();
+        String discoveryPwdAlias = config.getPasswordAlias();
+
+        Map<String, List<String>> componentHostNames = new HashMap<>();
+        String hostRolesURL = String.format("%s" + AMBARI_HOSTROLES_URI, discoveryAddress,
clusterName);
+        JSONObject hostRolesJSON = invokeREST(hostRolesURL, discoveryUser, discoveryPwdAlias);
+        if (hostRolesJSON != null) {
+            // Process the host roles JSON
+            JSONArray items = (JSONArray) hostRolesJSON.get("items");
+            for (Object obj : items) {
+                JSONArray components = (JSONArray) ((JSONObject) obj).get("components");
+                for (Object component : components) {
+                    JSONArray hostComponents = (JSONArray) ((JSONObject) component).get("host_components");
+                    for (Object hostComponent : hostComponents) {
+                        JSONObject hostRoles = (JSONObject) ((JSONObject) hostComponent).get("HostRoles");
+                        String serviceName = (String) hostRoles.get("service_name");
+                        String componentName = (String) hostRoles.get("component_name");
+
+                        serviceComponents.put(componentName, serviceName);
+
+//                    String hostName = (String) hostRoles.get("host_name");
+                        String hostName = (String) hostRoles.get("public_host_name"); //
Assuming public host name is most applicable
+                        log.discoveredServiceHost(serviceName, hostName);
+                        if (!componentHostNames.containsKey(componentName)) {
+                            componentHostNames.put(componentName, new ArrayList<String>());
+                        }
+                        componentHostNames.get(componentName).add(hostName);
+                    }
+                }
+            }
+        }
+
+        Map<String, Map<String, AmbariCluster.ServiceConfiguration>> serviceConfigurations
=
+                                                 new HashMap<String, Map<String, AmbariCluster.ServiceConfiguration>>();
+        String serviceConfigsURL = String.format("%s" + AMBARI_SERVICECONFIGS_URI, discoveryAddress,
clusterName);
+        JSONObject serviceConfigsJSON = invokeREST(serviceConfigsURL, discoveryUser, discoveryPwdAlias);
+        if (serviceConfigsJSON != null) {
+            // Process the service configurations
+            JSONArray serviceConfigs = (JSONArray) serviceConfigsJSON.get("items");
+            for (Object serviceConfig : serviceConfigs) {
+                String serviceName = (String) ((JSONObject) serviceConfig).get("service_name");
+                JSONArray configurations = (JSONArray) ((JSONObject) serviceConfig).get("configurations");
+                for (Object configuration : configurations) {
+                    String configType = (String) ((JSONObject) configuration).get("type");
+                    String configVersion = String.valueOf(((JSONObject) configuration).get("version"));
+
+                    Map<String, String> configProps = new HashMap<String, String>();
+                    JSONObject configProperties = (JSONObject) ((JSONObject) configuration).get("properties");
+                    for (String propertyName : configProperties.keySet()) {
+                        configProps.put(propertyName, String.valueOf(((JSONObject) configProperties).get(propertyName)));
+                    }
+                    if (!serviceConfigurations.containsKey(serviceName)) {
+                        serviceConfigurations.put(serviceName, new HashMap<String, AmbariCluster.ServiceConfiguration>());
+                    }
+                    serviceConfigurations.get(serviceName).put(configType, new AmbariCluster.ServiceConfiguration(configType,
configVersion, configProps));
+                    cluster.addServiceConfiguration(serviceName, configType, new AmbariCluster.ServiceConfiguration(configType,
configVersion, configProps));
+                }
+            }
+        }
+
+        // Construct the AmbariCluster model
+        for (String componentName : serviceComponents.keySet()) {
+            String serviceName = serviceComponents.get(componentName);
+            List<String> hostNames = componentHostNames.get(componentName);
+
+            Map<String, AmbariCluster.ServiceConfiguration> configs = serviceConfigurations.get(serviceName);
+            String configType = componentServiceConfigs.get(componentName);
+            if (configType != null) {
+                AmbariCluster.ServiceConfiguration svcConfig = configs.get(configType);
+                AmbariComponent c = new AmbariComponent(componentName,
+                                                        svcConfig.getVersion(),
+                                                        clusterName,
+                                                        serviceName,
+                                                        hostNames,
+                                                        svcConfig.getProperties());
+                cluster.addComponent(c);
+            }
+        }
+
+        return cluster;
+    }
+
+
+    protected JSONObject invokeREST(String url, String username, String passwordAlias) {
+        JSONObject result = null;
+
+        CloseableHttpResponse response = null;
+        try {
+            HttpGet request = new HttpGet(url);
+
+            // If no configured username, then use default username alias
+            String password = null;
+            if (username == null) {
+                if (aliasService != null) {
+                    try {
+                        char[] defaultUser = aliasService.getPasswordFromAliasForGateway(DEFAULT_USER_ALIAS);
+                        if (defaultUser != null) {
+                            username = new String(defaultUser);
+                        }
+                    } catch (AliasServiceException e) {
+                        log.aliasServiceUserError(DEFAULT_USER_ALIAS, e.getLocalizedMessage());
+                    }
+                }
+
+                // If username is still null
+                if (username == null) {
+                    log.aliasServiceUserNotFound();
+                    throw new ConfigurationException("No username is configured for Ambari
service discovery.");
+                }
+            }
+
+            if (aliasService != null) {
+                // If not password alias is configured, then try the default alias
+                if (passwordAlias == null) {
+                    passwordAlias = DEFAULT_PWD_ALIAS;
+                }
+                try {
+                    char[] pwd = aliasService.getPasswordFromAliasForGateway(passwordAlias);
+                    if (pwd != null) {
+                        password = new String(pwd);
+                    }
+
+                } catch (AliasServiceException e) {
+                    log.aliasServicePasswordError(passwordAlias, e.getLocalizedMessage());
+                }
+            }
+
+            // If the password could not be determined
+            if (password == null) {
+                log.aliasServicePasswordNotFound();
+                throw new ConfigurationException("No password is configured for Ambari service
discovery.");
+            }
+
+            // Add an auth header if credentials are available
+            String encodedCreds =
+                    org.apache.commons.codec.binary.Base64.encodeBase64String((username +
":" + password).getBytes());
+            request.addHeader(new BasicHeader("Authorization", "Basic " + encodedCreds));
+
+            response = httpClient.execute(request);
+
+            if (HttpStatus.SC_OK == response.getStatusLine().getStatusCode()) {
+                HttpEntity entity = response.getEntity();
+                if (entity != null) {
+                    result = (JSONObject) JSONValue.parse((EntityUtils.toString(entity)));
+                    log.debugJSON(result.toJSONString());
+                } else {
+                    log.noJSON(url);
+                }
+            } else {
+                log.unexpectedRestResponseStatusCode(url, response.getStatusLine().getStatusCode());
+            }
+
+        } catch (IOException e) {
+            log.restInvocationError(url, e);
+        } finally {
+            if(response != null) {
+                try {
+                    response.close();
+                } catch (IOException e) {
+                    // Ignore
+                }
+            }
+        }
+        return result;
+    }
+
+
+}

http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscoveryMessages.java
----------------------------------------------------------------------
diff --git a/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscoveryMessages.java
b/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscoveryMessages.java
new file mode 100644
index 0000000..caa16ed
--- /dev/null
+++ b/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscoveryMessages.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.gateway.topology.discovery.ambari;
+
+import org.apache.hadoop.gateway.i18n.messages.Message;
+import org.apache.hadoop.gateway.i18n.messages.MessageLevel;
+import org.apache.hadoop.gateway.i18n.messages.Messages;
+import org.apache.hadoop.gateway.i18n.messages.StackTrace;
+
+@Messages(logger="org.apache.gateway.topology.discovery.ambari")
+public interface AmbariServiceDiscoveryMessages {
+
+    @Message(level = MessageLevel.ERROR,
+            text = "Encountered an error during cluster {0} discovery: {1}")
+    void clusterDiscoveryError(final String clusterName,
+                               @StackTrace(level = MessageLevel.ERROR) Exception e);
+
+
+    @Message(level = MessageLevel.DEBUG,
+            text = "REST invocation {0} failed: {1}")
+    void restInvocationError(final String url,
+                             @StackTrace(level = MessageLevel.ERROR) Exception e);
+
+
+    @Message(level = MessageLevel.ERROR,
+            text = "Encountered an error attempting to determine the user for alias {0} :
{1}")
+    void aliasServiceUserError(final String alias, final String error);
+
+
+    @Message(level = MessageLevel.ERROR,
+            text = "Encountered an error attempting to determine the password for alias {0}
: {1}")
+    void aliasServicePasswordError(final String alias, final String error);
+
+
+    @Message(level = MessageLevel.ERROR,
+            text = "No user configured for Ambari service discovery.")
+    void aliasServiceUserNotFound();
+
+
+    @Message(level = MessageLevel.ERROR,
+            text = "No password configured for Ambari service discovery.")
+    void aliasServicePasswordNotFound();
+
+
+    @Message(level = MessageLevel.ERROR,
+            text = "Unexpected REST invocation response code for {0} : {1}")
+    void unexpectedRestResponseStatusCode(final String url, int responseStatusCode);
+
+
+    @Message(level = MessageLevel.ERROR,
+            text = "REST invocation {0} yielded a response without any JSON.")
+    void noJSON(final String url);
+
+
+    @Message(level = MessageLevel.DEBUG,
+            text = "REST invocation result: {0}")
+    void debugJSON(final String json);
+
+
+    @Message(level = MessageLevel.INFO,
+            text = "Discovered: Service: {0}, Host: {1}")
+    void discoveredServiceHost(final String serviceName, final String hostName);
+
+
+
+
+}

http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscoveryType.java
----------------------------------------------------------------------
diff --git a/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscoveryType.java
b/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscoveryType.java
new file mode 100644
index 0000000..723a786
--- /dev/null
+++ b/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscoveryType.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.gateway.topology.discovery.ambari;
+
+import org.apache.hadoop.gateway.topology.discovery.ServiceDiscovery;
+import org.apache.hadoop.gateway.topology.discovery.ServiceDiscoveryType;
+
+public class AmbariServiceDiscoveryType implements ServiceDiscoveryType {
+
+    private static final String IMPL = AmbariServiceDiscovery.class.getCanonicalName();
+
+    @Override
+    public String getType() {
+        return AmbariServiceDiscovery.TYPE;
+    }
+
+    @Override
+    public ServiceDiscovery newInstance() {
+        return new AmbariServiceDiscovery();
+    }
+}

http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceURLCreator.java
----------------------------------------------------------------------
diff --git a/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceURLCreator.java
b/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceURLCreator.java
new file mode 100644
index 0000000..0674642
--- /dev/null
+++ b/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceURLCreator.java
@@ -0,0 +1,184 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.gateway.topology.discovery.ambari;
+
+
+import java.util.ArrayList;
+import java.util.List;
+
+class AmbariServiceURLCreator {
+
+    private static final String NAMENODE_SERVICE        = "NAMENODE";
+    private static final String JOBTRACKER_SERVICE      = "JOBTRACKER";
+    private static final String WEBHDFS_SERVICE         = "WEBHDFS";
+    private static final String WEBHCAT_SERVICE         = "WEBHCAT";
+    private static final String OOZIE_SERVICE           = "OOZIE";
+    private static final String WEBHBASE_SERVICE        = "WEBHBASE";
+    private static final String HIVE_SERVICE            = "HIVE";
+    private static final String RESOURCEMANAGER_SERVICE = "RESOURCEMANAGER";
+
+
+    /**
+     * Derive the endpoint URL(s) for the specified service, based on the info from the specified
Cluster.
+     *
+     * @param cluster The cluster discovery results
+     * @param serviceName The name of a Hadoop service
+     *
+     * @return One or more endpoint URLs for the specified service.
+     */
+    public List<String> create(AmbariCluster cluster, String serviceName) {
+        List<String> result = null;
+
+        if (NAMENODE_SERVICE.equals(serviceName)) {
+            result = createNameNodeURL(cluster);
+        } else if (JOBTRACKER_SERVICE.equals(serviceName)) {
+            result = createJobTrackerURL(cluster);
+        } else if (WEBHDFS_SERVICE.equals(serviceName)) {
+            result = createWebHDFSURL(cluster);
+        } else if (WEBHCAT_SERVICE.equals(serviceName)) {
+            result = createWebHCatURL(cluster);
+        } else if (OOZIE_SERVICE.equals(serviceName)) {
+            result = createOozieURL(cluster);
+        } else if (WEBHBASE_SERVICE.equals(serviceName)) {
+            result = createWebHBaseURL(cluster);
+        } else if (HIVE_SERVICE.equals(serviceName)) {
+            result = createHiveURL(cluster);
+        } else if (RESOURCEMANAGER_SERVICE.equals(serviceName)) {
+            result = createResourceManagerURL(cluster);
+        }
+
+        return result;
+    }
+
+
+    private List<String> createNameNodeURL(AmbariCluster cluster) {
+        List<String> result = new ArrayList<>();
+
+        AmbariComponent comp = cluster.getComponent("NAMENODE");
+        if (comp != null) {
+            result.add("hdfs://" + comp.getConfigProperty("dfs.namenode.rpc-address"));
+        }
+
+        return result;
+    }
+
+
+    private List<String> createJobTrackerURL(AmbariCluster cluster) {
+        List<String> result = new ArrayList<>();
+
+        AmbariComponent comp = cluster.getComponent("RESOURCEMANAGER");
+        if (comp != null) {
+            result.add("rpc://" + comp.getConfigProperty("yarn.resourcemanager.address"));
+        }
+
+        return result;
+    }
+
+
+    private List<String> createWebHDFSURL(AmbariCluster cluster) {
+        List<String> result = new ArrayList<>();
+
+        AmbariCluster.ServiceConfiguration sc = cluster.getServiceConfiguration("HDFS", "hdfs-site");
+        if (sc != null) {
+            String address = sc.getProperties().get("dfs.namenode.http-address");
+            result.add("http://" + address + "/webhdfs");
+        }
+
+        return result;
+    }
+
+
+    private List<String> createWebHCatURL(AmbariCluster cluster) {
+        List<String> result = new ArrayList<>();
+
+        AmbariComponent webhcat = cluster.getComponent("WEBHCAT_SERVER");
+        if (webhcat != null) {
+            String port = webhcat.getConfigProperty("templeton.port");
+            String host = webhcat.getHostNames().get(0);
+
+            result.add("http://" + host + ":" + port + "/templeton");
+        }
+        return result;
+    }
+
+
+    private List<String> createOozieURL(AmbariCluster cluster) {
+        List<String> result = new ArrayList<>();
+
+        AmbariComponent comp = cluster.getComponent("OOZIE_SERVER");
+        if (comp != null) {
+            result.add(comp.getConfigProperty("oozie.base.url"));
+        }
+
+        return result;
+    }
+
+
+    private List<String> createWebHBaseURL(AmbariCluster cluster) {
+        List<String> result = new ArrayList<>();
+
+        AmbariComponent comp = cluster.getComponent("HBASE_MASTER");
+        if (comp != null) {
+            for (String host : comp.getHostNames()) {
+                result.add("http://" + host + ":60080");
+            }
+        }
+
+        return result;
+    }
+
+
+    private List<String> createHiveURL(AmbariCluster cluster) {
+        List<String> result = new ArrayList<>();
+
+        AmbariComponent hive = cluster.getComponent("HIVE_SERVER");
+        if (hive != null) {
+            String path = hive.getConfigProperty("hive.server2.thrift.http.path");
+            String port = hive.getConfigProperty("hive.server2.thrift.http.port");
+            String transport = hive.getConfigProperty("hive.server2.transport.mode");
+            String useSSL = hive.getConfigProperty("hive.server2.use.SSL");
+            String host = hive.getHostNames().get(0);
+
+            String scheme = null; // What is the scheme for the binary transport mode?
+            if ("http".equals(transport)) {
+                scheme = Boolean.valueOf(useSSL) ? "https" : "http";
+            }
+
+            result.add(scheme + "://" + host + ":" + port + "/" + path);
+        }
+        return result;
+    }
+
+
+    private List<String> createResourceManagerURL(AmbariCluster cluster) {
+        List<String> result = new ArrayList<>();
+
+        AmbariComponent resMan = cluster.getComponent("RESOURCEMANAGER");
+        if (resMan != null) {
+            String webappAddress = resMan.getConfigProperty("yarn.resourcemanager.webapp.address");
+            String httpPolicy = resMan.getConfigProperty("yarn.http.policy");
+            String scheme = ("HTTPS_ONLY".equalsIgnoreCase(httpPolicy)) ? "https" : "http";
+
+            result.add(scheme + "://" + webappAddress + "/ws");
+        }
+
+        return result;
+    }
+
+
+}

http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/b/gateway-discovery-ambari/src/main/resources/META-INF/services/org.apache.hadoop.gateway.topology.discovery.ServiceDiscoveryType
----------------------------------------------------------------------
diff --git a/b/gateway-discovery-ambari/src/main/resources/META-INF/services/org.apache.hadoop.gateway.topology.discovery.ServiceDiscoveryType
b/b/gateway-discovery-ambari/src/main/resources/META-INF/services/org.apache.hadoop.gateway.topology.discovery.ServiceDiscoveryType
new file mode 100644
index 0000000..1da4fc9
--- /dev/null
+++ b/b/gateway-discovery-ambari/src/main/resources/META-INF/services/org.apache.hadoop.gateway.topology.discovery.ServiceDiscoveryType
@@ -0,0 +1,19 @@
+##########################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##########################################################################
+
+org.apache.hadoop.gateway.topology.discovery.ambari.AmbariServiceDiscoveryType
\ No newline at end of file


Mime
View raw message