knox-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From m...@apache.org
Subject [knox] branch master updated: KNOX-2213 - Service Discovery Support for CM UI, API (#249)
Date Mon, 03 Feb 2020 17:16:43 GMT
This is an automated email from the ASF dual-hosted git repository.

more pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/knox.git


The following commit(s) were added to refs/heads/master by this push:
     new f1cec12  KNOX-2213 - Service Discovery Support for CM UI, API (#249)
f1cec12 is described below

commit f1cec12e720cdedb398ae050f60f98577dd01051
Author: Sandeep Moré <moresandeep@gmail.com>
AuthorDate: Mon Feb 3 12:16:35 2020 -0500

    KNOX-2213 - Service Discovery Support for CM UI, API (#249)
---
 .../cm/ClouderaManagerServiceDiscovery.java        | 42 +++++++++-
 .../ClouderaManagerAPIServiceModelGenerator.java   | 94 ++++++++++++++++++++++
 .../cm/ClouderaManagerUIServiceModelGenerator.java | 37 +++++++++
 ...way.topology.discovery.cm.ServiceModelGenerator |  2 +
 .../cm/ClouderaManagerServiceDiscoveryTest.java    | 32 +++++++-
 5 files changed, 202 insertions(+), 5 deletions(-)

diff --git a/gateway-discovery-cm/src/main/java/org/apache/knox/gateway/topology/discovery/cm/ClouderaManagerServiceDiscovery.java
b/gateway-discovery-cm/src/main/java/org/apache/knox/gateway/topology/discovery/cm/ClouderaManagerServiceDiscovery.java
index aa66fae..a905677 100644
--- a/gateway-discovery-cm/src/main/java/org/apache/knox/gateway/topology/discovery/cm/ClouderaManagerServiceDiscovery.java
+++ b/gateway-discovery-cm/src/main/java/org/apache/knox/gateway/topology/discovery/cm/ClouderaManagerServiceDiscovery.java
@@ -44,6 +44,7 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.ServiceLoader;
 import java.util.Set;
@@ -68,6 +69,9 @@ public class ClouderaManagerServiceDiscovery implements ServiceDiscovery
{
   static final String DEFAULT_USER_ALIAS = "cm.discovery.user";
   static final String DEFAULT_PWD_ALIAS  = "cm.discovery.password";
 
+  public static final String CM_SERVICE_TYPE  = "CM";
+  public static final String CM_ROLE_TYPE  = "CM_SERVER";
+
   private static Map<String, List<ServiceModelGenerator>> serviceModelGenerators
= new HashMap<>();
   static {
     ServiceLoader<ServiceModelGenerator> loader = ServiceLoader.load(ServiceModelGenerator.class);
@@ -216,18 +220,38 @@ public class ClouderaManagerServiceDiscovery implements ServiceDiscovery
{
     Set<ServiceModel> serviceModels = new HashSet<>();
 
     ApiServiceList serviceList = getClusterServices(servicesResourceApi, clusterName);
+
     if (serviceList != null) {
+      /*
+      Since Cloudera Manager does not have a service for itself, we will add a skeleton CM
+      service so that we can add CM service to topology when auto-discovery is
+      turned on and CM service is selected in the descriptor
+      */
+      final ApiService cmService = new ApiService();
+      cmService.setName(CM_SERVICE_TYPE.toLowerCase(Locale.ROOT));
+      cmService.setType(CM_SERVICE_TYPE);
+      serviceList.addItemsItem(cmService);
+
       for (ApiService service : serviceList.getItems()) {
         String serviceName = service.getName();
         log.discoveredService(serviceName, service.getType());
-        ApiServiceConfig serviceConfig =
-            getServiceConfig(servicesResourceApi, clusterName, serviceName);
+        ApiServiceConfig serviceConfig = null;
+        /* no reason to check service config for CM service */
+        if(!CM_SERVICE_TYPE.equals(service.getType())) {
+          serviceConfig =
+              getServiceConfig(servicesResourceApi, clusterName, serviceName);
+        }
         ApiRoleList roleList = getRoles(rolesResourceApi, clusterName, serviceName);
         if (roleList != null) {
           for (ApiRole role : roleList.getItems()) {
             String roleName = role.getName();
             log.discoveredServiceRole(roleName, role.getType());
-            ApiConfigList roleConfig = getRoleConfig(rolesResourceApi, clusterName, serviceName,
roleName);
+            ApiConfigList roleConfig = null;
+            /* no reason to check role config for CM service */
+            if(!CM_SERVICE_TYPE.equals(service.getType())) {
+              roleConfig =
+                  getRoleConfig(rolesResourceApi, clusterName, serviceName, roleName);
+            }
 
             List<ServiceModelGenerator> smgList = serviceModelGenerators.get(service.getType());
             if (smgList != null) {
@@ -280,7 +304,17 @@ public class ClouderaManagerServiceDiscovery implements ServiceDiscovery
{
                                       String serviceName) {
     ApiRoleList roles = null;
     try {
-      roles = rolesResourceApi.readRoles(clusterName, serviceName, "", VIEW_SUMMARY);
+      /* Populate roles for CM Service since they are not discoverable */
+      if(CM_SERVICE_TYPE.equalsIgnoreCase(serviceName)) {
+        roles = new ApiRoleList();
+        final ApiRole cmRole = new ApiRole();
+        cmRole.setName(CM_ROLE_TYPE);
+        cmRole.setType(CM_ROLE_TYPE);
+        roles.addItemsItem(cmRole);
+        return roles;
+      } else {
+        roles = rolesResourceApi.readRoles(clusterName, serviceName, "", VIEW_SUMMARY);
+      }
     } catch (Exception e) {
       log.failedToAccessServiceRoleConfigs(clusterName, e);
     }
diff --git a/gateway-discovery-cm/src/main/java/org/apache/knox/gateway/topology/discovery/cm/model/cm/ClouderaManagerAPIServiceModelGenerator.java
b/gateway-discovery-cm/src/main/java/org/apache/knox/gateway/topology/discovery/cm/model/cm/ClouderaManagerAPIServiceModelGenerator.java
new file mode 100644
index 0000000..0978362
--- /dev/null
+++ b/gateway-discovery-cm/src/main/java/org/apache/knox/gateway/topology/discovery/cm/model/cm/ClouderaManagerAPIServiceModelGenerator.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.knox.gateway.topology.discovery.cm.model.cm;
+
+import com.cloudera.api.swagger.client.ApiException;
+import com.cloudera.api.swagger.model.ApiConfigList;
+import com.cloudera.api.swagger.model.ApiRole;
+import com.cloudera.api.swagger.model.ApiService;
+import com.cloudera.api.swagger.model.ApiServiceConfig;
+import org.apache.knox.gateway.topology.discovery.cm.ClouderaManagerServiceDiscovery;
+import org.apache.knox.gateway.topology.discovery.cm.ServiceModel;
+import org.apache.knox.gateway.topology.discovery.cm.model.AbstractServiceModelGenerator;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Locale;
+
+public class ClouderaManagerAPIServiceModelGenerator
+    extends AbstractServiceModelGenerator {
+
+  public static final String SERVICE = "CM-API";
+  public static final String SERVICE_TYPE = ClouderaManagerServiceDiscovery.CM_SERVICE_TYPE;
+  public static final String ROLE_TYPE = ClouderaManagerServiceDiscovery.CM_ROLE_TYPE;
+
+  @Override
+  public String getService() {
+    return SERVICE;
+  }
+
+  @Override
+  public String getServiceType() {
+    return SERVICE_TYPE;
+  }
+
+  @Override
+  public String getRoleType() {
+    return ROLE_TYPE;
+  }
+
+  @Override
+  public ServiceModel.Type getModelType() {
+    return ServiceModel.Type.API;
+  }
+
+  /**
+   * This method functions differently than others. This method inquires the
+   * discovery client and uses the CM url used by the driver (which was
+   * populated by the descriptor).
+   *
+   * @param service
+   * @param serviceConfig
+   * @param role
+   * @param roleConfig
+   * @return
+   * @throws ApiException
+   */
+  @Override
+  public ServiceModel generateService(ApiService service,
+      ApiServiceConfig serviceConfig, ApiRole role, ApiConfigList roleConfig)
+      throws ApiException {
+
+    final String basePath = getClient().getBasePath();
+    URI uri;
+    try {
+      uri = new URI(basePath);
+    } catch (URISyntaxException e) {
+      throw new ApiException(e);
+    }
+
+    final String serviceURL = getModelType() == ServiceModel.Type.API ?
+        String.format(Locale.getDefault(), "%s://%s:%s/api", uri.getScheme(),
+            uri.getHost(), uri.getPort()) :
+        String.format(Locale.getDefault(), "%s://%s:%s", uri.getScheme(),
+            uri.getHost(), uri.getPort());
+
+    return new ServiceModel(getModelType(), getService(), getServiceType(),
+        getRoleType(), serviceURL);
+  }
+}
diff --git a/gateway-discovery-cm/src/main/java/org/apache/knox/gateway/topology/discovery/cm/model/cm/ClouderaManagerUIServiceModelGenerator.java
b/gateway-discovery-cm/src/main/java/org/apache/knox/gateway/topology/discovery/cm/model/cm/ClouderaManagerUIServiceModelGenerator.java
new file mode 100644
index 0000000..7ba2b26
--- /dev/null
+++ b/gateway-discovery-cm/src/main/java/org/apache/knox/gateway/topology/discovery/cm/model/cm/ClouderaManagerUIServiceModelGenerator.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.knox.gateway.topology.discovery.cm.model.cm;
+
+import org.apache.knox.gateway.topology.discovery.cm.ServiceModel;
+
+public class ClouderaManagerUIServiceModelGenerator
+    extends ClouderaManagerAPIServiceModelGenerator {
+
+  public static final String SERVICE = "CM-UI";
+
+  @Override
+  public String getService() {
+    return SERVICE;
+  }
+
+  @Override
+  public ServiceModel.Type getModelType() {
+    return ServiceModel.Type.UI;
+  }
+
+}
diff --git a/gateway-discovery-cm/src/main/resources/META-INF/services/org.apache.knox.gateway.topology.discovery.cm.ServiceModelGenerator
b/gateway-discovery-cm/src/main/resources/META-INF/services/org.apache.knox.gateway.topology.discovery.cm.ServiceModelGenerator
index e6350ec..328160b 100644
--- a/gateway-discovery-cm/src/main/resources/META-INF/services/org.apache.knox.gateway.topology.discovery.cm.ServiceModelGenerator
+++ b/gateway-discovery-cm/src/main/resources/META-INF/services/org.apache.knox.gateway.topology.discovery.cm.ServiceModelGenerator
@@ -49,4 +49,6 @@ org.apache.knox.gateway.topology.discovery.cm.model.zeppelin.ZeppelinUIServiceMo
 org.apache.knox.gateway.topology.discovery.cm.model.zeppelin.ZeppelinWSServiceModelGenerator
 org.apache.knox.gateway.topology.discovery.cm.model.nifi.NifiServiceModelGenerator
 org.apache.knox.gateway.topology.discovery.cm.model.nifi.NifiRegistryServiceModelGenerator
+org.apache.knox.gateway.topology.discovery.cm.model.cm.ClouderaManagerAPIServiceModelGenerator
+org.apache.knox.gateway.topology.discovery.cm.model.cm.ClouderaManagerUIServiceModelGenerator
 
diff --git a/gateway-discovery-cm/src/test/java/org/apache/knox/gateway/topology/discovery/cm/ClouderaManagerServiceDiscoveryTest.java
b/gateway-discovery-cm/src/test/java/org/apache/knox/gateway/topology/discovery/cm/ClouderaManagerServiceDiscoveryTest.java
index 853197d..d12dbc4 100644
--- a/gateway-discovery-cm/src/test/java/org/apache/knox/gateway/topology/discovery/cm/ClouderaManagerServiceDiscoveryTest.java
+++ b/gateway-discovery-cm/src/test/java/org/apache/knox/gateway/topology/discovery/cm/ClouderaManagerServiceDiscoveryTest.java
@@ -34,6 +34,7 @@ import org.apache.knox.gateway.topology.discovery.ServiceDiscovery;
 import org.apache.knox.gateway.topology.discovery.ServiceDiscoveryConfig;
 import org.apache.knox.gateway.topology.discovery.cm.model.atlas.AtlasAPIServiceModelGenerator;
 import org.apache.knox.gateway.topology.discovery.cm.model.atlas.AtlasServiceModelGenerator;
+import org.apache.knox.gateway.topology.discovery.cm.model.cm.ClouderaManagerUIServiceModelGenerator;
 import org.apache.knox.gateway.topology.discovery.cm.model.hbase.HBaseUIServiceModelGenerator;
 import org.apache.knox.gateway.topology.discovery.cm.model.hbase.WebHBaseServiceModelGenerator;
 import org.apache.knox.gateway.topology.discovery.cm.model.hdfs.NameNodeServiceModelGenerator;
@@ -69,6 +70,8 @@ import static org.junit.Assert.assertNotNull;
 
 public class ClouderaManagerServiceDiscoveryTest {
 
+  private static final String DISCOVERY_URL = "http://localhost:1234";
+
   @Test
   public void testJobTrackerServiceDiscovery() {
     final String hostName = "resourcemanager-host-1";
@@ -562,6 +565,33 @@ public class ClouderaManagerServiceDiscoveryTest {
     doTestNiFiRegistryDiscovery(true);
   }
 
+  @Test
+  public void testCMDiscoveryUI() {
+    doTestCMDiscovery("CM-UI");
+  }
+
+  @Test
+  public void testCMDiscoveryAPI() {
+    doTestCMDiscovery("CM-API");
+  }
+
+  private void doTestCMDiscovery(final String serviceName) {
+    ServiceDiscovery.Cluster cluster = doTestDiscovery("somehost",
+        serviceName+"-1", ClouderaManagerUIServiceModelGenerator.SERVICE_TYPE,
+        serviceName+"-1",
+        ClouderaManagerUIServiceModelGenerator.ROLE_TYPE,
+        Collections.emptyMap(),
+        Collections.emptyMap());
+
+    List<String> urls = cluster.getServiceURLs(serviceName);
+    assertEquals(1, urls.size());
+    if("CM-UI".equals(serviceName)) {
+      assertEquals(DISCOVERY_URL, urls.get(0));
+    } else {
+      assertEquals(DISCOVERY_URL+"/api", urls.get(0));
+    }
+  }
+
   private void doTestNiFiRegistryDiscovery(boolean sslEnabled) {
     final String hostName = "nifi-registry-host";
     final String port = "18080";
@@ -1071,7 +1101,7 @@ public class ClouderaManagerServiceDiscoveryTest {
 
 
   private static ServiceDiscoveryConfig createMockDiscoveryConfig() {
-    return createMockDiscoveryConfig("http://localhost:1234", "itsme");
+    return createMockDiscoveryConfig(DISCOVERY_URL, "itsme");
   }
 
   private static ServiceDiscoveryConfig createMockDiscoveryConfig(String address, String
username) {


Mime
View raw message