knox-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From pzamp...@apache.org
Subject knox git commit: KNOX-1269 - Generate services declared in descriptor without URLs or params, but with valid HaProvider ZK config
Date Mon, 23 Apr 2018 18:43:07 GMT
Repository: knox
Updated Branches:
  refs/heads/master e0fea744d -> 2420226cb


KNOX-1269 - Generate services declared in descriptor without URLs or params, but with valid
HaProvider ZK config


Project: http://git-wip-us.apache.org/repos/asf/knox/repo
Commit: http://git-wip-us.apache.org/repos/asf/knox/commit/2420226c
Tree: http://git-wip-us.apache.org/repos/asf/knox/tree/2420226c
Diff: http://git-wip-us.apache.org/repos/asf/knox/diff/2420226c

Branch: refs/heads/master
Commit: 2420226cbde79a8b7649e02122baadbbe4acbf06
Parents: e0fea74
Author: Phil Zampino <pzampino@apache.org>
Authored: Mon Apr 23 14:41:28 2018 -0400
Committer: Phil Zampino <pzampino@apache.org>
Committed: Mon Apr 23 14:41:28 2018 -0400

----------------------------------------------------------------------
 .../simple/SimpleDescriptorHandler.java         | 25 +++++++++++++++-
 .../simple/SimpleDescriptorHandlerTest.java     | 30 ++++++++++++++------
 2 files changed, 45 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/knox/blob/2420226c/gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorHandler.java
----------------------------------------------------------------------
diff --git a/gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorHandler.java
b/gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorHandler.java
index e83f961..28a8094 100644
--- a/gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorHandler.java
+++ b/gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorHandler.java
@@ -104,6 +104,7 @@ public class SimpleDescriptorHandler {
 
     public static Map<String, File> handle(GatewayConfig config, SimpleDescriptor desc,
File srcDirectory, File destDirectory, Service...gatewayServices) {
 
+        List<String> declaredServiceNames = new ArrayList<>();
         List<String> validServiceNames = new ArrayList<>();
         Map<String, String> serviceVersions = new HashMap<>();
         Map<String, Map<String, String>> serviceParams = new HashMap<>();
@@ -120,6 +121,7 @@ public class SimpleDescriptorHandler {
 
         for (SimpleDescriptor.Service descService : desc.getServices()) {
             String serviceName = descService.getName();
+            declaredServiceNames.add(serviceName);
 
             String serviceVer = descService.getVersion();
             if (serviceVer != null) {
@@ -173,7 +175,15 @@ public class SimpleDescriptorHandler {
         }
 
         // Generate the topology file
-        return generateTopology(desc, srcDirectory, destDirectory, cluster, validServiceNames,
serviceVersions, serviceURLs, serviceParams);
+        return generateTopology(desc,
+                                srcDirectory,
+                                destDirectory,
+                                cluster,
+                                declaredServiceNames,
+                                validServiceNames,
+                                serviceVersions,
+                                serviceURLs,
+                                serviceParams);
     }
 
 
@@ -338,6 +348,7 @@ public class SimpleDescriptorHandler {
                                                       final File srcDirectory,
                                                       final File destDirectory,
                                                       final ServiceDiscovery.Cluster cluster,
+                                                      final List<String> declaredServiceNames,
                                                       final List<String> validServiceNames,
                                                       final Map<String, String> serviceVersions,
                                                       final Map<String, List<String>>
serviceURLs,
@@ -425,6 +436,18 @@ public class SimpleDescriptorHandler {
             // Services
             // Sort the service names to write the services alphabetically
             List<String> serviceNames = new ArrayList<>(validServiceNames);
+
+            // Add any declared services, which were not validated, but which have ZK-based
HA provider config
+            for (String haServiceName : haServiceParams.keySet()) {
+                // If the service configured for HA was declared in the descriptor, then
add it to the services to be
+                // serialized (if it's not already included)
+                if (declaredServiceNames.contains(haServiceName)) {
+                    if (!serviceNames.contains(haServiceName)) {
+                        serviceNames.add(haServiceName);
+                    }
+                }
+            }
+
             Collections.sort(serviceNames);
 
             // Write the service declarations

http://git-wip-us.apache.org/repos/asf/knox/blob/2420226c/gateway-server/src/test/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorHandlerTest.java
----------------------------------------------------------------------
diff --git a/gateway-server/src/test/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorHandlerTest.java
b/gateway-server/src/test/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorHandlerTest.java
index aee2908..0063a4e 100644
--- a/gateway-server/src/test/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorHandlerTest.java
+++ b/gateway-server/src/test/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorHandlerTest.java
@@ -58,6 +58,7 @@ import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
+import static org.testng.Assert.assertNull;
 
 
 public class SimpleDescriptorHandlerTest {
@@ -123,6 +124,7 @@ public class SimpleDescriptorHandlerTest {
             "            <param><name>HIVE</name><value>enabled=auto;retrySleep=20;maxRetryAttempts=3</value></param>\n"
+
             "            <param><name>WEBHDFS</name><value>enabled=true;retrySleep=40;maxRetryAttempts=5</value></param>\n"
+
             "            <param><name>WEBHBASE</name><value>enabled=auto;retrySleep=30;maxRetryAttempts=3;maxFailoverAttempts=2;failoverSleep=10</value></param>\n"
+
+            "            <param><name>ATLAS</name><value>enabled=auto;retrySleep=30;maxRetryAttempts=3;maxFailoverAttempts=2;failoverSleep=10</value></param>\n"
+
             "        </provider>\n" +
             "    </gateway>\n";
 
@@ -535,6 +537,9 @@ public class SimpleDescriptorHandlerTest {
         final String WEBHBASE_HA_ENABLED   = "false";
         final String WEBHBASE_HA_ENSEMBLE  = "http://zkhost1:1281,http://zkhost2:1281";
 
+        final String ATLAS_HA_ENABLED   = "true";
+        final String ATLAS_HA_ENSEMBLE  = "http://zkhost5:1281,http://zkhost6:1281,http://zkhost7:1281";
+
         final Properties DISCOVERY_PROPERTIES = new Properties();
         DISCOVERY_PROPERTIES.setProperty(clusterName + ".name", clusterName);
         DISCOVERY_PROPERTIES.setProperty(clusterName + ".NAMENODE.url", "hdfs://namenodehost:8020");
@@ -547,6 +552,8 @@ public class SimpleDescriptorHandlerTest {
         DISCOVERY_PROPERTIES.setProperty(clusterName + ".HIVE.namespace", HIVE_HA_NAMESPACE);
         DISCOVERY_PROPERTIES.setProperty(clusterName + ".WEBHBASE.haEnabled", WEBHBASE_HA_ENABLED);
         DISCOVERY_PROPERTIES.setProperty(clusterName + ".WEBHBASE.ensemble", WEBHBASE_HA_ENSEMBLE);
+        DISCOVERY_PROPERTIES.setProperty(clusterName + ".ATLAS.haEnabled", ATLAS_HA_ENABLED);
+        DISCOVERY_PROPERTIES.setProperty(clusterName + ".ATLAS.ensemble", ATLAS_HA_ENSEMBLE);
 
         try {
             DISCOVERY_PROPERTIES.store(new FileOutputStream(discoveryConfig), null);
@@ -558,6 +565,8 @@ public class SimpleDescriptorHandlerTest {
         serviceURLs.put("NAMENODE", null);
         serviceURLs.put("WEBHDFS", null);
         serviceURLs.put("RESOURCEMANAGER", null);
+        serviceURLs.put("ATLAS", null);
+        serviceURLs.put("HIVE", null);
 
         // Write the externalized provider config to a temp file
         File providerConfig = new File(System.getProperty("java.io.tmpdir"), "ambari-cluster-policy.xml");
@@ -626,6 +635,7 @@ public class SimpleDescriptorHandlerTest {
             validateGeneratedProviderConfiguration(testProviderConfiguration, gatewayNode);
 
             // Validate the service declarations
+            List<String> generatedServiceDeclarations = new ArrayList<>();
             Map<String, List<String>> topologyServiceURLs = new HashMap<>();
             NodeList serviceNodes =
                 (NodeList) xpath.compile("/topology/service").evaluate(topologyXml, XPathConstants.NODESET);
@@ -636,6 +646,7 @@ public class SimpleDescriptorHandlerTest {
                 Node roleNode = (Node) xpath.compile("role/text()").evaluate(serviceNode,
XPathConstants.NODE);
                 assertNotNull(roleNode);
                 String role = roleNode.getNodeValue();
+                generatedServiceDeclarations.add(role);
 
                 // Validate the explicit version for the WEBHDFS service
                 if ("WEBHDFS".equals(role)) {
@@ -655,7 +666,7 @@ public class SimpleDescriptorHandlerTest {
                     assertEquals(2, urlNodes.getLength());
                 }
 
-                // Validate the HIV service params
+                // Validate the HIVE service params
                 if ("HIVE".equals(role)) {
                     // Expecting HA-related service params
                     NodeList paramNodes = (NodeList) xpath.compile("param").evaluate(serviceNode,
XPathConstants.NODESET);
@@ -675,23 +686,24 @@ public class SimpleDescriptorHandlerTest {
                     assertEquals(HIVE_HA_NAMESPACE, hiveServiceParams.get("zookeeperNamespace"));
                 }
 
-                // Validate the HIV service params
-                if ("WEBHBASE".equals(role)) {
+                // Validate the ATLAS service params
+                if ("ATLAS".equals(role)) {
                     // Expecting HA-related service params
                     NodeList paramNodes = (NodeList) xpath.compile("param").evaluate(serviceNode,
XPathConstants.NODESET);
                     assertNotNull(paramNodes);
-                    Map<String, String> webhbaseServiceParams = new HashMap<>();
+                    Map<String, String> atlasServiceParams = new HashMap<>();
                     for (int paramNodeIndex=0; paramNodeIndex < paramNodes.getLength();
paramNodeIndex++) {
                         Node paramNode = paramNodes.item(paramNodeIndex);
                         Node nameNode = (Node) xpath.compile("name/text()").evaluate(paramNode,
XPathConstants.NODE);
                         assertNotNull(nameNode);
                         Node valueNode = (Node) xpath.compile("value/text()").evaluate(paramNode,
XPathConstants.NODE);
                         assertNotNull(valueNode);
-                        webhbaseServiceParams.put(nameNode.getNodeValue(), valueNode.getNodeValue());
+                        atlasServiceParams.put(nameNode.getNodeValue(), valueNode.getNodeValue());
                     }
-                    assertEquals("Expected false because enabled=auto and service config
indicates HA is NOT enabled",
-                                 WEBHBASE_HA_ENABLED, webhbaseServiceParams.get("haEnabled"));
-                    assertEquals(WEBHBASE_HA_ENSEMBLE, webhbaseServiceParams.get("zookeeperEnsemble"));
+                    assertEquals("Expected true because enabled=auto and service config indicates
HA is enabled",
+                                 ATLAS_HA_ENABLED, atlasServiceParams.get("haEnabled"));
+                    assertEquals(ATLAS_HA_ENSEMBLE, atlasServiceParams.get("zookeeperEnsemble"));
+                    assertNull(atlasServiceParams.get("zookeeperNamespace"));
                 }
 
                 // Validate the URLs
@@ -711,7 +723,7 @@ public class SimpleDescriptorHandlerTest {
                     }
                 }
             }
-            assertEquals("Unexpected number of service declarations.", serviceURLs.size(),
topologyServiceURLs.size());
+            assertEquals("Unexpected number of service declarations.", serviceURLs.size(),
generatedServiceDeclarations.size());
 
         } catch (Exception e) {
             e.printStackTrace();


Mime
View raw message