knox-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From pzamp...@apache.org
Subject knox git commit: KNOX-1228 - Atlas HA ZooKeeper Config Discovery
Date Wed, 28 Mar 2018 13:45:47 GMT
Repository: knox
Updated Branches:
  refs/heads/master a6d2f5240 -> 98dbfe990


KNOX-1228 - Atlas HA ZooKeeper Config Discovery


Project: http://git-wip-us.apache.org/repos/asf/knox/repo
Commit: http://git-wip-us.apache.org/repos/asf/knox/commit/98dbfe99
Tree: http://git-wip-us.apache.org/repos/asf/knox/tree/98dbfe99
Diff: http://git-wip-us.apache.org/repos/asf/knox/diff/98dbfe99

Branch: refs/heads/master
Commit: 98dbfe990887c2ce032015781cc99d4ae8198cbe
Parents: a6d2f52
Author: Phil Zampino <pzampino@apache.org>
Authored: Thu Mar 22 13:22:13 2018 -0400
Committer: Phil Zampino <pzampino@apache.org>
Committed: Wed Mar 28 09:29:04 2018 -0400

----------------------------------------------------------------------
 .../discovery/ambari/AmbariCluster.java         | 38 +++++++++++-
 ...rvice-discovery-zk-config-mapping.properties | 14 ++++-
 .../discovery/ambari/AmbariClusterTest.java     | 61 ++++++++++++++++++--
 3 files changed, 106 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/knox/blob/98dbfe99/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariCluster.java
----------------------------------------------------------------------
diff --git a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariCluster.java
b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariCluster.java
index b5e4cff..9d2c92e 100644
--- a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariCluster.java
+++ b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariCluster.java
@@ -19,6 +19,7 @@ package org.apache.knox.gateway.topology.discovery.ambari;
 import org.apache.knox.gateway.i18n.messages.MessagesFactory;
 import org.apache.knox.gateway.topology.discovery.ServiceDiscovery;
 
+import java.io.File;
 import java.io.FileInputStream;
 import java.io.InputStream;
 import java.util.ArrayList;
@@ -33,6 +34,8 @@ class AmbariCluster implements ServiceDiscovery.Cluster {
 
     private static final String ZK_CONFIG_MAPPING_FILE = "ambari-service-discovery-zk-config-mapping.properties";
 
+    private static final String ZK_CONFIG_MAPPING_OVERRIDE_FILE = "ambari-discovery-zk-config.properties";
+
     static final String ZK_CONFIG_MAPPING_SYSTEM_PROPERTY =
                                                      "org.apache.knox.gateway.topology.discovery.ambari.zk.mapping";
 
@@ -48,7 +51,22 @@ class AmbariCluster implements ServiceDiscovery.Cluster {
             }
 
             // Attempt to apply overriding or additional mappings from external source
-            String overridesPath = System.getProperty(ZK_CONFIG_MAPPING_SYSTEM_PROPERTY);
+            String overridesPath = null;
+
+            // First, check for the well-known overrides config file
+            String gatewayConfDir = System.getProperty(ServiceDiscovery.CONFIG_DIR_PROPERTY);
+            if (gatewayConfDir != null) {
+                File overridesFile = new File(gatewayConfDir, ZK_CONFIG_MAPPING_OVERRIDE_FILE);
+                if (overridesFile.exists()) {
+                    overridesPath = overridesFile.getAbsolutePath();
+                }
+            }
+
+            // If no file in the config dir, check for the system property reference
+            if (overridesPath == null) {
+                overridesPath = System.getProperty(ZK_CONFIG_MAPPING_SYSTEM_PROPERTY);
+            }
+
             if (overridesPath != null) {
                 Properties overrides = new Properties();
                 try (InputStream in = new FileInputStream(overridesPath)) {
@@ -150,6 +168,24 @@ class AmbariCluster implements ServiceDiscovery.Cluster {
                     String namespaceProp = zooKeeperHAConfigMappings.getProperty(serviceName
+ ".namespace");
                     Map<String, String> scProps = sc.getProperties();
                     if (scProps != null) {
+
+                        if (ensembleProp != null) {
+                            // If there are multiple ensemble properties specified, then
iteratively check for the first
+                            // valid value, and use that one.
+                            String[] ensembleProps = ensembleProp.split(",");
+                            if (ensembleProps.length > 1) {
+                                for (String prop : ensembleProps) {
+                                    if (!prop.isEmpty()) {
+                                        String value = scProps.get(prop);
+                                        if (value != null) {
+                                            ensembleProp = prop;
+                                            break;
+                                        }
+                                    }
+                                }
+                            }
+                        }
+
                         result =
                             new ZooKeeperConfiguration(enabledProp != null ? scProps.get(enabledProp)
: null,
                                                        ensembleProp != null ? scProps.get(ensembleProp)
: null,

http://git-wip-us.apache.org/repos/asf/knox/blob/98dbfe99/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-zk-config-mapping.properties
----------------------------------------------------------------------
diff --git a/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-zk-config-mapping.properties
b/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-zk-config-mapping.properties
index 0a5faae..752382e 100644
--- a/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-zk-config-mapping.properties
+++ b/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-zk-config-mapping.properties
@@ -48,4 +48,16 @@ YARN.ensemble=yarn.resourcemanager.zk-address
 
 # WEBHCAT
 WEBHCAT.config=WEBHCAT:webhcat-site
-WEBHCAT.ensemble=templeton.zookeeper.hosts
\ No newline at end of file
+WEBHCAT.ensemble=templeton.zookeeper.hosts
+
+# ATLAS
+ATLAS.config=ATLAS:application-properties
+ATLAS.enabled=atlas.server.ha.enabled
+ATLAS.ensemble=atlas.server.ha.zookeeper.connect,atlas.kafka.zookeeper.connect
+ATLAS.namespace=atlas.server.ha.zookeeper.zkroot
+
+# ATLAS-API
+ATLAS-API.config=ATLAS:application-properties
+ATLAS-API.enabled=atlas.server.ha.enabled
+ATLAS-API.ensemble=atlas.server.ha.zookeeper.connect,atlas.kafka.zookeeper.connect
+ATLAS-API.namespace=atlas.server.ha.zookeeper.zkroot

http://git-wip-us.apache.org/repos/asf/knox/blob/98dbfe99/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariClusterTest.java
----------------------------------------------------------------------
diff --git a/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariClusterTest.java
b/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariClusterTest.java
index abec9d9..c51da6a 100644
--- a/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariClusterTest.java
+++ b/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariClusterTest.java
@@ -16,17 +16,12 @@
  */
 package org.apache.knox.gateway.topology.discovery.ambari;
 
-import org.apache.commons.io.FileUtils;
 import org.apache.knox.gateway.topology.discovery.ServiceDiscovery;
-import org.apache.knox.test.TestUtils;
 import org.easymock.EasyMock;
 import org.junit.Test;
 
-import java.io.File;
-import java.io.FileOutputStream;
 import java.util.HashMap;
 import java.util.Map;
-import java.util.Properties;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
@@ -143,6 +138,62 @@ public class AmbariClusterTest {
   }
 
 
+  /**
+   * The Atlas ZooKeeper ensemble determination is based on multiple properties with a prioritized
search order.
+   * This test verifies that the default property is used when the primary property value
is undefined.
+   */
+  @Test
+  public void testAtlasZooKeeperEnsemblePropsConfigurationSecondary() throws Exception {
+
+    final boolean isEnabled = true;
+    final String ensemble = "host1:2181,host2:2181,host3:2181";
+    //final String ensemblePropPrimary = "atlas.server.ha.zookeeper.connect";
+    final String ensemblePropSecondary = "atlas.kafka.zookeeper.connect";
+    final String namespace = "/apache_atlas";
+
+    Map<String, String> serviceConfigProps = new HashMap<>();
+    // Since the primary property is undefined, the ZooKeeper config should query the value
of the secondary property
+    serviceConfigProps.put(ensemblePropSecondary, ensemble);
+    serviceConfigProps.put("atlas.server.ha.zookeeper.zkroot", namespace);
+    serviceConfigProps.put("atlas.server.ha.enabled", "true");
+
+    AmbariCluster.ZooKeeperConfig config =
+                            getZooKeeperConfiguration("ATLAS", "ATLAS", "application-properties",
serviceConfigProps);
+    assertNotNull(config);
+    assertEquals(isEnabled, config.isEnabled());
+    assertEquals(ensemble, config.getEnsemble());
+    assertEquals(namespace, config.getNamespace());
+  }
+
+
+  /**
+   * The Atlas ZooKeeper ensemble determination is based on multiple properties with a prioritized
search order.
+   * This test verifies that the primary property value is used when it's defined.
+   */
+  @Test
+  public void testAtlasZooKeeperEnsemblePropsConfigurationPrimary() throws Exception {
+
+    final boolean isEnabled = true;
+    final String ensemble = "host1:2181,host2:2181,host3:2181";
+    final String ensemblePropPrimary = "atlas.server.ha.zookeeper.connect";
+    final String ensemblePropSecondary = "atlas.kafka.zookeeper.connect";
+    final String namespace = "/apache_atlas";
+
+    Map<String, String> serviceConfigProps = new HashMap<>();
+    serviceConfigProps.put(ensemblePropSecondary, "invalid");
+    serviceConfigProps.put(ensemblePropPrimary, ensemble); // Since the primary property
is defined, its value should be used
+    serviceConfigProps.put("atlas.server.ha.zookeeper.zkroot", namespace);
+    serviceConfigProps.put("atlas.server.ha.enabled", "true");
+
+    AmbariCluster.ZooKeeperConfig config =
+        getZooKeeperConfiguration("ATLAS", "ATLAS", "application-properties", serviceConfigProps);
+    assertNotNull(config);
+    assertEquals(isEnabled, config.isEnabled());
+    assertEquals(ensemble, config.getEnsemble());
+    assertEquals(namespace, config.getNamespace());
+  }
+
+
 
   private ServiceDiscovery.Cluster.ZooKeeperConfig getZooKeeperConfiguration(final String
             serviceName,
                                                                              final String
             configType,


Mime
View raw message