knox-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From pzamp...@apache.org
Subject knox git commit: KNOX-1193 - Discovery support for SOLR, STORM and STORM-LOGVIEWER, and config for additional support
Date Fri, 02 Mar 2018 21:13:21 GMT
Repository: knox
Updated Branches:
  refs/heads/master 1a50523f3 -> 21ac567db


KNOX-1193 - Discovery support for SOLR, STORM and STORM-LOGVIEWER, and config for additional
support


Project: http://git-wip-us.apache.org/repos/asf/knox/repo
Commit: http://git-wip-us.apache.org/repos/asf/knox/commit/21ac567d
Tree: http://git-wip-us.apache.org/repos/asf/knox/tree/21ac567d
Diff: http://git-wip-us.apache.org/repos/asf/knox/diff/21ac567d

Branch: refs/heads/master
Commit: 21ac567db07c3f2196dfb41cb7bf9bbb786801a2
Parents: 1a50523
Author: Phil Zampino <pzampino@apache.org>
Authored: Fri Mar 2 15:52:06 2018 -0500
Committer: Phil Zampino <pzampino@apache.org>
Committed: Fri Mar 2 15:59:33 2018 -0500

----------------------------------------------------------------------
 .../ambari/AmbariDynamicServiceURLCreator.java  | 21 +++++++++-
 .../ambari/AmbariServiceDiscovery.java          | 21 +++++++++-
 ...iscovery-component-config-mapping.properties | 26 ++++++------
 .../ambari-service-discovery-url-mappings.xml   | 42 ++++++++++++++++++++
 .../topology/impl/DefaultTopologyService.java   |  6 +++
 .../topology/discovery/ServiceDiscovery.java    |  2 +
 6 files changed, 104 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/knox/blob/21ac567d/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariDynamicServiceURLCreator.java
----------------------------------------------------------------------
diff --git a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariDynamicServiceURLCreator.java
b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariDynamicServiceURLCreator.java
index 96d462b..7ccd170 100644
--- a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariDynamicServiceURLCreator.java
+++ b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariDynamicServiceURLCreator.java
@@ -17,6 +17,7 @@
 package org.apache.knox.gateway.topology.discovery.ambari;
 
 import org.apache.knox.gateway.i18n.messages.MessagesFactory;
+import org.apache.knox.gateway.topology.discovery.ServiceDiscovery;
 
 import java.io.ByteArrayInputStream;
 import java.io.File;
@@ -30,6 +31,8 @@ import java.util.Map;
 
 class AmbariDynamicServiceURLCreator implements ServiceURLCreator {
 
+    static final String MAPPING_CONFIG_OVERRIDE_FILE = "ambari-discovery-url-mappings.xml";
+
     static final String MAPPING_CONFIG_OVERRIDE_PROPERTY = "org.apache.gateway.topology.discovery.ambari.config";
 
     private AmbariServiceDiscoveryMessages log = MessagesFactory.get(AmbariServiceDiscoveryMessages.class);
@@ -44,7 +47,23 @@ class AmbariDynamicServiceURLCreator implements ServiceURLCreator {
         config = new ServiceURLPropertyConfig();
 
         // Attempt to apply overriding or additional mappings from external source
-        String mappingConfiguration = System.getProperty(MAPPING_CONFIG_OVERRIDE_PROPERTY);
+        String mappingConfiguration = null;
+
+        // First, check for the well-known override mapping file
+        String gatewayConfDir = System.getProperty(ServiceDiscovery.CONFIG_DIR_PROPERTY);
+        if (gatewayConfDir != null) {
+            File overridesFile = new File(gatewayConfDir, MAPPING_CONFIG_OVERRIDE_FILE);
+            if (overridesFile.exists()) {
+                mappingConfiguration = overridesFile.getAbsolutePath();
+            }
+        }
+
+        // If no file in the config dir, check for the system property reference
+        if (mappingConfiguration == null) {
+            mappingConfiguration = System.getProperty(MAPPING_CONFIG_OVERRIDE_PROPERTY);
+        }
+
+        // If there is an overrides configuration file specified either way, then apply it
         if (mappingConfiguration != null) {
             File mappingConfigFile = new File(mappingConfiguration);
             if (mappingConfigFile.exists()) {

http://git-wip-us.apache.org/repos/asf/knox/blob/21ac567d/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariServiceDiscovery.java
----------------------------------------------------------------------
diff --git a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariServiceDiscovery.java
b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariServiceDiscovery.java
index 80009cc..957d723 100644
--- a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariServiceDiscovery.java
+++ b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariServiceDiscovery.java
@@ -16,6 +16,7 @@
  */
 package org.apache.knox.gateway.topology.discovery.ambari;
 
+import java.io.File;
 import java.io.FileInputStream;
 import java.io.InputStream;
 import java.lang.reflect.Method;
@@ -53,6 +54,8 @@ class AmbariServiceDiscovery implements ServiceDiscovery {
     private static final String COMPONENT_CONFIG_MAPPING_FILE =
                                                         "ambari-service-discovery-component-config-mapping.properties";
 
+    private static final String COMPONENT_CONFIG_OVERRIDES_FILENAME = "ambari-discovery-component-config.properties";
+
     private static final String GATEWAY_SERVICES_ACCESSOR_CLASS  = "org.apache.knox.gateway.GatewayServer";
     private static final String GATEWAY_SERVICES_ACCESSOR_METHOD = "getGatewayServices";
 
@@ -87,7 +90,23 @@ class AmbariServiceDiscovery implements ServiceDiscovery {
             }
 
             // Attempt to apply overriding or additional mappings from external source
-            String overridesPath = System.getProperty(COMPONENT_CONFIG_MAPPING_SYSTEM_PROPERTY);
+            String overridesPath = null;
+
+            // First, check for the well-known overrides config file
+            String gatewayConfDir = System.getProperty(CONFIG_DIR_PROPERTY);
+            if (gatewayConfDir != null) {
+                File overridesFile = new File(gatewayConfDir, COMPONENT_CONFIG_OVERRIDES_FILENAME);
+                if (overridesFile.exists()) {
+                    overridesPath = overridesFile.getAbsolutePath();
+                }
+            }
+
+            // If no file in the config dir, check for the system property reference
+            if (overridesPath == null) {
+                overridesPath = System.getProperty(COMPONENT_CONFIG_MAPPING_SYSTEM_PROPERTY);
+            }
+
+            // If there is an overrides configuration file specified either way, then apply
it
             if (overridesPath != null) {
                 Properties overrides = new Properties();
                 try (InputStream in = new FileInputStream(overridesPath)) {

http://git-wip-us.apache.org/repos/asf/knox/blob/21ac567d/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-component-config-mapping.properties
----------------------------------------------------------------------
diff --git a/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-component-config-mapping.properties
b/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-component-config-mapping.properties
index 48dd7b1..bf53db0 100644
--- a/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-component-config-mapping.properties
+++ b/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-component-config-mapping.properties
@@ -20,22 +20,24 @@
 # Hadoop service component configuration mappings
 # Service component name = service component configuration type
 ##########################################################################
-NAMENODE=hdfs-site
-RESOURCEMANAGER=yarn-site
-HISTORYSERVER=mapred-site
-OOZIE_SERVER=oozie-site
-HIVE_SERVER=hive-site
-WEBHCAT_SERVER=webhcat-site
-HBASE_MASTER=hbase-site
-DRUID_COORDINATOR=druid-coordinator
+ATLAS_SERVER=application-properties
 DRUID_BROKER=druid-broker
-DRUID_ROUTER=druid-router
+DRUID_COORDINATOR=druid-coordinator
 DRUID_OVERLORD=druid-overlord
+DRUID_ROUTER=druid-router
 DRUID_SUPERSET=druid-superset
-ATLAS_SERVER=application-properties
-ZEPPELIN_MASTER=zeppelin-config
-RANGER_ADMIN=ranger-admin-site
 FALCON_SERVER=falcon-env
+HBASE_MASTER=hbase-site
+HISTORYSERVER=mapred-site
+HIVE_SERVER=hive-site
+INFRA_SOLR=infra-solr-env
 LIVY_SERVER=livy-conf
+NAMENODE=hdfs-site
+OOZIE_SERVER=oozie-site
+RANGER_ADMIN=ranger-admin-site
+RESOURCEMANAGER=yarn-site
 SPARK_JOBHISTORYSERVER=spark-defaults
+STORM_UI_SERVER=storm-site
+WEBHCAT_SERVER=webhcat-site
+ZEPPELIN_MASTER=zeppelin-config
 

http://git-wip-us.apache.org/repos/asf/knox/blob/21ac567d/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-url-mappings.xml
----------------------------------------------------------------------
diff --git a/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-url-mappings.xml
b/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-url-mappings.xml
index 6b6907d..3f37c0a 100644
--- a/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-url-mappings.xml
+++ b/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-url-mappings.xml
@@ -568,4 +568,46 @@
         </properties>
     </service>
 
+    <service name="SOLR">
+        <url-pattern>http://{HOST}:{PORT}</url-pattern>
+        <properties>
+            <property name="HOST">
+                <component>INFRA_SOLR</component>
+                <hostname/>
+            </property>
+            <property name="PORT">
+                <component>INFRA_SOLR</component>
+                <config-property>infra_solr_port</config-property>
+            </property>
+        </properties>
+    </service>
+
+    <service name="STORM">
+        <url-pattern>http://{HOST}:{PORT}</url-pattern>
+        <properties>
+            <property name="HOST">
+                <component>STORM_UI_SERVER</component>
+                <hostname/>
+            </property>
+            <property name="PORT">
+                <component>STORM_UI_SERVER</component>
+                <config-property>ui.port</config-property>
+            </property>
+        </properties>
+    </service>
+
+    <service name="STORM-LOGVIEWER">
+        <url-pattern>http://{HOST}:{PORT}</url-pattern>
+        <properties>
+            <property name="HOST">
+                <component>STORM_UI_SERVER</component>
+                <hostname/>
+            </property>
+            <property name="PORT">
+                <component>STORM_UI_SERVER</component>
+                <config-property>logviewer.port</config-property>
+            </property>
+        </properties>
+    </service>
+
 </service-discovery-url-mappings>

http://git-wip-us.apache.org/repos/asf/knox/blob/21ac567d/gateway-server/src/main/java/org/apache/knox/gateway/services/topology/impl/DefaultTopologyService.java
----------------------------------------------------------------------
diff --git a/gateway-server/src/main/java/org/apache/knox/gateway/services/topology/impl/DefaultTopologyService.java
b/gateway-server/src/main/java/org/apache/knox/gateway/services/topology/impl/DefaultTopologyService.java
index 52a7dfa..ccd4130 100644
--- a/gateway-server/src/main/java/org/apache/knox/gateway/services/topology/impl/DefaultTopologyService.java
+++ b/gateway-server/src/main/java/org/apache/knox/gateway/services/topology/impl/DefaultTopologyService.java
@@ -51,6 +51,7 @@ import org.apache.knox.gateway.topology.TopologyMonitor;
 import org.apache.knox.gateway.topology.TopologyProvider;
 import org.apache.knox.gateway.topology.builder.TopologyBuilder;
 import org.apache.knox.gateway.topology.discovery.ClusterConfigurationMonitor;
+import org.apache.knox.gateway.topology.discovery.ServiceDiscovery;
 import org.apache.knox.gateway.topology.monitor.RemoteConfigurationMonitor;
 import org.apache.knox.gateway.topology.monitor.RemoteConfigurationMonitorFactory;
 import org.apache.knox.gateway.topology.simple.ProviderConfigurationParser;
@@ -622,6 +623,11 @@ public class DefaultTopologyService
   @Override
   public void init(GatewayConfig config, Map<String, String> options) throws ServiceLifecycleException
{
 
+    String gatewayConfDir = config.getGatewayConfDir();
+    if (gatewayConfDir != null) {
+      System.setProperty(ServiceDiscovery.CONFIG_DIR_PROPERTY, gatewayConfDir);
+    }
+
     try {
       listeners  = new HashSet<>();
       topologies = new HashMap<>();

http://git-wip-us.apache.org/repos/asf/knox/blob/21ac567d/gateway-spi/src/main/java/org/apache/knox/gateway/topology/discovery/ServiceDiscovery.java
----------------------------------------------------------------------
diff --git a/gateway-spi/src/main/java/org/apache/knox/gateway/topology/discovery/ServiceDiscovery.java
b/gateway-spi/src/main/java/org/apache/knox/gateway/topology/discovery/ServiceDiscovery.java
index 01b2178..14d6c75 100644
--- a/gateway-spi/src/main/java/org/apache/knox/gateway/topology/discovery/ServiceDiscovery.java
+++ b/gateway-spi/src/main/java/org/apache/knox/gateway/topology/discovery/ServiceDiscovery.java
@@ -26,6 +26,8 @@ import java.util.Map;
  */
 public interface ServiceDiscovery {
 
+    String CONFIG_DIR_PROPERTY = "org.apache.knox.gateway.conf.dir";
+
     /**
      * This is the type specified in a simple descriptor to indicate which ServiceDiscovery
implementation to employ.
      *


Mime
View raw message