knox-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From pzamp...@apache.org
Subject knox git commit: KNOX-1293 - Support for Ambari discovery of SPARK2 services
Date Mon, 07 May 2018 13:50:51 GMT
Repository: knox
Updated Branches:
  refs/heads/master 11796c96b -> 691a21212


KNOX-1293 - Support for Ambari discovery of SPARK2 services


Project: http://git-wip-us.apache.org/repos/asf/knox/repo
Commit: http://git-wip-us.apache.org/repos/asf/knox/commit/691a2121
Tree: http://git-wip-us.apache.org/repos/asf/knox/tree/691a2121
Diff: http://git-wip-us.apache.org/repos/asf/knox/diff/691a2121

Branch: refs/heads/master
Commit: 691a212129fbf50a091d2b5c05e070d88929eaf8
Parents: 11796c9
Author: Phil Zampino <pzampino@apache.org>
Authored: Tue May 1 22:57:22 2018 -0400
Committer: Phil Zampino <pzampino@apache.org>
Committed: Mon May 7 09:35:16 2018 -0400

----------------------------------------------------------------------
 .../discovery/ambari/LivyServiceURLCreator.java |  36 +++++
 .../ambari/SparkCommonServiceURLCreator.java    |  64 ++++++++
 .../ambari/SparkHistoryUIServiceURLCreator.java |  36 +++++
 ....topology.discovery.ambari.ServiceURLCreator |   4 +-
 ...iscovery-component-config-mapping.properties |   2 +
 .../ambari-service-discovery-url-mappings.xml   |  28 ----
 .../ambari/SparkServiceURLCreatorTest.java      | 160 +++++++++++++++++++
 7 files changed, 301 insertions(+), 29 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/knox/blob/691a2121/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/LivyServiceURLCreator.java
----------------------------------------------------------------------
diff --git a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/LivyServiceURLCreator.java
b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/LivyServiceURLCreator.java
new file mode 100644
index 0000000..6c12b18
--- /dev/null
+++ b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/LivyServiceURLCreator.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.knox.gateway.topology.discovery.ambari;
+
+public class LivyServiceURLCreator extends SparkCommonServiceURLCreator {
+
+  private static final String RESOURCE_ROLE = "LIVYSERVER";
+
+  @Override
+  public void init(AmbariCluster cluster) {
+    super.init(cluster);
+    primaryComponentName   = "LIVY_SERVER";
+    secondaryComponentName = "LIVY2_SERVER";
+    portConfigProperty     = "livy.server.port";
+  }
+
+  @Override
+  public String getTargetService() {
+    return RESOURCE_ROLE;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/knox/blob/691a2121/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/SparkCommonServiceURLCreator.java
----------------------------------------------------------------------
diff --git a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/SparkCommonServiceURLCreator.java
b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/SparkCommonServiceURLCreator.java
new file mode 100644
index 0000000..ae223d3
--- /dev/null
+++ b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/SparkCommonServiceURLCreator.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.knox.gateway.topology.discovery.ambari;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public abstract class SparkCommonServiceURLCreator implements ServiceURLCreator {
+
+  private static final String URL_TEMPLATE = "http://%s:%s";
+
+  protected AmbariCluster cluster = null;
+
+  String primaryComponentName = null;
+
+  String secondaryComponentName = null;
+
+  String portConfigProperty = null;
+
+  @Override
+  public void init(AmbariCluster cluster) {
+    this.cluster = cluster;
+  }
+
+  @Override
+  public List<String> create(String service, Map<String, String> serviceParams)
{
+    List<String> urls = new ArrayList<>();
+
+    if (getTargetService().equalsIgnoreCase(service)) {
+      AmbariComponent comp = cluster.getComponent(primaryComponentName);
+      if (comp == null) {
+        comp = cluster.getComponent(secondaryComponentName);
+      }
+
+      if (comp != null) {
+        String port = comp.getConfigProperty(portConfigProperty);
+        List<String> hostNames = comp.getHostNames();
+        for (String host : hostNames) {
+          urls.add(String.format(URL_TEMPLATE, host, port));
+        }
+      }
+    }
+
+    return urls;
+  }
+
+
+
+}

http://git-wip-us.apache.org/repos/asf/knox/blob/691a2121/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/SparkHistoryUIServiceURLCreator.java
----------------------------------------------------------------------
diff --git a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/SparkHistoryUIServiceURLCreator.java
b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/SparkHistoryUIServiceURLCreator.java
new file mode 100644
index 0000000..a0de1d5
--- /dev/null
+++ b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/SparkHistoryUIServiceURLCreator.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.knox.gateway.topology.discovery.ambari;
+
+public class SparkHistoryUIServiceURLCreator extends SparkCommonServiceURLCreator {
+
+  private static final String RESOURCE_ROLE = "SPARKHISTORYUI";
+
+  @Override
+  public void init(AmbariCluster cluster) {
+    super.init(cluster);
+    primaryComponentName   = "SPARK_JOBHISTORYSERVER";
+    secondaryComponentName = "SPARK2_JOBHISTORYSERVER";
+    portConfigProperty     = "spark.history.ui.port";
+  }
+
+  @Override
+  public String getTargetService() {
+    return RESOURCE_ROLE;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/knox/blob/691a2121/gateway-discovery-ambari/src/main/resources/META-INF/services/org.apache.knox.gateway.topology.discovery.ambari.ServiceURLCreator
----------------------------------------------------------------------
diff --git a/gateway-discovery-ambari/src/main/resources/META-INF/services/org.apache.knox.gateway.topology.discovery.ambari.ServiceURLCreator
b/gateway-discovery-ambari/src/main/resources/META-INF/services/org.apache.knox.gateway.topology.discovery.ambari.ServiceURLCreator
index a6e2a7e..89edbfc 100644
--- a/gateway-discovery-ambari/src/main/resources/META-INF/services/org.apache.knox.gateway.topology.discovery.ambari.ServiceURLCreator
+++ b/gateway-discovery-ambari/src/main/resources/META-INF/services/org.apache.knox.gateway.topology.discovery.ambari.ServiceURLCreator
@@ -21,4 +21,6 @@ org.apache.knox.gateway.topology.discovery.ambari.WebHdfsUrlCreator
 org.apache.knox.gateway.topology.discovery.ambari.HdfsUIUrlCreator
 org.apache.knox.gateway.topology.discovery.ambari.ResourceManagerURLCreator
 org.apache.knox.gateway.topology.discovery.ambari.YarnUIURLCreator
-org.apache.knox.gateway.topology.discovery.ambari.YarnUIV2URLCreator
\ No newline at end of file
+org.apache.knox.gateway.topology.discovery.ambari.YarnUIV2URLCreator
+org.apache.knox.gateway.topology.discovery.ambari.SparkHistoryUIServiceURLCreator
+org.apache.knox.gateway.topology.discovery.ambari.LivyServiceURLCreator
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/knox/blob/691a2121/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-component-config-mapping.properties
----------------------------------------------------------------------
diff --git a/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-component-config-mapping.properties
b/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-component-config-mapping.properties
index bf53db0..884e143 100644
--- a/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-component-config-mapping.properties
+++ b/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-component-config-mapping.properties
@@ -32,11 +32,13 @@ HISTORYSERVER=mapred-site
 HIVE_SERVER=hive-site
 INFRA_SOLR=infra-solr-env
 LIVY_SERVER=livy-conf
+LIVY2_SERVER=livy2-conf
 NAMENODE=hdfs-site
 OOZIE_SERVER=oozie-site
 RANGER_ADMIN=ranger-admin-site
 RESOURCEMANAGER=yarn-site
 SPARK_JOBHISTORYSERVER=spark-defaults
+SPARK2_JOBHISTORYSERVER=spark2-defaults
 STORM_UI_SERVER=storm-site
 WEBHCAT_SERVER=webhcat-site
 ZEPPELIN_MASTER=zeppelin-config

http://git-wip-us.apache.org/repos/asf/knox/blob/691a2121/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-url-mappings.xml
----------------------------------------------------------------------
diff --git a/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-url-mappings.xml
b/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-url-mappings.xml
index a1f1c88..a3e1def 100644
--- a/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-url-mappings.xml
+++ b/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-url-mappings.xml
@@ -485,34 +485,6 @@
         </properties>
     </service>
 
-    <service name="SPARKHISTORYUI">
-        <url-pattern>http://{HOST}:{PORT}</url-pattern>
-        <properties>
-            <property name="HOST">
-                <component>SPARK_JOBHISTORYSERVER</component>
-                <hostname/>
-            </property>
-            <property name="PORT">
-                <component>SPARK_JOBHISTORYSERVER</component>
-                <config-property>spark.history.ui.port</config-property>
-            </property>
-        </properties>
-    </service>
-
-    <service name="LIVYSERVER">
-        <url-pattern>http://{HOST}:{PORT}</url-pattern>
-        <properties>
-            <property name="HOST">
-                <component>LIVY_SERVER</component>
-                <hostname/>
-            </property>
-            <property name="PORT">
-                <component>LIVY_SERVER</component>
-                <config-property>livy.server.port</config-property>
-            </property>
-        </properties>
-    </service>
-
     <service name="SOLR">
         <url-pattern>http://{HOST}:{PORT}</url-pattern>
         <properties>

http://git-wip-us.apache.org/repos/asf/knox/blob/691a2121/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/SparkServiceURLCreatorTest.java
----------------------------------------------------------------------
diff --git a/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/SparkServiceURLCreatorTest.java
b/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/SparkServiceURLCreatorTest.java
new file mode 100644
index 0000000..e12b653
--- /dev/null
+++ b/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/SparkServiceURLCreatorTest.java
@@ -0,0 +1,160 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.knox.gateway.topology.discovery.ambari;
+
+import org.easymock.EasyMock;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+
+public class SparkServiceURLCreatorTest {
+
+  @Test
+  public void testSparkHistoryUI() {
+    doTestSparkHistoryUI("SPARK_JOBHISTORYSERVER");
+  }
+
+  @Test
+  public void testSpark2HistoryUI() {
+    doTestSparkHistoryUI("SPARK2_JOBHISTORYSERVER");
+  }
+
+  private void doTestSparkHistoryUI(String componentName) {
+    final String PORT = "4545";
+
+    AmbariComponent ac = EasyMock.createNiceMock(AmbariComponent.class);
+    List<String> hostNames = Arrays.asList("host1", "host2");
+    EasyMock.expect(ac.getHostNames()).andReturn(hostNames).anyTimes();
+    EasyMock.expect(ac.getConfigProperty("spark.history.ui.port")).andReturn(PORT).anyTimes();
+    EasyMock.replay(ac);
+
+    AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class);
+    EasyMock.expect(cluster.getComponent(componentName)).andReturn(ac).anyTimes();
+    EasyMock.replay(cluster);
+
+    SparkHistoryUIServiceURLCreator c = new SparkHistoryUIServiceURLCreator();
+    c.init(cluster);
+    List<String> urls = c.create("SPARKHISTORYUI", null);
+    assertNotNull(urls);
+    assertFalse(urls.isEmpty());
+    assertEquals(2, urls.size());
+    assertEquals("http://host1:" + PORT, urls.get(0));
+    assertEquals("http://host2:" + PORT, urls.get(1));
+  }
+
+
+  @Test
+  public void testSparkAndSpark2HistoryUI() {
+    final String PORT  = "4545";
+    final String PORT2 = "6767";
+
+    AmbariComponent ac = EasyMock.createNiceMock(AmbariComponent.class);
+    EasyMock.expect(ac.getHostNames()).andReturn(Arrays.asList("host1", "host2")).anyTimes();
+    EasyMock.expect(ac.getConfigProperty("spark.history.ui.port")).andReturn(PORT).anyTimes();
+    EasyMock.replay(ac);
+
+    AmbariComponent ac2 = EasyMock.createNiceMock(AmbariComponent.class);
+    EasyMock.expect(ac2.getHostNames()).andReturn(Arrays.asList("host3", "host4")).anyTimes();
+    EasyMock.expect(ac2.getConfigProperty("spark.history.ui.port")).andReturn(PORT2).anyTimes();
+    EasyMock.replay(ac2);
+
+    AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class);
+    EasyMock.expect(cluster.getComponent("SPARK_JOBHISTORYSERVER")).andReturn(ac).anyTimes();
+    EasyMock.expect(cluster.getComponent("SPARK2_JOBHISTORYSERVER")).andReturn(ac2).anyTimes();
+    EasyMock.replay(cluster);
+
+    SparkHistoryUIServiceURLCreator c = new SparkHistoryUIServiceURLCreator();
+    c.init(cluster);
+    List<String> urls = c.create("SPARKHISTORYUI", null);
+    assertNotNull(urls);
+    assertFalse(urls.isEmpty());
+    assertEquals(2, urls.size());
+    assertEquals("http://host1:" + PORT, urls.get(0));
+    assertEquals("http://host2:" + PORT, urls.get(1));
+  }
+
+
+  @Test
+  public void testLivyServer() {
+    doTestLivyServer("LIVY_SERVER");
+  }
+
+  @Test
+  public void testLivy2Server() {
+    doTestLivyServer("LIVY2_SERVER");
+  }
+
+  private void doTestLivyServer(String componentName) {
+    final String PORT = "4545";
+
+    AmbariComponent ac = EasyMock.createNiceMock(AmbariComponent.class);
+    List<String> hostNames = Arrays.asList("host1", "host2");
+    EasyMock.expect(ac.getHostNames()).andReturn(hostNames).anyTimes();
+    EasyMock.expect(ac.getConfigProperty("livy.server.port")).andReturn(PORT).anyTimes();
+    EasyMock.replay(ac);
+
+    AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class);
+    EasyMock.expect(cluster.getComponent(componentName)).andReturn(ac).anyTimes();
+    EasyMock.replay(cluster);
+
+    LivyServiceURLCreator c = new LivyServiceURLCreator();
+    c.init(cluster);
+    List<String> urls = c.create("LIVYSERVER", null);
+    assertNotNull(urls);
+    assertFalse(urls.isEmpty());
+    assertEquals(2, urls.size());
+    assertEquals("http://host1:" + PORT, urls.get(0));
+    assertEquals("http://host2:" + PORT, urls.get(1));
+  }
+
+  @Test
+  public void testLivyAndLivy2Server() {
+    final String PORT  = "4545";
+    final String PORT2 = "2323";
+
+    AmbariComponent ac = EasyMock.createNiceMock(AmbariComponent.class);
+    EasyMock.expect(ac.getHostNames()).andReturn(Arrays.asList("host1", "host2")).anyTimes();
+    EasyMock.expect(ac.getConfigProperty("livy.server.port")).andReturn(PORT).anyTimes();
+    EasyMock.replay(ac);
+
+    AmbariComponent ac2 = EasyMock.createNiceMock(AmbariComponent.class);
+    EasyMock.expect(ac2.getHostNames()).andReturn(Arrays.asList("host3", "host4")).anyTimes();
+    EasyMock.expect(ac2.getConfigProperty("livy.server.port")).andReturn(PORT2).anyTimes();
+    EasyMock.replay(ac2);
+
+    AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class);
+    EasyMock.expect(cluster.getComponent("LIVY_SERVER")).andReturn(ac).anyTimes();
+    EasyMock.expect(cluster.getComponent("LIVY2_SERVER")).andReturn(ac2).anyTimes();
+    EasyMock.replay(cluster);
+
+    LivyServiceURLCreator c = new LivyServiceURLCreator();
+    c.init(cluster);
+    List<String> urls = c.create("LIVYSERVER", null);
+    assertNotNull(urls);
+    assertFalse(urls.isEmpty());
+    assertEquals(2, urls.size());
+    assertEquals("http://host1:" + PORT, urls.get(0));
+    assertEquals("http://host2:" + PORT, urls.get(1));
+  }
+
+
+}


Mime
View raw message