knox-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From pzamp...@apache.org
Subject knox git commit: KNOX-1193 - Add service discovery support for Spark Thrift UI
Date Mon, 07 May 2018 20:01:49 GMT
Repository: knox
Updated Branches:
  refs/heads/master 691a21212 -> b4455c4c6


KNOX-1193 - Add service discovery support for Spark Thrift UI


Project: http://git-wip-us.apache.org/repos/asf/knox/repo
Commit: http://git-wip-us.apache.org/repos/asf/knox/commit/b4455c4c
Tree: http://git-wip-us.apache.org/repos/asf/knox/tree/b4455c4c
Diff: http://git-wip-us.apache.org/repos/asf/knox/diff/b4455c4c

Branch: refs/heads/master
Commit: b4455c4c63b209d8e778d9e88984adf8786cd570
Parents: 691a212
Author: Phil Zampino <pzampino@apache.org>
Authored: Mon May 7 15:45:55 2018 -0400
Committer: Phil Zampino <pzampino@apache.org>
Committed: Mon May 7 15:45:55 2018 -0400

----------------------------------------------------------------------
 .../SparkThriftServerUIServiceURLCreator.java   | 68 +++++++++++++++++++
 ....topology.discovery.ambari.ServiceURLCreator |  3 +-
 ...iscovery-component-config-mapping.properties |  2 +
 .../ambari/SparkServiceURLCreatorTest.java      | 69 ++++++++++++++++++++
 4 files changed, 141 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/knox/blob/b4455c4c/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/SparkThriftServerUIServiceURLCreator.java
----------------------------------------------------------------------
diff --git a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/SparkThriftServerUIServiceURLCreator.java
b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/SparkThriftServerUIServiceURLCreator.java
new file mode 100644
index 0000000..3db0883
--- /dev/null
+++ b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/SparkThriftServerUIServiceURLCreator.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.knox.gateway.topology.discovery.ambari;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class SparkThriftServerUIServiceURLCreator extends SparkCommonServiceURLCreator {
+
+  private static final String RESOURCE_ROLE = "THRIFTSERVERUI";
+
+  @Override
+  public void init(AmbariCluster cluster) {
+    super.init(cluster);
+    primaryComponentName   = "SPARK_THRIFTSERVER";
+    secondaryComponentName = "SPARK2_THRIFTSERVER";
+    portConfigProperty     = "hive.server2.thrift.http.port";
+  }
+
+  @Override
+  public String getTargetService() {
+    return RESOURCE_ROLE;
+  }
+
+  @Override
+  public List<String> create(String service, Map<String, String> serviceParams)
{
+    List<String> thriftURLs = new ArrayList<>();
+
+    // Discover the URLs using the common mechanism
+    List<String> urls = super.create(service, serviceParams);
+
+    // If at least one URL was discovered, validate the transport mode, and append the endpoint
path if configured
+    if (urls != null && !urls.isEmpty()) {
+      AmbariComponent comp = cluster.getComponent(primaryComponentName);
+      if (comp == null) {
+        comp = cluster.getComponent(secondaryComponentName);
+      }
+
+      if (comp != null) {
+        String transportMode = comp.getConfigProperty("hive.server2.transport.mode");
+        if (transportMode.equalsIgnoreCase("http")) {
+          String endpoint = comp.getConfigProperty("hive.server2.http.endpoint");
+          for (String url : urls) {
+            thriftURLs.add(url + (endpoint != null ? "/" + endpoint : ""));
+          }
+        }
+      }
+    }
+
+    return thriftURLs;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/knox/blob/b4455c4c/gateway-discovery-ambari/src/main/resources/META-INF/services/org.apache.knox.gateway.topology.discovery.ambari.ServiceURLCreator
----------------------------------------------------------------------
diff --git a/gateway-discovery-ambari/src/main/resources/META-INF/services/org.apache.knox.gateway.topology.discovery.ambari.ServiceURLCreator
b/gateway-discovery-ambari/src/main/resources/META-INF/services/org.apache.knox.gateway.topology.discovery.ambari.ServiceURLCreator
index 89edbfc..f385ae6 100644
--- a/gateway-discovery-ambari/src/main/resources/META-INF/services/org.apache.knox.gateway.topology.discovery.ambari.ServiceURLCreator
+++ b/gateway-discovery-ambari/src/main/resources/META-INF/services/org.apache.knox.gateway.topology.discovery.ambari.ServiceURLCreator
@@ -23,4 +23,5 @@ org.apache.knox.gateway.topology.discovery.ambari.ResourceManagerURLCreator
 org.apache.knox.gateway.topology.discovery.ambari.YarnUIURLCreator
 org.apache.knox.gateway.topology.discovery.ambari.YarnUIV2URLCreator
 org.apache.knox.gateway.topology.discovery.ambari.SparkHistoryUIServiceURLCreator
-org.apache.knox.gateway.topology.discovery.ambari.LivyServiceURLCreator
\ No newline at end of file
+org.apache.knox.gateway.topology.discovery.ambari.LivyServiceURLCreator
+org.apache.knox.gateway.topology.discovery.ambari.SparkThriftServerUIServiceURLCreator

http://git-wip-us.apache.org/repos/asf/knox/blob/b4455c4c/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-component-config-mapping.properties
----------------------------------------------------------------------
diff --git a/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-component-config-mapping.properties
b/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-component-config-mapping.properties
index 884e143..de3474a 100644
--- a/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-component-config-mapping.properties
+++ b/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-component-config-mapping.properties
@@ -40,6 +40,8 @@ RESOURCEMANAGER=yarn-site
 SPARK_JOBHISTORYSERVER=spark-defaults
 SPARK2_JOBHISTORYSERVER=spark2-defaults
 STORM_UI_SERVER=storm-site
+SPARK_THRIFTSERVER=spark-hive-site-override
+SPARK2_THRIFTSERVER=spark2-hive-site-override
 WEBHCAT_SERVER=webhcat-site
 ZEPPELIN_MASTER=zeppelin-config
 

http://git-wip-us.apache.org/repos/asf/knox/blob/b4455c4c/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/SparkServiceURLCreatorTest.java
----------------------------------------------------------------------
diff --git a/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/SparkServiceURLCreatorTest.java
b/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/SparkServiceURLCreatorTest.java
index e12b653..9ee6dc8 100644
--- a/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/SparkServiceURLCreatorTest.java
+++ b/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/SparkServiceURLCreatorTest.java
@@ -25,6 +25,7 @@ import java.util.List;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
 
 public class SparkServiceURLCreatorTest {
 
@@ -157,4 +158,72 @@ public class SparkServiceURLCreatorTest {
   }
 
 
+  @Test
+  public void testSparkThriftUI() {
+    doTestSparkThriftUI("SPARK_THRIFTSERVER", null);
+  }
+
+  @Test
+  public void testSpark2ThriftUI() {
+    doTestSparkThriftUI("SPARK2_THRIFTSERVER", null);
+  }
+
+  @Test
+  public void testSparkThriftUIWithEndpointPath() { doTestSparkThriftUIWithEndpointPath("SPARK_THRIFTSERVER");
}
+
+  @Test
+  public void testSpark2ThriftUIWithEndpointPath() { doTestSparkThriftUIWithEndpointPath("SPARK2_THRIFTSERVER");
}
+
+  @Test
+  public void testSparkThriftUIBinaryTransport() { doTestSparkThriftUIBinaryTransport("SPARK_THRIFTSERVER");
}
+
+  @Test
+  public void testSpark2ThriftUIBinaryTransport() { doTestSparkThriftUIBinaryTransport("SPARK2_THRIFTSERVER");
}
+
+
+  private void doTestSparkThriftUIWithEndpointPath(String componentName) {
+    doTestSparkThriftUI(componentName, "http", "mypath");
+  }
+
+  private void doTestSparkThriftUIBinaryTransport(String componentName) {
+    doTestSparkThriftUI(componentName, "binary", null);
+  }
+
+
+  private void doTestSparkThriftUI(String componentName, String endpointPath) {
+    doTestSparkThriftUI(componentName, "http", endpointPath);
+  }
+
+
+  private void doTestSparkThriftUI(String componentName, String transportMode, String endpointPath)
{
+    final String PORT = "4545";
+
+    AmbariComponent ac = EasyMock.createNiceMock(AmbariComponent.class);
+    List<String> hostNames = Arrays.asList("host1", "host2");
+    EasyMock.expect(ac.getHostNames()).andReturn(hostNames).anyTimes();
+    EasyMock.expect(ac.getConfigProperty("hive.server2.thrift.http.port")).andReturn(PORT).anyTimes();
+    EasyMock.expect(ac.getConfigProperty("hive.server2.transport.mode")).andReturn(transportMode).anyTimes();
+    EasyMock.expect(ac.getConfigProperty("hive.server2.http.endpoint")).andReturn(endpointPath).anyTimes();
+    EasyMock.replay(ac);
+
+    AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class);
+    EasyMock.expect(cluster.getComponent(componentName)).andReturn(ac).anyTimes();
+    EasyMock.replay(cluster);
+
+    SparkThriftServerUIServiceURLCreator c = new SparkThriftServerUIServiceURLCreator();
+    c.init(cluster);
+    List<String> urls = c.create("THRIFTSERVERUI", null);
+    assertNotNull(urls);
+
+    if ("http".equalsIgnoreCase(transportMode)) {
+      assertFalse(urls.isEmpty());
+      assertEquals(2, urls.size());
+      assertEquals("http://host1:" + PORT + (endpointPath != null ? "/" + endpointPath :
""), urls.get(0));
+      assertEquals("http://host2:" + PORT + (endpointPath != null ? "/" + endpointPath :
""), urls.get(1));
+    } else {
+      assertTrue(urls.isEmpty());
+    }
+  }
+
+
 }


Mime
View raw message