helix-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hu...@apache.org
Subject [helix] 02/44: Implementation of ClusterService's getClusterTopology method
Date Sat, 25 May 2019 01:19:36 GMT
This is an automated email from the ASF dual-hosted git repository.

hulee pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git

commit 6e3f4734f0f43ad54f3ac7e89023a47c37d82e58
Author: Yi Wang <ywang4@linkedin.com>
AuthorDate: Tue Mar 19 14:16:53 2019 -0700

    Implementation of ClusterService's getClusterTopology method
    
    RB=1601257
    G=helix-reviewers
    A=jxue
    
    Signed-off-by: Hunter Lee <hulee@linkedin.com>
---
 .../java/org/apache/helix/model/ClusterConfig.java | 17 ++++-
 .../org/apache/helix/model/TestClusterConfig.java  | 18 +++++
 helix-rest/pom.xml                                 |  5 ++
 .../rest/server/json/cluster/ClusterTopology.java  | 26 +++----
 .../server/resources/helix/ClusterAccessor.java    | 20 ++++-
 .../rest/server/service/ClusterServiceImpl.java    | 83 ++++++++++++++++++++
 .../helix/rest/server/AbstractTestClass.java       |  4 +
 .../helix/rest/server/TestClusterAccessor.java     | 21 ++++++
 .../server/json/cluster/TestClusterTopology.java   |  5 +-
 .../rest/server/service/TestClusterService.java    | 88 ++++++++++++++++++++++
 10 files changed, 265 insertions(+), 22 deletions(-)

diff --git a/helix-core/src/main/java/org/apache/helix/model/ClusterConfig.java b/helix-core/src/main/java/org/apache/helix/model/ClusterConfig.java
index f78f586..ec9b1d3 100644
--- a/helix-core/src/main/java/org/apache/helix/model/ClusterConfig.java
+++ b/helix-core/src/main/java/org/apache/helix/model/ClusterConfig.java
@@ -19,13 +19,14 @@ package org.apache.helix.model;
  * under the License.
  */
 
-import com.google.common.collect.Maps;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+
 import org.apache.helix.HelixException;
 import org.apache.helix.HelixProperty;
 import org.apache.helix.ZNRecord;
@@ -34,10 +35,13 @@ import org.apache.helix.api.config.StateTransitionThrottleConfig;
 import org.apache.helix.api.config.StateTransitionTimeoutConfig;
 import org.apache.helix.api.config.ViewClusterSourceConfig;
 
+import com.google.common.collect.Maps;
+
 /**
  * Cluster configurations
  */
 public class ClusterConfig extends HelixProperty {
+  private static final String TOPOLOGY_SPLITTER = "/";
   /**
    * Configurable characteristics of a cluster.
    * NOTE: Do NOT use this field name directly, use its corresponding getter/setter in the
@@ -334,6 +338,17 @@ public class ClusterConfig extends HelixProperty {
   }
 
   /**
+   * Get cluster topology by level.
+   * E.g, {zone, rack, host, instance}
+   * @return
+   */
+  public String[] getTopologyLevel() {
+    String topology = getTopology();
+    String[] parts = topology.split(TOPOLOGY_SPLITTER);
+    return Arrays.copyOfRange(parts, 1, parts.length);
+  }
+
+  /**
    * Set cluster fault zone type, this should be set combined with {@link #setTopology(String)}.
    * @param faultZoneType
    */
diff --git a/helix-core/src/test/java/org/apache/helix/model/TestClusterConfig.java b/helix-core/src/test/java/org/apache/helix/model/TestClusterConfig.java
new file mode 100644
index 0000000..0b4fb0a
--- /dev/null
+++ b/helix-core/src/test/java/org/apache/helix/model/TestClusterConfig.java
@@ -0,0 +1,18 @@
+package org.apache.helix.model;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+
+public class TestClusterConfig {
+
+  @Test
+  public void testGetZoneId() {
+    ClusterConfig clusterConfig = new ClusterConfig("test");
+    clusterConfig.setTopology("/zone/rack/host/instance");
+
+    String[] levels = clusterConfig.getTopologyLevel();
+
+    Assert.assertEquals(levels.length, 4);
+  }
+}
diff --git a/helix-rest/pom.xml b/helix-rest/pom.xml
index 01eda64..390d578 100644
--- a/helix-rest/pom.xml
+++ b/helix-rest/pom.xml
@@ -147,6 +147,11 @@ under the License.
       <type>test-jar</type>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-all</artifactId>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
   <build>
     <resources>
diff --git a/helix-rest/src/main/java/org/apache/helix/rest/server/json/cluster/ClusterTopology.java
b/helix-rest/src/main/java/org/apache/helix/rest/server/json/cluster/ClusterTopology.java
index a04f65f..c581c5d 100644
--- a/helix-rest/src/main/java/org/apache/helix/rest/server/json/cluster/ClusterTopology.java
+++ b/helix-rest/src/main/java/org/apache/helix/rest/server/json/cluster/ClusterTopology.java
@@ -10,9 +10,8 @@ import com.fasterxml.jackson.annotation.JsonProperty;
  * The Cluster Topology represents the hierarchy of the cluster:
  * Cluster
  * - Zone
- * -- Rack
+ * -- Rack(Optional)
  * --- Instance
- * ---- Partition
  * Each layer consists its id and metadata
  */
 public class ClusterTopology {
@@ -26,6 +25,14 @@ public class ClusterTopology {
     this.zones = zones;
   }
 
+  public String getClusterId() {
+    return clusterId;
+  }
+
+  public List<Zone> getZones() {
+    return zones;
+  }
+
   public static final class Zone {
     @JsonProperty("id")
     private final String id;
@@ -53,24 +60,9 @@ public class ClusterTopology {
   public static final class Instance {
     @JsonProperty("id")
     private final String id;
-    @JsonProperty("partitions")
-    private List<String> partitions;
 
     public Instance(String id) {
       this.id = id;
     }
-
-    public Instance(String id, List<String> partitions) {
-      this.id = id;
-      this.partitions = partitions;
-    }
-
-    public List<String> getPartitions() {
-      return partitions;
-    }
-
-    public void setPartitions(List<String> partitions) {
-      this.partitions = partitions;
-    }
   }
 }
diff --git a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
index 4427d50..2266f97 100644
--- a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
+++ b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
@@ -19,12 +19,12 @@ package org.apache.helix.rest.server.resources.helix;
  * under the License.
  */
 
-import com.google.common.collect.ImmutableMap;
 import java.io.IOException;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+
 import javax.ws.rs.DELETE;
 import javax.ws.rs.DefaultValue;
 import javax.ws.rs.GET;
@@ -34,6 +34,7 @@ import javax.ws.rs.Path;
 import javax.ws.rs.PathParam;
 import javax.ws.rs.QueryParam;
 import javax.ws.rs.core.Response;
+
 import org.apache.helix.AccessOption;
 import org.apache.helix.ConfigAccessor;
 import org.apache.helix.HelixAdmin;
@@ -51,11 +52,17 @@ import org.apache.helix.model.MaintenanceSignal;
 import org.apache.helix.model.Message;
 import org.apache.helix.model.StateModelDefinition;
 import org.apache.helix.model.builder.HelixConfigScopeBuilder;
+import org.apache.helix.rest.server.json.cluster.ClusterTopology;
+import org.apache.helix.rest.server.service.ClusterService;
+import org.apache.helix.rest.server.service.ClusterServiceImpl;
 import org.apache.helix.tools.ClusterSetup;
 import org.codehaus.jackson.type.TypeReference;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.ImmutableMap;
+
 @Path("/clusters")
 public class ClusterAccessor extends AbstractHelixResource {
   private static Logger _logger = LoggerFactory.getLogger(ClusterAccessor.class.getName());
@@ -260,6 +267,17 @@ public class ClusterAccessor extends AbstractHelixResource {
     return JSONRepresentation(config.getRecord());
   }
 
+  @GET
+  @Path("{clusterId}/topology")
+  public Response getClusterTopology(@PathParam("clusterId") String clusterId) throws IOException
{
+    //TODO reduce the GC by dependency injection
+    ClusterService clusterService = new ClusterServiceImpl(getDataAccssor(clusterId), getConfigAccessor());
+    ObjectMapper objectMapper = new ObjectMapper();
+    ClusterTopology clusterTopology = clusterService.getClusterTopology(clusterId);
+
+    return OK(objectMapper.writeValueAsString(clusterTopology));
+  }
+
   @POST
   @Path("{clusterId}/configs")
   public Response updateClusterConfig(
diff --git a/helix-rest/src/main/java/org/apache/helix/rest/server/service/ClusterServiceImpl.java
b/helix-rest/src/main/java/org/apache/helix/rest/server/service/ClusterServiceImpl.java
new file mode 100644
index 0000000..5fb9c61
--- /dev/null
+++ b/helix-rest/src/main/java/org/apache/helix/rest/server/service/ClusterServiceImpl.java
@@ -0,0 +1,83 @@
+package org.apache.helix.rest.server.service;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.helix.AccessOption;
+import org.apache.helix.ConfigAccessor;
+import org.apache.helix.HelixDataAccessor;
+import org.apache.helix.PropertyKey;
+import org.apache.helix.model.InstanceConfig;
+import org.apache.helix.model.LiveInstance;
+import org.apache.helix.rest.server.json.cluster.ClusterInfo;
+import org.apache.helix.rest.server.json.cluster.ClusterTopology;
+
+public class ClusterServiceImpl implements ClusterService {
+  private final HelixDataAccessor _dataAccessor;
+  private final ConfigAccessor _configAccessor;
+
+  public ClusterServiceImpl(HelixDataAccessor dataAccessor, ConfigAccessor configAccessor)
{
+    _dataAccessor = dataAccessor;
+    _configAccessor = configAccessor;
+  }
+
+  @Override
+  public ClusterTopology getClusterTopology(String cluster) {
+    String zoneField = _configAccessor.getClusterConfig(cluster).getFaultZoneType();
+    PropertyKey.Builder keyBuilder = _dataAccessor.keyBuilder();
+    List<InstanceConfig> instanceConfigs =
+        _dataAccessor.getChildValues(keyBuilder.instanceConfigs());
+    Map<String, List<ClusterTopology.Instance>> instanceMapByZone = new HashMap<>();
+    if (instanceConfigs != null && !instanceConfigs.isEmpty()) {
+      for (InstanceConfig instanceConfig : instanceConfigs) {
+        if (!instanceConfig.getDomainAsMap().containsKey(zoneField)) {
+          continue;
+        }
+        final String instanceName = instanceConfig.getInstanceName();
+        final ClusterTopology.Instance instance = new ClusterTopology.Instance(instanceName);
+        final String zoneId = instanceConfig.getDomainAsMap().get(zoneField);
+        if (instanceMapByZone.containsKey(zoneId)) {
+          instanceMapByZone.get(zoneId).add(instance);
+        } else {
+          instanceMapByZone.put(zoneId, new ArrayList<ClusterTopology.Instance>() {
+            {
+              add(instance);
+            }
+          });
+        }
+      }
+    }
+    List<ClusterTopology.Zone> zones = new ArrayList<>();
+    for (String zoneId : instanceMapByZone.keySet()) {
+      ClusterTopology.Zone zone = new ClusterTopology.Zone(zoneId);
+      zone.setInstances(instanceMapByZone.get(zoneId));
+      zones.add(zone);
+    }
+
+    return new ClusterTopology(cluster, zones);
+  }
+
+  @Override
+  public ClusterInfo getClusterInfo(String clusterId) {
+    ClusterInfo.Builder builder = new ClusterInfo.Builder(clusterId);
+    PropertyKey.Builder keyBuilder = _dataAccessor.keyBuilder();
+    LiveInstance controller =
+        _dataAccessor.getProperty(_dataAccessor.keyBuilder().controllerLeader());
+    if (controller != null) {
+      builder.controller(controller.getInstanceName());
+    } else {
+      builder.controller("No Lead Controller");
+    }
+
+    return builder
+        .paused(_dataAccessor.getBaseDataAccessor().exists(keyBuilder.pause().getPath(),
+            AccessOption.PERSISTENT))
+        .maintenance(_dataAccessor.getBaseDataAccessor().exists(keyBuilder.maintenance().getPath(),
+            AccessOption.PERSISTENT))
+        .idealStates(_dataAccessor.getChildNames(keyBuilder.idealStates()))
+        .instances(_dataAccessor.getChildNames(keyBuilder.instances()))
+        .liveInstances(_dataAccessor.getChildNames(keyBuilder.liveInstances())).build();
+  }
+}
diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/AbstractTestClass.java
b/helix-rest/src/test/java/org/apache/helix/rest/server/AbstractTestClass.java
index 01dd7b8..9caed00 100644
--- a/helix-rest/src/test/java/org/apache/helix/rest/server/AbstractTestClass.java
+++ b/helix-rest/src/test/java/org/apache/helix/rest/server/AbstractTestClass.java
@@ -51,6 +51,7 @@ import org.apache.helix.manager.zk.ZNRecordSerializer;
 import org.apache.helix.manager.zk.ZkBaseDataAccessor;
 import org.apache.helix.manager.zk.client.DedicatedZkClientFactory;
 import org.apache.helix.manager.zk.client.HelixZkClient;
+import org.apache.helix.model.ClusterConfig;
 import org.apache.helix.rest.common.ContextPropertyKeys;
 import org.apache.helix.rest.common.HelixRestNamespace;
 import org.apache.helix.rest.server.auditlog.AuditLog;
@@ -264,6 +265,9 @@ public class AbstractTestClass extends JerseyTestNg.ContainerPerClassTest
{
     for (String cluster : _clusters) {
       Set<String> instances = createInstances(cluster, 10);
       Set<String> liveInstances = startInstances(cluster, instances, 6);
+      ClusterConfig clusterConfig = new ClusterConfig(cluster);
+      clusterConfig.setFaultZoneType("helixZoneId");
+      _configAccessor.setClusterConfig(cluster, clusterConfig);
       createResourceConfigs(cluster, 8);
       _workflowMap.put(cluster, createWorkflows(cluster, 3));
       Set<String> resources = createResources(cluster, 8);
diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
index 59da6e1..fc5d94a 100644
--- a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
+++ b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
@@ -38,12 +38,14 @@ import org.apache.helix.ZNRecord;
 import org.apache.helix.manager.zk.ZKHelixDataAccessor;
 import org.apache.helix.manager.zk.ZKUtil;
 import org.apache.helix.model.ClusterConfig;
+import org.apache.helix.model.InstanceConfig;
 import org.apache.helix.model.MaintenanceSignal;
 import org.apache.helix.rest.common.HelixRestNamespace;
 import org.apache.helix.rest.server.auditlog.AuditLog;
 import org.apache.helix.rest.server.resources.AbstractResource;
 import org.apache.helix.rest.server.resources.AbstractResource.Command;
 import org.apache.helix.rest.server.resources.helix.ClusterAccessor;
+import org.apache.helix.rest.server.util.JerseyUriRequestBuilder;
 import org.codehaus.jackson.JsonNode;
 import org.codehaus.jackson.map.ObjectMapper;
 import org.codehaus.jackson.type.TypeReference;
@@ -86,6 +88,25 @@ public class TestClusterAccessor extends AbstractTestClass {
   }
 
   @Test(dependsOnMethods = "testGetClusters")
+  public void testGetClusterTopology() {
+    System.out.println("Start test :" + TestHelper.getTestMethodName());
+    String cluster = "TestCluster_1";
+    String instance = cluster + "localhost_12920";
+    // set the fake zone id in instance configuration
+    HelixDataAccessor helixDataAccessor = new ZKHelixDataAccessor(cluster, _baseAccessor);
+    InstanceConfig instanceConfig =
+        helixDataAccessor.getProperty(helixDataAccessor.keyBuilder().instanceConfig(instance));
+    instanceConfig.setDomain("helixZoneId=123");
+    helixDataAccessor.setProperty(helixDataAccessor.keyBuilder().instanceConfig(instance),
+        instanceConfig);
+
+    String response = new JerseyUriRequestBuilder("clusters/{}/topology").format(cluster).get(this);
+
+    Assert.assertEquals(response,
+        "{\"id\":\"TestCluster_1\",\"zones\":[{\"id\":\"123\",\"instances\":[{\"id\":\"TestCluster_1localhost_12920\"}]}]}");
+  }
+
+  @Test(dependsOnMethods = "testGetClusterTopology")
   public void testAddConfigFields() throws IOException {
     System.out.println("Start test :" + TestHelper.getTestMethodName());
     String cluster = _clusters.iterator().next();
diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/json/cluster/TestClusterTopology.java
b/helix-rest/src/test/java/org/apache/helix/rest/server/json/cluster/TestClusterTopology.java
index a2b90fe..eadaa58 100644
--- a/helix-rest/src/test/java/org/apache/helix/rest/server/json/cluster/TestClusterTopology.java
+++ b/helix-rest/src/test/java/org/apache/helix/rest/server/json/cluster/TestClusterTopology.java
@@ -13,9 +13,8 @@ public class TestClusterTopology {
 
   @Test
   public void whenSerializingClusterTopology() throws IOException {
-    List<String> partitions = ImmutableList.of("db0", "db1");
     List<ClusterTopology.Instance> instances =
-        ImmutableList.of(new ClusterTopology.Instance("instance", partitions));
+        ImmutableList.of(new ClusterTopology.Instance("instance"));
 
     List<ClusterTopology.Zone> zones = ImmutableList.of(new ClusterTopology.Zone("zone",
instances));
 
@@ -24,6 +23,6 @@ public class TestClusterTopology {
     String result = mapper.writeValueAsString(clusterTopology);
 
     Assert.assertEquals(result,
-        "{\"id\":\"cluster0\",\"zones\":[{\"id\":\"zone\",\"instances\":[{\"id\":\"instance\",\"partitions\":[\"db0\",\"db1\"]}]}]}");
+        "{\"id\":\"cluster0\",\"zones\":[{\"id\":\"zone\",\"instances\":[{\"id\":\"instance\"}]}]}");
   }
 }
diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/service/TestClusterService.java
b/helix-rest/src/test/java/org/apache/helix/rest/server/service/TestClusterService.java
new file mode 100644
index 0000000..a988326
--- /dev/null
+++ b/helix-rest/src/test/java/org/apache/helix/rest/server/service/TestClusterService.java
@@ -0,0 +1,88 @@
+package org.apache.helix.rest.server.service;
+
+import static org.mockito.Mockito.*;
+
+import java.util.List;
+
+import org.apache.helix.ConfigAccessor;
+import org.apache.helix.HelixDataAccessor;
+import org.apache.helix.HelixProperty;
+import org.apache.helix.PropertyKey;
+import org.apache.helix.model.ClusterConfig;
+import org.apache.helix.model.InstanceConfig;
+import org.apache.helix.rest.server.json.cluster.ClusterTopology;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import com.google.common.collect.ImmutableList;
+
+
+public class TestClusterService {
+  private static final String TEST_CLUSTER = "Test_Cluster";
+
+  @Test
+  public void testGetClusterTopology_whenMultiZones() {
+    InstanceConfig instanceConfig1 = new InstanceConfig("instance0");
+    instanceConfig1.setDomain("helixZoneId=zone0");
+    InstanceConfig instanceConfig2 = new InstanceConfig("instance1");
+    instanceConfig2.setDomain("helixZoneId=zone1");
+    List<HelixProperty> instanceConfigs = (List) ImmutableList.of(instanceConfig1,
instanceConfig2);
+
+    Mock mock = new Mock();
+    when(mock.dataAccessor.keyBuilder()).thenReturn(new PropertyKey.Builder(TEST_CLUSTER));
+    when(mock.dataAccessor.getChildValues(any(PropertyKey.class))).thenReturn(instanceConfigs);
+
+    ClusterTopology clusterTopology = mock.clusterService.getClusterTopology(TEST_CLUSTER);
+
+    Assert.assertEquals(clusterTopology.getZones().size(), 2);
+    Assert.assertEquals(clusterTopology.getClusterId(), TEST_CLUSTER);
+  }
+
+  @Test
+  public void testGetClusterTopology_whenZeroZones() {
+    InstanceConfig instanceConfig1 = new InstanceConfig("instance0");
+    InstanceConfig instanceConfig2 = new InstanceConfig("instance1");
+    List<HelixProperty> instanceConfigs = (List) ImmutableList.of(instanceConfig1,
instanceConfig2);
+
+    Mock mock = new Mock();
+    when(mock.dataAccessor.keyBuilder()).thenReturn(new PropertyKey.Builder(TEST_CLUSTER));
+    when(mock.dataAccessor.getChildValues(any(PropertyKey.class))).thenReturn(instanceConfigs);
+
+    ClusterTopology clusterTopology = mock.clusterService.getClusterTopology(TEST_CLUSTER);
+
+    Assert.assertEquals(clusterTopology.getZones().size(), 0);
+    Assert.assertEquals(clusterTopology.getClusterId(), TEST_CLUSTER);
+  }
+
+  @Test
+  public void testGetClusterTopology_whenZoneHasMultiInstances() {
+    InstanceConfig instanceConfig1 = new InstanceConfig("instance0");
+    instanceConfig1.setDomain("helixZoneId=zone0");
+    InstanceConfig instanceConfig2 = new InstanceConfig("instance1");
+    instanceConfig2.setDomain("helixZoneId=zone0");
+    List<HelixProperty> instanceConfigs = (List) ImmutableList.of(instanceConfig1,
instanceConfig2);
+
+    Mock mock = new Mock();
+    when(mock.dataAccessor.keyBuilder()).thenReturn(new PropertyKey.Builder(TEST_CLUSTER));
+    when(mock.dataAccessor.getChildValues(any(PropertyKey.class))).thenReturn(instanceConfigs);
+
+    ClusterTopology clusterTopology = mock.clusterService.getClusterTopology(TEST_CLUSTER);
+
+    Assert.assertEquals(clusterTopology.getZones().size(), 1);
+    Assert.assertEquals(clusterTopology.getZones().get(0).getInstances().size(), 2);
+    Assert.assertEquals(clusterTopology.getClusterId(), TEST_CLUSTER);
+  }
+
+  private final class Mock {
+    private HelixDataAccessor dataAccessor = mock(HelixDataAccessor.class);
+    private ConfigAccessor configAccessor = mock(ConfigAccessor.class);
+    private ClusterService clusterService;
+
+    Mock() {
+      ClusterConfig mockConfig = new ClusterConfig(TEST_CLUSTER);
+      mockConfig.setFaultZoneType("helixZoneId");
+      when(configAccessor.getClusterConfig(TEST_CLUSTER)).thenReturn(mockConfig);
+      clusterService = new ClusterServiceImpl(dataAccessor, configAccessor);
+    }
+  }
+}


Mime
View raw message