helix-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hu...@apache.org
Subject [helix] 14/14: Interface design for zone mapping information
Date Wed, 22 May 2019 23:41:39 GMT
This is an automated email from the ASF dual-hosted git repository.

hulee pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git

commit aebb0cfed286819029875ad60e102851d8d53cf5
Author: Yi Wang <ywang4@linkedin.com>
AuthorDate: Tue Mar 19 14:16:16 2019 -0700

    Interface design for zone mapping information
    
    RB=1578905
    BUG=helix-1646
    G=helix-reviewers
    A=jxue
    
    Signed-off-by: Hunter Lee <hulee@linkedin.com>
---
 .../controller/rebalancer/topology/Topology.java   |  2 +-
 .../org/apache/helix/model/InstanceConfig.java     | 19 ++++-
 .../TestConstraintRebalanceStrategy.java           |  2 +-
 .../rebalancer/CrushRebalancers/TestNodeSwap.java  |  3 +-
 .../org/apache/helix/model/TestInstanceConfig.java | 22 ++++++
 helix-rest/pom.xml                                 | 10 +++
 .../rest/server/json/cluster/ClusterInfo.java      | 80 ++++++++++++++++++++++
 .../rest/server/json/cluster/ClusterTopology.java  | 76 ++++++++++++++++++++
 .../helix/rest/server/service/ClusterService.java  | 25 +++++++
 .../rest/server/json/cluster/TestClusterInfo.java  | 27 ++++++++
 .../server/json/cluster/TestClusterTopology.java   | 29 ++++++++
 11 files changed, 290 insertions(+), 5 deletions(-)

diff --git a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/topology/Topology.java
b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/topology/Topology.java
index f5b6141..505052e 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/topology/Topology.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/topology/Topology.java
@@ -270,7 +270,7 @@ public class Topology {
 
     for (String ins : _allInstances) {
       InstanceConfig insConfig = _instanceConfigMap.get(ins);
-      String domain = insConfig.getDomain();
+      String domain = insConfig.getDomainAsString();
       if (domain == null) {
         if (insConfig.getInstanceEnabled() && (_clusterConfig.getDisabledInstances()
== null
             || !_clusterConfig.getDisabledInstances().containsKey(ins))) {
diff --git a/helix-core/src/main/java/org/apache/helix/model/InstanceConfig.java b/helix-core/src/main/java/org/apache/helix/model/InstanceConfig.java
index 3cc3c58..74ba9d7 100644
--- a/helix-core/src/main/java/org/apache/helix/model/InstanceConfig.java
+++ b/helix-core/src/main/java/org/apache/helix/model/InstanceConfig.java
@@ -35,6 +35,8 @@ import org.apache.helix.util.HelixUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.base.Splitter;
+
 /**
  * Instance configurations
  */
@@ -55,6 +57,7 @@ public class InstanceConfig extends HelixProperty {
     DELAY_REBALANCE_ENABLED,
     MAX_CONCURRENT_TASK
   }
+
   public static final int WEIGHT_NOT_SET = -1;
   public static final int MAX_CONCURRENT_TASK_NOT_SET = -1;
 
@@ -126,11 +129,25 @@ public class InstanceConfig extends HelixProperty {
    * Domain represents a hierarchy identifier for an instance.
    * @return
    */
-  public String getDomain() {
+  public String getDomainAsString() {
     return _record.getSimpleField(InstanceConfigProperty.DOMAIN.name());
   }
 
   /**
+   * Parse the key value pairs of domain and return a map structure
+   * @return
+   */
+  public Map<String, String> getDomainAsMap() {
+    String domain = getDomainAsString();
+    if (domain == null || domain.isEmpty()) {
+      return Collections.emptyMap();
+    }
+
+    return Splitter.on(',').trimResults()
+        .withKeyValueSeparator(Splitter.on('=').limit(2).trimResults()).split(domain);
+  }
+
+  /**
    * Domain represents a hierarchy identifier for an instance.
    * Example:  "cluster=myCluster,zone=myZone1,rack=myRack,host=hostname,instance=instance001".
    * @return
diff --git a/helix-core/src/test/java/org/apache/helix/controller/rebalancer/TestConstraintRebalanceStrategy.java
b/helix-core/src/test/java/org/apache/helix/controller/rebalancer/TestConstraintRebalanceStrategy.java
index cb4bb75..a9e53f8 100644
--- a/helix-core/src/test/java/org/apache/helix/controller/rebalancer/TestConstraintRebalanceStrategy.java
+++ b/helix-core/src/test/java/org/apache/helix/controller/rebalancer/TestConstraintRebalanceStrategy.java
@@ -447,7 +447,7 @@ public class TestConstraintRebalanceStrategy {
       domainPartitionMap.clear();
       for (String partition : partitionMap.keySet()) {
         for (String instance : partitionMap.get(partition).keySet()) {
-          String domain = instanceConfigs.get(instance).getDomain().split(",")[0].split("=")[1];
+          String domain = instanceConfigs.get(instance).getDomainAsString().split(",")[0].split("=")[1];
           if (domainPartitionMap.containsKey(domain)) {
             Assert.assertFalse(domainPartitionMap.get(domain).contains(partition));
           } else {
diff --git a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/CrushRebalancers/TestNodeSwap.java
b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/CrushRebalancers/TestNodeSwap.java
index 3d20f0a..61e4d55 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/CrushRebalancers/TestNodeSwap.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/CrushRebalancers/TestNodeSwap.java
@@ -28,7 +28,6 @@ import java.util.Map;
 import java.util.Set;
 
 import org.apache.helix.ConfigAccessor;
-import org.apache.helix.TestHelper;
 import org.apache.helix.common.ZkTestBase;
 import org.apache.helix.controller.rebalancer.strategy.CrushEdRebalanceStrategy;
 import org.apache.helix.controller.rebalancer.strategy.CrushRebalanceStrategy;
@@ -180,7 +179,7 @@ public class TestNodeSwap extends ZkTestBase {
     _gSetupTool.addInstanceToCluster(CLUSTER_NAME, newParticipantName);
     InstanceConfig newConfig =
         configAccessor.getInstanceConfig(CLUSTER_NAME, newParticipantName);
-    newConfig.setDomain(instanceConfig.getDomain());
+    newConfig.setDomain(instanceConfig.getDomainAsString());
     _gSetupTool.getClusterManagementTool()
         .setInstanceConfig(CLUSTER_NAME, newParticipantName, newConfig);
 
diff --git a/helix-core/src/test/java/org/apache/helix/model/TestInstanceConfig.java b/helix-core/src/test/java/org/apache/helix/model/TestInstanceConfig.java
index 69a3d9f..38b1c92 100644
--- a/helix-core/src/test/java/org/apache/helix/model/TestInstanceConfig.java
+++ b/helix-core/src/test/java/org/apache/helix/model/TestInstanceConfig.java
@@ -19,9 +19,13 @@ package org.apache.helix.model;
  * under the License.
  */
 
+import java.util.Map;
+
+import org.apache.helix.ZNRecord;
 import org.testng.Assert;
 import org.testng.annotations.Test;
 
+
 /**
  * Created with IntelliJ IDEA.
  * User: zzhang
@@ -36,4 +40,22 @@ public class TestInstanceConfig {
     Assert.assertTrue(config.isValid(),
         "HELIX-65: should not check host/port existence for instance-config");
   }
+
+  @Test
+  public void testGetParsedDomain() {
+    InstanceConfig instanceConfig = new InstanceConfig(new ZNRecord("id"));
+    instanceConfig.setDomain("cluster=myCluster,zone=myZone1,rack=myRack,host=hostname,instance=instance001");
+
+    Map<String, String> parsedDomain = instanceConfig.getDomainAsMap();
+    Assert.assertEquals(parsedDomain.size(), 5);
+    Assert.assertEquals(parsedDomain.get("zone"), "myZone1");
+  }
+
+  @Test
+  public void testGetParsedDomain_emptyDomain() {
+    InstanceConfig instanceConfig = new InstanceConfig(new ZNRecord("id"));
+
+    Map<String, String> parsedDomain = instanceConfig.getDomainAsMap();
+    Assert.assertTrue(parsedDomain.isEmpty());
+  }
 }
diff --git a/helix-rest/pom.xml b/helix-rest/pom.xml
index c3e9403..01eda64 100644
--- a/helix-rest/pom.xml
+++ b/helix-rest/pom.xml
@@ -122,6 +122,16 @@ under the License.
       <version>1.8.5</version>
     </dependency>
     <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-annotations</artifactId>
+      <version>2.9.5</version>
+    </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-databind</artifactId>
+      <version>2.9.5</version>
+    </dependency>
+    <dependency>
       <groupId>commons-cli</groupId>
       <artifactId>commons-cli</artifactId>
       <version>1.2</version>
diff --git a/helix-rest/src/main/java/org/apache/helix/rest/server/json/cluster/ClusterInfo.java
b/helix-rest/src/main/java/org/apache/helix/rest/server/json/cluster/ClusterInfo.java
new file mode 100644
index 0000000..ac1831f
--- /dev/null
+++ b/helix-rest/src/main/java/org/apache/helix/rest/server/json/cluster/ClusterInfo.java
@@ -0,0 +1,80 @@
+package org.apache.helix.rest.server.json.cluster;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.List;
+
+
+public class ClusterInfo {
+  @JsonProperty("id")
+  private final String id;
+  @JsonProperty("controller")
+  private final String controller;
+  @JsonProperty("paused")
+  private final boolean paused;
+  @JsonProperty("maintenance")
+  private final boolean maintenance;
+  @JsonProperty("resources")
+  private final List<String> idealStates;
+  @JsonProperty("instances")
+  private final List<String> instances;
+  @JsonProperty("liveInstances")
+  private final List<String> liveInstances;
+
+  private ClusterInfo(Builder builder) {
+    id = builder.id;
+    controller = builder.controller;
+    paused = builder.paused;
+    maintenance = builder.maintenance;
+    idealStates = builder.idealStates;
+    instances = builder.instances;
+    liveInstances = builder.liveInstances;
+  }
+
+  public static final class Builder {
+    private String id;
+    private String controller;
+    private boolean paused;
+    private boolean maintenance;
+    private List<String> idealStates;
+    private List<String> instances;
+    private List<String> liveInstances;
+
+    public Builder(String id) {
+      this.id = id;
+    }
+
+    public Builder controller(String controller) {
+      this.controller = controller;
+      return this;
+    }
+
+    public Builder paused(boolean paused) {
+      this.paused = paused;
+      return this;
+    }
+
+    public Builder maintenance(boolean maintenance) {
+      this.maintenance = maintenance;
+      return this;
+    }
+
+    public Builder idealStates(List<String> idealStates) {
+      this.idealStates = idealStates;
+      return this;
+    }
+
+    public Builder instances(List<String> instances) {
+      this.instances = instances;
+      return this;
+    }
+
+    public Builder liveInstances(List<String> liveInstances) {
+      this.liveInstances = liveInstances;
+      return this;
+    }
+
+    public ClusterInfo build() {
+      return new ClusterInfo(this);
+    }
+  }
+}
diff --git a/helix-rest/src/main/java/org/apache/helix/rest/server/json/cluster/ClusterTopology.java
b/helix-rest/src/main/java/org/apache/helix/rest/server/json/cluster/ClusterTopology.java
new file mode 100644
index 0000000..a04f65f
--- /dev/null
+++ b/helix-rest/src/main/java/org/apache/helix/rest/server/json/cluster/ClusterTopology.java
@@ -0,0 +1,76 @@
+package org.apache.helix.rest.server.json.cluster;
+
+import java.util.List;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+
+/**
+ * POJO class that can be easily convert to JSON object
+ * The Cluster Topology represents the hierarchy of the cluster:
+ * Cluster
+ * - Zone
+ * -- Rack
+ * --- Instance
+ * ---- Partition
+ * Each layer consists its id and metadata
+ */
+public class ClusterTopology {
+  @JsonProperty("id")
+  private final String clusterId;
+  @JsonProperty("zones")
+  private List<Zone> zones;
+
+  public ClusterTopology(String clusterId, List<Zone> zones) {
+    this.clusterId = clusterId;
+    this.zones = zones;
+  }
+
+  public static final class Zone {
+    @JsonProperty("id")
+    private final String id;
+    @JsonProperty("instances")
+    private List<Instance> instances;
+
+    public Zone(String id) {
+      this.id = id;
+    }
+
+    public Zone(String id, List<Instance> instances) {
+      this.id = id;
+      this.instances = instances;
+    }
+
+    public List<Instance> getInstances() {
+      return instances;
+    }
+
+    public void setInstances(List<Instance> instances) {
+      this.instances = instances;
+    }
+  }
+
+  public static final class Instance {
+    @JsonProperty("id")
+    private final String id;
+    @JsonProperty("partitions")
+    private List<String> partitions;
+
+    public Instance(String id) {
+      this.id = id;
+    }
+
+    public Instance(String id, List<String> partitions) {
+      this.id = id;
+      this.partitions = partitions;
+    }
+
+    public List<String> getPartitions() {
+      return partitions;
+    }
+
+    public void setPartitions(List<String> partitions) {
+      this.partitions = partitions;
+    }
+  }
+}
diff --git a/helix-rest/src/main/java/org/apache/helix/rest/server/service/ClusterService.java
b/helix-rest/src/main/java/org/apache/helix/rest/server/service/ClusterService.java
new file mode 100644
index 0000000..7234099
--- /dev/null
+++ b/helix-rest/src/main/java/org/apache/helix/rest/server/service/ClusterService.java
@@ -0,0 +1,25 @@
+package org.apache.helix.rest.server.service;
+
+import org.apache.helix.rest.server.json.cluster.ClusterInfo;
+import org.apache.helix.rest.server.json.cluster.ClusterTopology;
+
+
+/**
+ * A rest wrapper service that provides information about cluster
+ * TODO add more business logic and simplify the workload on ClusterAccessor
+ */
+public interface ClusterService {
+  /**
+   * Get cluster topology
+   * @param cluster
+   * @return
+   */
+  ClusterTopology getClusterTopology(String cluster);
+
+  /**
+   * Get cluster basic information
+   * @param clusterId
+   * @return
+   */
+  ClusterInfo getClusterInfo(String clusterId);
+}
diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/json/cluster/TestClusterInfo.java
b/helix-rest/src/test/java/org/apache/helix/rest/server/json/cluster/TestClusterInfo.java
new file mode 100644
index 0000000..1b89ec8
--- /dev/null
+++ b/helix-rest/src/test/java/org/apache/helix/rest/server/json/cluster/TestClusterInfo.java
@@ -0,0 +1,27 @@
+package org.apache.helix.rest.server.json.cluster;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.ImmutableList;
+
+public class TestClusterInfo {
+  @Test
+  public void whenSerializingClusterInfo() throws JsonProcessingException {
+    ClusterInfo clusterInfo = new ClusterInfo.Builder("cluster0")
+        .controller("controller")
+        .idealStates(ImmutableList.of("idealState0"))
+        .instances(ImmutableList.of("instance0"))
+        .maintenance(true)
+        .paused(true)
+        .liveInstances(ImmutableList.of("instance0"))
+        .build();
+    ObjectMapper mapper = new ObjectMapper();
+    String result = mapper.writeValueAsString(clusterInfo);
+
+    Assert.assertEquals(result,
+        "{\"id\":\"cluster0\",\"controller\":\"controller\",\"paused\":true,\"maintenance\":true,\"resources\":[\"idealState0\"],\"instances\":[\"instance0\"],\"liveInstances\":[\"instance0\"]}");
+  }
+}
diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/json/cluster/TestClusterTopology.java
b/helix-rest/src/test/java/org/apache/helix/rest/server/json/cluster/TestClusterTopology.java
new file mode 100644
index 0000000..a2b90fe
--- /dev/null
+++ b/helix-rest/src/test/java/org/apache/helix/rest/server/json/cluster/TestClusterTopology.java
@@ -0,0 +1,29 @@
+package org.apache.helix.rest.server.json.cluster;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.ImmutableList;
+
+public class TestClusterTopology {
+
+  @Test
+  public void whenSerializingClusterTopology() throws IOException {
+    List<String> partitions = ImmutableList.of("db0", "db1");
+    List<ClusterTopology.Instance> instances =
+        ImmutableList.of(new ClusterTopology.Instance("instance", partitions));
+
+    List<ClusterTopology.Zone> zones = ImmutableList.of(new ClusterTopology.Zone("zone",
instances));
+
+    ClusterTopology clusterTopology = new ClusterTopology("cluster0", zones);
+    ObjectMapper mapper = new ObjectMapper();
+    String result = mapper.writeValueAsString(clusterTopology);
+
+    Assert.assertEquals(result,
+        "{\"id\":\"cluster0\",\"zones\":[{\"id\":\"zone\",\"instances\":[{\"id\":\"instance\",\"partitions\":[\"db0\",\"db1\"]}]}]}");
+  }
+}


Mime
View raw message