helix-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hu...@apache.org
Subject [helix] branch zooscalability updated: Implement getAllMappingUnderPath and getMetadataStoreRealm in ZkMetadataStoreDirectory
Date Wed, 12 Feb 2020 23:16:38 GMT
This is an automated email from the ASF dual-hosted git repository.

hulee pushed a commit to branch zooscalability
in repository https://gitbox.apache.org/repos/asf/helix.git


The following commit(s) were added to refs/heads/zooscalability by this push:
     new 1041ef9  Implement getAllMappingUnderPath and getMetadataStoreRealm in ZkMetadataStoreDirectory
1041ef9 is described below

commit 1041ef97ba2e247b0745651128d9ed39d4d2f287
Author: Neal Sun <nealsun.0428@gmail.com>
AuthorDate: Wed Feb 12 15:16:28 2020 -0800

    Implement getAllMappingUnderPath and getMetadataStoreRealm in ZkMetadataStoreDirectory
    
    This PR implements getAllMappingUnderPath and getMetadataStoreRealm in ZkMetadataStoreDirectory.
Also, getRoutingData in ZkRoutingDataReader is updated : not having realm addresses and not
having sharding keys for some realm addresses no longer raise exceptions. We now allow realms
to not have sharding keys and return them as a part of routing data.
---
 .../helix/rest/metadatastore/TrieRoutingData.java  |  6 +---
 .../metadatastore/ZkMetadataStoreDirectory.java    | 16 +++++++---
 .../accessor/ZkRoutingDataReader.java              | 27 ++++++-----------
 .../rest/metadatastore/TestTrieRoutingData.java    | 13 --------
 .../TestZkMetadataStoreDirectory.java              | 35 ++++++++++++++++++++++
 .../accessor/TestZkRoutingDataReader.java          | 18 +++++------
 6 files changed, 64 insertions(+), 51 deletions(-)

diff --git a/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/TrieRoutingData.java
b/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/TrieRoutingData.java
index 28add4c..923f818 100644
--- a/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/TrieRoutingData.java
+++ b/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/TrieRoutingData.java
@@ -162,15 +162,11 @@ public class TrieRoutingData implements MetadataStoreRoutingData {
    * @throws InvalidRoutingDataException - when there is an empty sharding key (edge case
that
    *           always renders the routing data invalid); when there is a sharding key which
already
    *           contains a sharding key (invalid); when there is a sharding key that is a
part of
-   *           another sharding key (invalid)
+   *           another sharding key (invalid); when a sharding key doesn't have a leading
delimiter
    */
   private void constructTrie(Map<String, List<String>> routingData)
       throws InvalidRoutingDataException {
     for (Map.Entry<String, List<String>> entry : routingData.entrySet()) {
-      if (entry.getValue().isEmpty()) {
-        throw new InvalidRoutingDataException(
-            "Realm address does not have associating sharding keys: " + entry.getKey());
-      }
       for (String shardingKey : entry.getValue()) {
         // Missing leading delimiter is invalid
         if (shardingKey.isEmpty() || !shardingKey.substring(0, 1).equals(DELIMITER)) {
diff --git a/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/ZkMetadataStoreDirectory.java
b/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/ZkMetadataStoreDirectory.java
index 536d058..3be9b72 100644
--- a/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/ZkMetadataStoreDirectory.java
+++ b/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/ZkMetadataStoreDirectory.java
@@ -79,6 +79,8 @@ public class ZkMetadataStoreDirectory implements MetadataStoreDirectory,
Routing
       // Populate realmToShardingKeys with ZkRoutingDataReader
       _realmToShardingKeysMap.put(routingEntry.getKey(),
           _routingDataReaderMap.get(routingEntry.getKey()).getRoutingData());
+      _routingDataMap.put(routingEntry.getKey(),
+          new TrieRoutingData(_realmToShardingKeysMap.get(routingEntry.getKey())));
     }
   }
 
@@ -119,14 +121,20 @@ public class ZkMetadataStoreDirectory implements MetadataStoreDirectory,
Routing
 
   @Override
   public Map<String, String> getAllMappingUnderPath(String namespace, String path)
{
-    // TODO: get it from routingData
-    throw new UnsupportedOperationException();
+    if (!_routingDataMap.containsKey(namespace)) {
+      throw new NoSuchElementException(
+          "Failed to get all mapping under path: Namespace " + namespace + " is not found!");
+    }
+    return _routingDataMap.get(namespace).getAllMappingUnderPath(path);
   }
 
   @Override
   public String getMetadataStoreRealm(String namespace, String shardingKey) {
-    // TODO: get it from routingData
-    throw new UnsupportedOperationException();
+    if (!_routingDataMap.containsKey(namespace)) {
+      throw new NoSuchElementException(
+          "Failed to get metadata store realm: Namespace " + namespace + " is not found!");
+    }
+    return _routingDataMap.get(namespace).getMetadataStoreRealm(shardingKey);
   }
 
   @Override
diff --git a/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/accessor/ZkRoutingDataReader.java
b/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/accessor/ZkRoutingDataReader.java
index 9decf23..44ce110 100644
--- a/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/accessor/ZkRoutingDataReader.java
+++ b/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/accessor/ZkRoutingDataReader.java
@@ -72,36 +72,27 @@ public class ZkRoutingDataReader implements MetadataStoreRoutingDataReader,
IZkD
   /**
    * Returns (realm, list of ZK path sharding keys) mappings.
    * @return Map <realm, list of ZK path sharding keys>
-   * @throws InvalidRoutingDataException
+   * @throws InvalidRoutingDataException - when the node on
+   *           MetadataStoreRoutingConstants.ROUTING_DATA_PATH is missing
    */
-  public Map<String, List<String>> getRoutingData()
-      throws InvalidRoutingDataException {
+  public Map<String, List<String>> getRoutingData() throws InvalidRoutingDataException
{
     Map<String, List<String>> routingData = new HashMap<>();
-    List<String> children;
+    List<String> allRealmAddresses;
     try {
-      children = _zkClient.getChildren(MetadataStoreRoutingConstants.ROUTING_DATA_PATH);
+      allRealmAddresses = _zkClient.getChildren(MetadataStoreRoutingConstants.ROUTING_DATA_PATH);
     } catch (ZkNoNodeException e) {
       throw new InvalidRoutingDataException(
           "Routing data directory ZNode " + MetadataStoreRoutingConstants.ROUTING_DATA_PATH
               + " does not exist. Routing ZooKeeper address: " + _zkAddress);
     }
-    if (children == null || children.isEmpty()) {
-      throw new InvalidRoutingDataException(
-          "There are no metadata store realms defined. Routing ZooKeeper address: " + _zkAddress);
-    }
-    for (String child : children) {
+    for (String realmAddress : allRealmAddresses) {
       ZNRecord record =
-          _zkClient.readData(MetadataStoreRoutingConstants.ROUTING_DATA_PATH + "/" + child);
+          _zkClient.readData(MetadataStoreRoutingConstants.ROUTING_DATA_PATH + "/" + realmAddress);
       List<String> shardingKeys =
           record.getListField(MetadataStoreRoutingConstants.ZNRECORD_LIST_FIELD_KEY);
-      if (shardingKeys == null || shardingKeys.isEmpty()) {
-        throw new InvalidRoutingDataException(
-            "Realm address ZNode " + MetadataStoreRoutingConstants.ROUTING_DATA_PATH + "/"
+ child
-                + " does not have a value for key "
-                + MetadataStoreRoutingConstants.ZNRECORD_LIST_FIELD_KEY
-                + ". Routing ZooKeeper address: " + _zkAddress);
+      if (shardingKeys != null) {
+        routingData.put(realmAddress, shardingKeys);
       }
-      routingData.put(child, shardingKeys);
     }
     return routingData;
   }
diff --git a/helix-rest/src/test/java/org/apache/helix/rest/metadatastore/TestTrieRoutingData.java
b/helix-rest/src/test/java/org/apache/helix/rest/metadatastore/TestTrieRoutingData.java
index bf71456..4de68a6 100644
--- a/helix-rest/src/test/java/org/apache/helix/rest/metadatastore/TestTrieRoutingData.java
+++ b/helix-rest/src/test/java/org/apache/helix/rest/metadatastore/TestTrieRoutingData.java
@@ -67,19 +67,6 @@ public class TestTrieRoutingData {
   }
 
   @Test
-  public void testConstructionEmptyShardingKeys() {
-    Map<String, List<String>> routingData = new HashMap<>();
-    routingData.put("realmAddress1", Collections.emptyList());
-    try {
-      new TrieRoutingData(routingData);
-      Assert.fail("Expecting InvalidRoutingDataException");
-    } catch (InvalidRoutingDataException e) {
-      Assert.assertTrue(e.getMessage()
-          .contains("Realm address does not have associating sharding keys: realmAddress1"));
-    }
-  }
-
-  @Test
   public void testConstructionShardingKeyNoLeadingSlash() {
     Map<String, List<String>> routingData = new HashMap<>();
     routingData.put("realmAddress1", Arrays.asList("/g", "/h/i", "/h/j"));
diff --git a/helix-rest/src/test/java/org/apache/helix/rest/metadatastore/TestZkMetadataStoreDirectory.java
b/helix-rest/src/test/java/org/apache/helix/rest/metadatastore/TestZkMetadataStoreDirectory.java
index c0741ee..68b93b3 100644
--- a/helix-rest/src/test/java/org/apache/helix/rest/metadatastore/TestZkMetadataStoreDirectory.java
+++ b/helix-rest/src/test/java/org/apache/helix/rest/metadatastore/TestZkMetadataStoreDirectory.java
@@ -21,6 +21,7 @@ package org.apache.helix.rest.metadatastore;
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -152,6 +153,40 @@ public class TestZkMetadataStoreDirectory extends AbstractTestClass {
   }
 
   @Test(dependsOnMethods = "testGetAllShardingKeysInRealm")
+  public void testGetAllMappingUnderPath() {
+    Map<String, String> mappingFromRoot = new HashMap<>();
+    TEST_SHARDING_KEYS_1.forEach(shardingKey -> {
+      mappingFromRoot.put(shardingKey, TEST_REALM_1);
+    });
+    TEST_SHARDING_KEYS_2.forEach(shardingKey -> {
+      mappingFromRoot.put(shardingKey, TEST_REALM_2);
+    });
+
+    Map<String, String> mappingFromLeaf = new HashMap<>(1);
+    mappingFromLeaf.put("/sharding/key/1/a", TEST_REALM_1);
+
+    for (String namespace : _routingZkAddrMap.keySet()) {
+      Assert.assertEquals(_metadataStoreDirectory.getAllMappingUnderPath(namespace, "/"),
+          mappingFromRoot);
+      Assert.assertEquals(
+          _metadataStoreDirectory.getAllMappingUnderPath(namespace, "/sharding/key/1/a"),
+          mappingFromLeaf);
+    }
+  }
+
+  @Test(dependsOnMethods = "testGetAllMappingUnderPath")
+  public void testGetMetadataStoreRealm() {
+    for (String namespace : _routingZkAddrMap.keySet()) {
+      Assert.assertEquals(
+          _metadataStoreDirectory.getMetadataStoreRealm(namespace, "/sharding/key/1/a/x/y/z"),
+          TEST_REALM_1);
+      Assert.assertEquals(
+          _metadataStoreDirectory.getMetadataStoreRealm(namespace, "/sharding/key/1/d/x/y/z"),
+          TEST_REALM_2);
+    }
+  }
+
+  @Test(dependsOnMethods = "testGetMetadataStoreRealm")
   public void testDataChangeCallback()
       throws Exception {
     // For all namespaces (Routing ZKs), add an extra sharding key to TEST_REALM_1
diff --git a/helix-rest/src/test/java/org/apache/helix/rest/metadatastore/accessor/TestZkRoutingDataReader.java
b/helix-rest/src/test/java/org/apache/helix/rest/metadatastore/accessor/TestZkRoutingDataReader.java
index aa46429..5781a85 100644
--- a/helix-rest/src/test/java/org/apache/helix/rest/metadatastore/accessor/TestZkRoutingDataReader.java
+++ b/helix-rest/src/test/java/org/apache/helix/rest/metadatastore/accessor/TestZkRoutingDataReader.java
@@ -104,11 +104,10 @@ public class TestZkRoutingDataReader extends AbstractTestClass {
     _baseAccessor.create(MetadataStoreRoutingConstants.ROUTING_DATA_PATH, new ZNRecord("test"),
         AccessOption.PERSISTENT);
     try {
-      _zkRoutingDataReader.getRoutingData();
-      Assert.fail("Expecting InvalidRoutingDataException");
+      Map<String, List<String>> routingData = _zkRoutingDataReader.getRoutingData();
+      Assert.assertEquals(routingData.size(), 0);
     } catch (InvalidRoutingDataException e) {
-      Assert.assertTrue(e.getMessage().contains(
-          "There are no metadata store realms defined. Routing ZooKeeper address: " + ZK_ADDR));
+      Assert.fail("Not expecting InvalidRoutingDataException");
     }
   }
 
@@ -120,14 +119,11 @@ public class TestZkRoutingDataReader extends AbstractTestClass {
     _baseAccessor.create(MetadataStoreRoutingConstants.ROUTING_DATA_PATH + "/testRealmAddress1",
         testZnRecord1, AccessOption.PERSISTENT);
     try {
-      _zkRoutingDataReader.getRoutingData();
-      Assert.fail("Expecting InvalidRoutingDataException");
+      Map<String, List<String>> routingData = _zkRoutingDataReader.getRoutingData();
+      Assert.assertEquals(routingData.size(), 1);
+      Assert.assertTrue(routingData.get("testRealmAddress1").isEmpty());
     } catch (InvalidRoutingDataException e) {
-      Assert.assertTrue(e.getMessage().contains(
-          "Realm address ZNode " + MetadataStoreRoutingConstants.ROUTING_DATA_PATH
-              + "/testRealmAddress1 does not have a value for key "
-              + MetadataStoreRoutingConstants.ZNRECORD_LIST_FIELD_KEY
-              + ". Routing ZooKeeper address: " + ZK_ADDR));
+      Assert.fail("Not expecting InvalidRoutingDataException");
     }
   }
 }


Mime
View raw message