helix-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hu...@apache.org
Subject [helix] 13/43: Add write REST endpoints to helix rest for metadata store directory (#757)
Date Tue, 17 Mar 2020 00:03:44 GMT
This is an automated email from the ASF dual-hosted git repository.

hulee pushed a commit to branch zooscalability
in repository https://gitbox.apache.org/repos/asf/helix.git

commit 3e73a8295bc8799ef598b07203f0a66ac99a674d
Author: Huizhi Lu <ihuizhi.lu@gmail.com>
AuthorDate: Thu Feb 13 15:35:46 2020 -0800

    Add write REST endpoints to helix rest for metadata store directory (#757)
    
    We have metadata store directory service to help scale out zookeeper. Metadata store directory
service provides REST APIs to access. This commit adds MSDS write endpoints to Helix REST.
    
    Changelist:
    - Add REST write endpoints to MetadataStoreDirectoryAccessor
    - Add unit tests for the new REST write endpoints
    - Fix unit tests by cleaning up routing data path in ZK
---
 .../MetadataStoreDirectoryAccessor.java            |  86 ++++++++--
 .../accessor/TestZkRoutingDataReader.java          |  21 ++-
 .../TestMetadataStoreDirectoryAccessor.java        | 173 +++++++++++++++++++--
 3 files changed, 249 insertions(+), 31 deletions(-)

diff --git a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/metadatastore/MetadataStoreDirectoryAccessor.java
b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/metadatastore/MetadataStoreDirectoryAccessor.java
index 78543d6..e731b28 100644
--- a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/metadatastore/MetadataStoreDirectoryAccessor.java
+++ b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/metadatastore/MetadataStoreDirectoryAccessor.java
@@ -25,8 +25,11 @@ import java.util.List;
 import java.util.Map;
 import java.util.NoSuchElementException;
 import javax.annotation.PostConstruct;
+import javax.ws.rs.DELETE;
 import javax.ws.rs.GET;
+import javax.ws.rs.PUT;
 import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
 import javax.ws.rs.QueryParam;
 import javax.ws.rs.core.Response;
 
@@ -51,15 +54,15 @@ import org.slf4j.LoggerFactory;
 public class MetadataStoreDirectoryAccessor extends AbstractResource {
   private static final Logger LOG = LoggerFactory.getLogger(MetadataStoreDirectoryAccessor.class);
 
-  private HelixRestNamespace _namespace;
-
-  // Double-checked locking for thread-safe object.
+  private String _namespace;
   private MetadataStoreDirectory _metadataStoreDirectory;
 
   @PostConstruct
   private void postConstruct() {
-    getHelixNamespace();
-    buildMetadataStoreDirectory();
+    HelixRestNamespace helixRestNamespace = getHelixNamespace();
+    _namespace = helixRestNamespace.getName();
+
+    buildMetadataStoreDirectory(_namespace, helixRestNamespace.getMetadataStoreAddress());
   }
 
   /**
@@ -72,8 +75,7 @@ public class MetadataStoreDirectoryAccessor extends AbstractResource {
   public Response getAllMetadataStoreRealms() {
     Map<String, Collection<String>> responseMap;
     try {
-      Collection<String> realms =
-          _metadataStoreDirectory.getAllMetadataStoreRealms(_namespace.getName());
+      Collection<String> realms = _metadataStoreDirectory.getAllMetadataStoreRealms(_namespace);
 
       responseMap = new HashMap<>(1);
       responseMap.put(MetadataStoreRoutingConstants.METADATA_STORE_REALMS, realms);
@@ -84,6 +86,30 @@ public class MetadataStoreDirectoryAccessor extends AbstractResource {
     return JSONRepresentation(responseMap);
   }
 
+  @PUT
+  @Path("/metadata-store-realms/{realm}")
+  public Response addMetadataStoreRealm(@PathParam("realm") String realm) {
+    try {
+      _metadataStoreDirectory.addMetadataStoreRealm(_namespace, realm);
+    } catch (IllegalArgumentException ex) {
+      return notFound(ex.getMessage());
+    }
+
+    return created();
+  }
+
+  @DELETE
+  @Path("/metadata-store-realms/{realm}")
+  public Response deleteMetadataStoreRealm(@PathParam("realm") String realm) {
+    try {
+      _metadataStoreDirectory.deleteMetadataStoreRealm(_namespace, realm);
+    } catch (IllegalArgumentException ex) {
+      return notFound(ex.getMessage());
+    }
+
+    return OK();
+  }
+
   /**
    * Gets sharding keys mapped at path "HTTP GET /sharding-keys" which returns all sharding
keys in
    * a namespace, or path "HTTP GET /sharding-keys?realm={realmName}" which returns sharding
keys in
@@ -101,15 +127,13 @@ public class MetadataStoreDirectoryAccessor extends AbstractResource
{
       // If realm is not set in query param, the endpoint is: "/sharding-keys"
       // to get all sharding keys in a namespace.
       if (realm == null) {
-        shardingKeys =
-            _metadataStoreDirectory.getAllShardingKeys(_namespace.getName());
+        shardingKeys = _metadataStoreDirectory.getAllShardingKeys(_namespace);
         // To avoid allocating unnecessary resource, limit the map's capacity only for
         // SHARDING_KEYS.
         responseMap = new HashMap<>(1);
       } else {
         // For endpoint: "/sharding-keys?realm={realmName}"
-        shardingKeys = _metadataStoreDirectory
-            .getAllShardingKeysInRealm(_namespace.getName(), realm);
+        shardingKeys = _metadataStoreDirectory.getAllShardingKeysInRealm(_namespace, realm);
         // To avoid allocating unnecessary resource, limit the map's capacity only for
         // SHARDING_KEYS and SINGLE_METADATA_STORE_REALM.
         responseMap = new HashMap<>(2);
@@ -124,7 +148,34 @@ public class MetadataStoreDirectoryAccessor extends AbstractResource
{
     return JSONRepresentation(responseMap);
   }
 
-  private void getHelixNamespace() {
+  @PUT
+  @Path("/metadata-store-realms/{realm}/sharding-keys/{sharding-key: .+}")
+  public Response addShardingKey(@PathParam("realm") String realm,
+      @PathParam("sharding-key") String shardingKey) {
+    try {
+      _metadataStoreDirectory.addShardingKey(_namespace, realm, shardingKey);
+    } catch (IllegalArgumentException ex) {
+      return notFound(ex.getMessage());
+    }
+
+    return created();
+  }
+
+  @DELETE
+  @Path("/metadata-store-realms/{realm}/sharding-keys/{sharding-key: .+}")
+  public Response deleteShardingKey(@PathParam("realm") String realm,
+      @PathParam("sharding-key") String shardingKey) {
+    try {
+      _metadataStoreDirectory.deleteShardingKey(_namespace, realm, shardingKey);
+    } catch (IllegalArgumentException ex) {
+      return notFound(ex.getMessage());
+    }
+
+    return OK();
+  }
+
+  private HelixRestNamespace getHelixNamespace() {
+    HelixRestNamespace helixRestNamespace = null;
     // A default servlet does not have context property key METADATA, so the namespace
     // is retrieved from property ALL_NAMESPACES.
     if (HelixRestUtils.isDefaultServlet(_servletRequest.getServletPath())) {
@@ -134,20 +185,21 @@ public class MetadataStoreDirectoryAccessor extends AbstractResource
{
           .get(ContextPropertyKeys.ALL_NAMESPACES.name());
       for (HelixRestNamespace ns : namespaces) {
         if (HelixRestNamespace.DEFAULT_NAMESPACE_NAME.equals(ns.getName())) {
-          _namespace = ns;
+          helixRestNamespace = ns;
           break;
         }
       }
     } else {
       // Get namespace from property METADATA for a common servlet.
-      _namespace = (HelixRestNamespace) _application.getProperties()
+      helixRestNamespace = (HelixRestNamespace) _application.getProperties()
           .get(ContextPropertyKeys.METADATA.name());
     }
+
+    return helixRestNamespace;
   }
 
-  private void buildMetadataStoreDirectory() {
-    Map<String, String> routingZkAddressMap =
-        ImmutableMap.of(_namespace.getName(), _namespace.getMetadataStoreAddress());
+  private void buildMetadataStoreDirectory(String namespace, String address) {
+    Map<String, String> routingZkAddressMap = ImmutableMap.of(namespace, address);
     try {
       _metadataStoreDirectory = new ZkMetadataStoreDirectory(routingZkAddressMap);
     } catch (InvalidRoutingDataException ex) {
diff --git a/helix-rest/src/test/java/org/apache/helix/rest/metadatastore/accessor/TestZkRoutingDataReader.java
b/helix-rest/src/test/java/org/apache/helix/rest/metadatastore/accessor/TestZkRoutingDataReader.java
index 5781a85..c188840 100644
--- a/helix-rest/src/test/java/org/apache/helix/rest/metadatastore/accessor/TestZkRoutingDataReader.java
+++ b/helix-rest/src/test/java/org/apache/helix/rest/metadatastore/accessor/TestZkRoutingDataReader.java
@@ -25,6 +25,7 @@ import java.util.List;
 import java.util.Map;
 
 import org.apache.helix.AccessOption;
+import org.apache.helix.TestHelper;
 import org.apache.helix.rest.metadatastore.constant.MetadataStoreRoutingConstants;
 import org.apache.helix.rest.metadatastore.exceptions.InvalidRoutingDataException;
 import org.apache.helix.rest.server.AbstractTestClass;
@@ -41,13 +42,15 @@ public class TestZkRoutingDataReader extends AbstractTestClass {
   private MetadataStoreRoutingDataReader _zkRoutingDataReader;
 
   @BeforeClass
-  public void beforeClass() {
+  public void beforeClass() throws Exception {
+    deleteRoutingDataPath();
     _zkRoutingDataReader = new ZkRoutingDataReader(DUMMY_NAMESPACE, ZK_ADDR, null);
   }
 
   @AfterClass
-  public void afterClass() {
+  public void afterClass() throws Exception {
     _zkRoutingDataReader.close();
+    deleteRoutingDataPath();
   }
 
   @AfterMethod
@@ -126,4 +129,18 @@ public class TestZkRoutingDataReader extends AbstractTestClass {
       Assert.fail("Not expecting InvalidRoutingDataException");
     }
   }
+
+  private void deleteRoutingDataPath() throws Exception {
+    Assert.assertTrue(TestHelper.verify(() -> {
+      ZK_SERVER_MAP.get(ZK_ADDR).getZkClient()
+          .deleteRecursively(MetadataStoreRoutingConstants.ROUTING_DATA_PATH);
+
+      if (ZK_SERVER_MAP.get(ZK_ADDR).getZkClient()
+          .exists(MetadataStoreRoutingConstants.ROUTING_DATA_PATH)) {
+        return false;
+      }
+
+      return true;
+    }, TestHelper.WAIT_DURATION), "Routing data path should be deleted after the tests.");
+  }
 }
diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/resources/zookeeper/TestMetadataStoreDirectoryAccessor.java
b/helix-rest/src/test/java/org/apache/helix/rest/server/resources/zookeeper/TestMetadataStoreDirectoryAccessor.java
index 7242c84..c08d845 100644
--- a/helix-rest/src/test/java/org/apache/helix/rest/server/resources/zookeeper/TestMetadataStoreDirectoryAccessor.java
+++ b/helix-rest/src/test/java/org/apache/helix/rest/server/resources/zookeeper/TestMetadataStoreDirectoryAccessor.java
@@ -27,14 +27,21 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import javax.ws.rs.client.Entity;
+import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
 
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
-import org.apache.helix.ZNRecord;
-import org.apache.helix.manager.zk.ZNRecordSerializer;
+import org.apache.helix.TestHelper;
+import org.apache.helix.rest.common.HelixRestNamespace;
+import org.apache.helix.rest.metadatastore.MetadataStoreDirectory;
+import org.apache.helix.rest.metadatastore.ZkMetadataStoreDirectory;
 import org.apache.helix.rest.metadatastore.constant.MetadataStoreRoutingConstants;
 import org.apache.helix.rest.server.AbstractTestClass;
 import org.apache.helix.rest.server.util.JerseyUriRequestBuilder;
+import org.apache.helix.zookeeper.datamodel.ZNRecord;
+import org.apache.helix.zookeeper.datamodel.serializer.ZNRecordSerializer;
 import org.testng.Assert;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
@@ -45,20 +52,33 @@ public class TestMetadataStoreDirectoryAccessor extends AbstractTestClass
{
   /*
    * The following are constants to be used for testing.
    */
+  private static final String TEST_NAMESPACE_URI_PREFIX = "/namespaces/" + TEST_NAMESPACE;
+  private static final String NON_EXISTING_NAMESPACE_URI_PREFIX =
+      "/namespaces/not-existed-namespace/metadata-store-realms/";
   private static final String TEST_REALM_1 = "testRealm1";
   private static final List<String> TEST_SHARDING_KEYS_1 =
       Arrays.asList("/sharding/key/1/a", "/sharding/key/1/b", "/sharding/key/1/c");
   private static final String TEST_REALM_2 = "testRealm2";
   private static final List<String> TEST_SHARDING_KEYS_2 =
       Arrays.asList("/sharding/key/1/d", "/sharding/key/1/e", "/sharding/key/1/f");
+  private static final String TEST_REALM_3 = "testRealm3";
+  private static final String TEST_SHARDING_KEY = "/sharding/key/1/x";
 
   // List of all ZK addresses, each of which corresponds to a namespace/routing ZK
   private List<String> _zkList;
+  private MetadataStoreDirectory _metadataStoreDirectory;
 
   @BeforeClass
-  public void beforeClass() {
+  public void beforeClass() throws Exception {
     _zkList = new ArrayList<>(ZK_SERVER_MAP.keySet());
 
+    deleteRoutingDataPath();
+
+    // Populate routingZkAddrMap according namespaces in helix rest server.
+    // <Namespace, ZkAddr> mapping
+    Map<String, String> routingZkAddrMap = ImmutableMap
+        .of(HelixRestNamespace.DEFAULT_NAMESPACE_NAME, ZK_ADDR, TEST_NAMESPACE, _zkAddrTestNS);
+
     // Write dummy mappings in ZK
     // Create a node that represents a realm address and add 3 sharding keys to it
     ZNRecord znRecord = new ZNRecord("RoutingInfo");
@@ -85,12 +105,18 @@ public class TestMetadataStoreDirectoryAccessor extends AbstractTestClass
{
           .writeData(MetadataStoreRoutingConstants.ROUTING_DATA_PATH + "/" + TEST_REALM_2,
               znRecord);
     });
+
+    // Create metadataStoreDirectory
+    _metadataStoreDirectory = new ZkMetadataStoreDirectory(routingZkAddrMap);
   }
 
   @Test
   public void testGetAllMetadataStoreRealms() throws IOException {
-    String responseBody =
-        get("metadata-store-realms", null, Response.Status.OK.getStatusCode(), true);
+    get(NON_EXISTING_NAMESPACE_URI_PREFIX + "metadata-store-realms", null,
+        Response.Status.NOT_FOUND.getStatusCode(), false);
+
+    String responseBody = get(TEST_NAMESPACE_URI_PREFIX + "/metadata-store-realms", null,
+        Response.Status.OK.getStatusCode(), true);
     // It is safe to cast the object and suppress warnings.
     @SuppressWarnings("unchecked")
     Map<String, Collection<String>> queriedRealmsMap =
@@ -106,12 +132,70 @@ public class TestMetadataStoreDirectoryAccessor extends AbstractTestClass
{
     Assert.assertEquals(queriedRealmsSet, expectedRealms);
   }
 
+  @Test
+  public void testAddMetadataStoreRealm() {
+    Collection<String> previousRealms =
+        _metadataStoreDirectory.getAllMetadataStoreRealms(TEST_NAMESPACE);
+    Set<String> expectedRealmsSet = new HashSet<>(previousRealms);
+
+    Assert.assertFalse(expectedRealmsSet.contains(TEST_REALM_3),
+        "Metadata store directory should not have realm: " + TEST_REALM_3);
+
+    // Test a request that has not found response.
+    put(NON_EXISTING_NAMESPACE_URI_PREFIX + TEST_REALM_3, null,
+        Entity.entity("", MediaType.APPLICATION_JSON_TYPE),
+        Response.Status.NOT_FOUND.getStatusCode());
+
+    // Successful request.
+    put(TEST_NAMESPACE_URI_PREFIX + "/metadata-store-realms/" + TEST_REALM_3, null,
+        Entity.entity("", MediaType.APPLICATION_JSON_TYPE),
+        Response.Status.CREATED.getStatusCode());
+
+    Collection<String> updatedRealms =
+        _metadataStoreDirectory.getAllMetadataStoreRealms(TEST_NAMESPACE);
+    Set<String> updateRealmsSet = new HashSet<>(updatedRealms);
+    expectedRealmsSet.add(TEST_REALM_3);
+
+    // TODO: enable asserts and add verify for refreshed MSD once write operations are ready.
+//    Assert.assertEquals(updateRealmsSet, previousRealms);
+  }
+
+  @Test(dependsOnMethods = "testAddMetadataStoreRealm")
+  public void testDeleteMetadataStoreRealm() {
+    Collection<String> previousRealms =
+        _metadataStoreDirectory.getAllMetadataStoreRealms(TEST_NAMESPACE);
+    Set<String> expectedRealmsSet = new HashSet<>(previousRealms);
+
+//    Assert.assertTrue(expectedRealmsSet.contains(TEST_REALM_3),
+//        "Metadata store directory should have realm: " + TEST_REALM_3);
+
+    // Test a request that has not found response.
+    delete(NON_EXISTING_NAMESPACE_URI_PREFIX + TEST_REALM_3,
+        Response.Status.NOT_FOUND.getStatusCode());
+
+    // Successful request.
+    delete(TEST_NAMESPACE_URI_PREFIX + "/metadata-store-realms/" + TEST_REALM_3,
+        Response.Status.OK.getStatusCode());
+
+    Collection<String> updatedRealms =
+        _metadataStoreDirectory.getAllMetadataStoreRealms(TEST_NAMESPACE);
+    Set<String> updateRealmsSet = new HashSet<>(updatedRealms);
+    expectedRealmsSet.remove(TEST_REALM_3);
+
+//    Assert.assertEquals(updateRealmsSet, previousRealms);
+  }
+
   /*
    * Tests REST endpoints: "/sharding-keys"
    */
   @Test
   public void testGetShardingKeysInNamespace() throws IOException {
-    String responseBody = get("/sharding-keys", null, Response.Status.OK.getStatusCode(),
true);
+    get(NON_EXISTING_NAMESPACE_URI_PREFIX + "sharding-keys", null,
+        Response.Status.NOT_FOUND.getStatusCode(), true);
+
+    String responseBody =
+        get(TEST_NAMESPACE_URI_PREFIX + "/sharding-keys", null, Response.Status.OK.getStatusCode(),
+            true);
     // It is safe to cast the object and suppress warnings.
     @SuppressWarnings("unchecked")
     Map<String, Collection<String>> queriedShardingKeysMap =
@@ -135,15 +219,16 @@ public class TestMetadataStoreDirectoryAccessor extends AbstractTestClass
{
   @Test
   public void testGetShardingKeysInRealm() throws IOException {
     // Test NOT_FOUND response for a non existed realm.
-    new JerseyUriRequestBuilder("/sharding-keys?realm=nonExistedRealm")
+    new JerseyUriRequestBuilder(TEST_NAMESPACE_URI_PREFIX + "/sharding-keys?realm=nonExistedRealm")
         .expectedReturnStatusCode(Response.Status.NOT_FOUND.getStatusCode()).get(this);
 
     // Query param realm is set to empty, so NOT_FOUND response is returned.
-    new JerseyUriRequestBuilder("/sharding-keys?realm=")
+    new JerseyUriRequestBuilder(TEST_NAMESPACE_URI_PREFIX + "/sharding-keys?realm=")
         .expectedReturnStatusCode(Response.Status.NOT_FOUND.getStatusCode()).get(this);
 
     // Success response.
-    String responseBody = new JerseyUriRequestBuilder("/sharding-keys?realm=" + TEST_REALM_1)
+    String responseBody = new JerseyUriRequestBuilder(
+        TEST_NAMESPACE_URI_PREFIX + "/sharding-keys?realm=" + TEST_REALM_1)
         .isBodyReturnExpected(true).get(this);
     // It is safe to cast the object and suppress warnings.
     @SuppressWarnings("unchecked")
@@ -167,9 +252,73 @@ public class TestMetadataStoreDirectoryAccessor extends AbstractTestClass
{
     Assert.assertEquals(queriedShardingKeys, expectedShardingKeys);
   }
 
+  @Test
+  public void testAddShardingKey() {
+    Set<String> expectedShardingKeysSet = new HashSet<>(
+        _metadataStoreDirectory.getAllShardingKeysInRealm(TEST_NAMESPACE, TEST_REALM_1));
+
+    Assert.assertFalse(expectedShardingKeysSet.contains(TEST_SHARDING_KEY),
+        "Realm does not have sharding key: " + TEST_SHARDING_KEY);
+
+    // Request that gets not found response.
+    put(NON_EXISTING_NAMESPACE_URI_PREFIX + TEST_REALM_1 + "/sharding-keys/" + TEST_SHARDING_KEY,
+        null, Entity.entity("", MediaType.APPLICATION_JSON_TYPE),
+        Response.Status.NOT_FOUND.getStatusCode());
+
+    // Successful request.
+    put(TEST_NAMESPACE_URI_PREFIX + "/metadata-store-realms/" + TEST_REALM_1 + "/sharding-keys/"
+            + TEST_SHARDING_KEY, null, Entity.entity("", MediaType.APPLICATION_JSON_TYPE),
+        Response.Status.CREATED.getStatusCode());
+
+    Set<String> updatedShardingKeysSet = new HashSet<>(
+        _metadataStoreDirectory.getAllShardingKeysInRealm(TEST_NAMESPACE, TEST_REALM_1));
+    expectedShardingKeysSet.add(TEST_SHARDING_KEY);
+
+//    Assert.assertEquals(updatedShardingKeysSet, expectedShardingKeysSet);
+  }
+
+  @Test(dependsOnMethods = "testAddShardingKey")
+  public void testDeleteShardingKey() {
+    Set<String> expectedShardingKeysSet = new HashSet<>(
+        _metadataStoreDirectory.getAllShardingKeysInRealm(TEST_NAMESPACE, TEST_REALM_1));
+
+//    Assert.assertTrue(expectedShardingKeysSet.contains(TEST_SHARDING_KEY),
+//        "Realm should have sharding key: " + TEST_SHARDING_KEY);
+
+    // Request that gets not found response.
+    delete(NON_EXISTING_NAMESPACE_URI_PREFIX + TEST_REALM_1 + "/sharding-keys/" + TEST_SHARDING_KEY,
+        Response.Status.NOT_FOUND.getStatusCode());
+
+    // Successful request.
+    delete(TEST_NAMESPACE_URI_PREFIX + "/metadata-store-realms/" + TEST_REALM_1 + "/sharding-keys/"
+        + TEST_SHARDING_KEY, Response.Status.OK.getStatusCode());
+
+    Set<String> updatedShardingKeysSet = new HashSet<>(
+        _metadataStoreDirectory.getAllShardingKeysInRealm(TEST_NAMESPACE, TEST_REALM_1));
+    expectedShardingKeysSet.remove(TEST_SHARDING_KEY);
+
+//    Assert.assertEquals(updatedShardingKeysSet, expectedShardingKeysSet);
+  }
+
   @AfterClass
-  public void afterClass() {
-    _zkList.forEach(zk -> ZK_SERVER_MAP.get(zk).getZkClient()
-        .deleteRecursive(MetadataStoreRoutingConstants.ROUTING_DATA_PATH));
+  public void afterClass() throws Exception {
+    _metadataStoreDirectory.close();
+    deleteRoutingDataPath();
+  }
+
+  private void deleteRoutingDataPath() throws Exception {
+    Assert.assertTrue(TestHelper.verify(() -> {
+      _zkList.forEach(zk -> ZK_SERVER_MAP.get(zk).getZkClient()
+          .deleteRecursively(MetadataStoreRoutingConstants.ROUTING_DATA_PATH));
+
+      for (String zk : _zkList) {
+        if (ZK_SERVER_MAP.get(zk).getZkClient()
+            .exists(MetadataStoreRoutingConstants.ROUTING_DATA_PATH)) {
+          return false;
+        }
+      }
+
+      return true;
+    }, TestHelper.WAIT_DURATION), "Routing data path should be deleted after the tests.");
   }
 }


Mime
View raw message