helix-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hu...@apache.org
Subject [helix] branch zooscalability updated: Add REST read endpoints to Helix Rest to provide resource access to metadata store directory (#744)
Date Wed, 12 Feb 2020 02:09:32 GMT
This is an automated email from the ASF dual-hosted git repository.

hulee pushed a commit to branch zooscalability
in repository https://gitbox.apache.org/repos/asf/helix.git


The following commit(s) were added to refs/heads/zooscalability by this push:
     new d750b09  Add REST read endpoints to Helix Rest to provide resource access to metadata
store directory (#744)
d750b09 is described below

commit d750b09622dfd73adb7bceaaa82349d9a7151135
Author: Huizhi Lu <hulu@linkedin.com>
AuthorDate: Tue Feb 11 18:09:25 2020 -0800

    Add REST read endpoints to Helix Rest to provide resource access to metadata store directory
(#744)
    
    Change list:
    - Add read endpoints to read sharding keys and realms
    - Add 3 unit tests to test the new endpoints in TestMetadataStoreDirectoryAccessor
---
 .../org/apache/helix/rest/common/ServletType.java  |   7 +-
 .../constant/MetadataStoreRoutingConstants.java    |  11 ++
 .../MetadataStoreDirectoryAccessor.java            | 160 +++++++++++++++++++
 .../TestMetadataStoreDirectoryAccessor.java        | 175 +++++++++++++++++++++
 4 files changed, 351 insertions(+), 2 deletions(-)

diff --git a/helix-rest/src/main/java/org/apache/helix/rest/common/ServletType.java b/helix-rest/src/main/java/org/apache/helix/rest/common/ServletType.java
index f068f95..7c77429 100644
--- a/helix-rest/src/main/java/org/apache/helix/rest/common/ServletType.java
+++ b/helix-rest/src/main/java/org/apache/helix/rest/common/ServletType.java
@@ -21,6 +21,7 @@ package org.apache.helix.rest.common;
 
 import org.apache.helix.rest.server.resources.helix.AbstractHelixResource;
 import org.apache.helix.rest.server.resources.metadata.NamespacesAccessor;
+import org.apache.helix.rest.server.resources.metadatastore.MetadataStoreDirectoryAccessor;
 import org.apache.helix.rest.server.resources.zookeeper.ZooKeeperAccessor;
 
 
@@ -32,7 +33,8 @@ public enum ServletType {
       new String[] {
           AbstractHelixResource.class.getPackage().getName(),
           NamespacesAccessor.class.getPackage().getName(),
-          ZooKeeperAccessor.class.getPackage().getName()
+          ZooKeeperAccessor.class.getPackage().getName(),
+          MetadataStoreDirectoryAccessor.class.getPackage().getName()
       }),
 
   /**
@@ -41,7 +43,8 @@ public enum ServletType {
   COMMON_SERVLET("/namespaces/%s/*",
       new String[] {
           AbstractHelixResource.class.getPackage().getName(),
-          ZooKeeperAccessor.class.getPackage().getName()
+          ZooKeeperAccessor.class.getPackage().getName(),
+          MetadataStoreDirectoryAccessor.class.getPackage().getName()
       });
 
   private final String _servletPathSpecTemplate;
diff --git a/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/constant/MetadataStoreRoutingConstants.java
b/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/constant/MetadataStoreRoutingConstants.java
index e4240e7..846aa30 100644
--- a/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/constant/MetadataStoreRoutingConstants.java
+++ b/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/constant/MetadataStoreRoutingConstants.java
@@ -27,4 +27,15 @@ public class MetadataStoreRoutingConstants {
 
   // Leader election ZNode for ZkRoutingDataWriter
   public static final String LEADER_ELECTION_ZNODE = "/_ZK_ROUTING_DATA_WRITER_LEADER";
+
+  // Field name in JSON REST response of getting metadata store realms in one namespace.
+  public static final String METADATA_STORE_REALMS = "metadataStoreRealms";
+
+  // Field name in JSON REST response of getting sharding keys in one realm.
+  public static final String SINGLE_METADATA_STORE_REALM = "metadataStoreRealm";
+
+  // Field name in JSON REST response of getting sharding keys.
+  public static final String SHARDING_KEYS = "shardingKeys";
+
+
 }
diff --git a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/metadatastore/MetadataStoreDirectoryAccessor.java
b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/metadatastore/MetadataStoreDirectoryAccessor.java
new file mode 100644
index 0000000..78543d6
--- /dev/null
+++ b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/metadatastore/MetadataStoreDirectoryAccessor.java
@@ -0,0 +1,160 @@
+package org.apache.helix.rest.server.resources.metadatastore;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import javax.annotation.PostConstruct;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Response;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.helix.rest.common.ContextPropertyKeys;
+import org.apache.helix.rest.common.HelixRestNamespace;
+import org.apache.helix.rest.common.HelixRestUtils;
+import org.apache.helix.rest.metadatastore.MetadataStoreDirectory;
+import org.apache.helix.rest.metadatastore.ZkMetadataStoreDirectory;
+import org.apache.helix.rest.metadatastore.constant.MetadataStoreRoutingConstants;
+import org.apache.helix.rest.metadatastore.exceptions.InvalidRoutingDataException;
+import org.apache.helix.rest.server.resources.AbstractResource;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * Provides REST endpoints for accessing metadata store directory service,
+ * which responds to read/write requests of metadata store realms, sharding keys, etc..
+ */
+@Path("")
+public class MetadataStoreDirectoryAccessor extends AbstractResource {
+  private static final Logger LOG = LoggerFactory.getLogger(MetadataStoreDirectoryAccessor.class);
+
+  private HelixRestNamespace _namespace;
+
+  // Double-checked locking for thread-safe object.
+  private MetadataStoreDirectory _metadataStoreDirectory;
+
+  @PostConstruct
+  private void postConstruct() {
+    getHelixNamespace();
+    buildMetadataStoreDirectory();
+  }
+
+  /**
+   * Gets all metadata store realms in a namespace with the endpoint.
+   *
+   * @return Json representation of all realms.
+   */
+  @GET
+  @Path("/metadata-store-realms")
+  public Response getAllMetadataStoreRealms() {
+    Map<String, Collection<String>> responseMap;
+    try {
+      Collection<String> realms =
+          _metadataStoreDirectory.getAllMetadataStoreRealms(_namespace.getName());
+
+      responseMap = new HashMap<>(1);
+      responseMap.put(MetadataStoreRoutingConstants.METADATA_STORE_REALMS, realms);
+    } catch (NoSuchElementException ex) {
+      return notFound(ex.getMessage());
+    }
+
+    return JSONRepresentation(responseMap);
+  }
+
+  /**
+   * Gets sharding keys mapped at path "HTTP GET /sharding-keys" which returns all sharding
keys in
+   * a namespace, or path "HTTP GET /sharding-keys?realm={realmName}" which returns sharding
keys in
+   * a realm.
+   *
+   * @param realm Query param in endpoint path
+   * @return Json representation of a map: shardingKeys -> collection of sharding keys.
+   */
+  @GET
+  @Path("/sharding-keys")
+  public Response getShardingKeys(@QueryParam("realm") String realm) {
+    Map<String, Object> responseMap;
+    Collection<String> shardingKeys;
+    try {
+      // If realm is not set in query param, the endpoint is: "/sharding-keys"
+      // to get all sharding keys in a namespace.
+      if (realm == null) {
+        shardingKeys =
+            _metadataStoreDirectory.getAllShardingKeys(_namespace.getName());
+        // To avoid allocating unnecessary resource, limit the map's capacity only for
+        // SHARDING_KEYS.
+        responseMap = new HashMap<>(1);
+      } else {
+        // For endpoint: "/sharding-keys?realm={realmName}"
+        shardingKeys = _metadataStoreDirectory
+            .getAllShardingKeysInRealm(_namespace.getName(), realm);
+        // To avoid allocating unnecessary resource, limit the map's capacity only for
+        // SHARDING_KEYS and SINGLE_METADATA_STORE_REALM.
+        responseMap = new HashMap<>(2);
+        responseMap.put(MetadataStoreRoutingConstants.SINGLE_METADATA_STORE_REALM, realm);
+      }
+    } catch (NoSuchElementException ex) {
+      return notFound(ex.getMessage());
+    }
+
+    responseMap.put(MetadataStoreRoutingConstants.SHARDING_KEYS, shardingKeys);
+
+    return JSONRepresentation(responseMap);
+  }
+
+  private void getHelixNamespace() {
+    // A default servlet does not have context property key METADATA, so the namespace
+    // is retrieved from property ALL_NAMESPACES.
+    if (HelixRestUtils.isDefaultServlet(_servletRequest.getServletPath())) {
+      // It is safe to ignore uncheck warnings for this cast.
+      @SuppressWarnings("unchecked")
+      List<HelixRestNamespace> namespaces = (List<HelixRestNamespace>) _application.getProperties()
+          .get(ContextPropertyKeys.ALL_NAMESPACES.name());
+      for (HelixRestNamespace ns : namespaces) {
+        if (HelixRestNamespace.DEFAULT_NAMESPACE_NAME.equals(ns.getName())) {
+          _namespace = ns;
+          break;
+        }
+      }
+    } else {
+      // Get namespace from property METADATA for a common servlet.
+      _namespace = (HelixRestNamespace) _application.getProperties()
+          .get(ContextPropertyKeys.METADATA.name());
+    }
+  }
+
+  private void buildMetadataStoreDirectory() {
+    Map<String, String> routingZkAddressMap =
+        ImmutableMap.of(_namespace.getName(), _namespace.getMetadataStoreAddress());
+    try {
+      _metadataStoreDirectory = new ZkMetadataStoreDirectory(routingZkAddressMap);
+    } catch (InvalidRoutingDataException ex) {
+      // In this case, the InvalidRoutingDataException should not happen because routing
+      // ZK address is always valid here.
+      LOG.warn("Unable to create metadata store directory for routing ZK address: {}",
+          routingZkAddressMap, ex);
+    }
+  }
+}
diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/resources/zookeeper/TestMetadataStoreDirectoryAccessor.java
b/helix-rest/src/test/java/org/apache/helix/rest/server/resources/zookeeper/TestMetadataStoreDirectoryAccessor.java
new file mode 100644
index 0000000..7242c84
--- /dev/null
+++ b/helix-rest/src/test/java/org/apache/helix/rest/server/resources/zookeeper/TestMetadataStoreDirectoryAccessor.java
@@ -0,0 +1,175 @@
+package org.apache.helix.rest.server.resources.zookeeper;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import javax.ws.rs.core.Response;
+
+import com.google.common.collect.ImmutableSet;
+import org.apache.helix.ZNRecord;
+import org.apache.helix.manager.zk.ZNRecordSerializer;
+import org.apache.helix.rest.metadatastore.constant.MetadataStoreRoutingConstants;
+import org.apache.helix.rest.server.AbstractTestClass;
+import org.apache.helix.rest.server.util.JerseyUriRequestBuilder;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+
+public class TestMetadataStoreDirectoryAccessor extends AbstractTestClass {
+  /*
+   * The following are constants to be used for testing.
+   */
+  private static final String TEST_REALM_1 = "testRealm1";
+  private static final List<String> TEST_SHARDING_KEYS_1 =
+      Arrays.asList("/sharding/key/1/a", "/sharding/key/1/b", "/sharding/key/1/c");
+  private static final String TEST_REALM_2 = "testRealm2";
+  private static final List<String> TEST_SHARDING_KEYS_2 =
+      Arrays.asList("/sharding/key/1/d", "/sharding/key/1/e", "/sharding/key/1/f");
+
+  // List of all ZK addresses, each of which corresponds to a namespace/routing ZK
+  private List<String> _zkList;
+
+  @BeforeClass
+  public void beforeClass() {
+    _zkList = new ArrayList<>(ZK_SERVER_MAP.keySet());
+
+    // Write dummy mappings in ZK
+    // Create a node that represents a realm address and add 3 sharding keys to it
+    ZNRecord znRecord = new ZNRecord("RoutingInfo");
+
+    _zkList.forEach(zk -> {
+      ZK_SERVER_MAP.get(zk).getZkClient().setZkSerializer(new ZNRecordSerializer());
+      // Write first realm and sharding keys pair
+      znRecord.setListField(MetadataStoreRoutingConstants.ZNRECORD_LIST_FIELD_KEY,
+          TEST_SHARDING_KEYS_1);
+      ZK_SERVER_MAP.get(zk).getZkClient()
+          .createPersistent(MetadataStoreRoutingConstants.ROUTING_DATA_PATH + "/" + TEST_REALM_1,
+              true);
+      ZK_SERVER_MAP.get(zk).getZkClient()
+          .writeData(MetadataStoreRoutingConstants.ROUTING_DATA_PATH + "/" + TEST_REALM_1,
+              znRecord);
+
+      // Create another realm and sharding keys pair
+      znRecord.setListField(MetadataStoreRoutingConstants.ZNRECORD_LIST_FIELD_KEY,
+          TEST_SHARDING_KEYS_2);
+      ZK_SERVER_MAP.get(zk).getZkClient()
+          .createPersistent(MetadataStoreRoutingConstants.ROUTING_DATA_PATH + "/" + TEST_REALM_2,
+              true);
+      ZK_SERVER_MAP.get(zk).getZkClient()
+          .writeData(MetadataStoreRoutingConstants.ROUTING_DATA_PATH + "/" + TEST_REALM_2,
+              znRecord);
+    });
+  }
+
+  @Test
+  public void testGetAllMetadataStoreRealms() throws IOException {
+    String responseBody =
+        get("metadata-store-realms", null, Response.Status.OK.getStatusCode(), true);
+    // It is safe to cast the object and suppress warnings.
+    @SuppressWarnings("unchecked")
+    Map<String, Collection<String>> queriedRealmsMap =
+        OBJECT_MAPPER.readValue(responseBody, Map.class);
+
+    Assert.assertTrue(
+        queriedRealmsMap.containsKey(MetadataStoreRoutingConstants.METADATA_STORE_REALMS));
+
+    Set<String> queriedRealmsSet =
+        new HashSet<>(queriedRealmsMap.get(MetadataStoreRoutingConstants.METADATA_STORE_REALMS));
+    Set<String> expectedRealms = ImmutableSet.of(TEST_REALM_1, TEST_REALM_2);
+
+    Assert.assertEquals(queriedRealmsSet, expectedRealms);
+  }
+
+  /*
+   * Tests REST endpoints: "/sharding-keys"
+   */
+  @Test
+  public void testGetShardingKeysInNamespace() throws IOException {
+    String responseBody = get("/sharding-keys", null, Response.Status.OK.getStatusCode(),
true);
+    // It is safe to cast the object and suppress warnings.
+    @SuppressWarnings("unchecked")
+    Map<String, Collection<String>> queriedShardingKeysMap =
+        OBJECT_MAPPER.readValue(responseBody, Map.class);
+
+    Assert.assertTrue(
+        queriedShardingKeysMap.containsKey(MetadataStoreRoutingConstants.SHARDING_KEYS));
+
+    Set<String> queriedShardingKeys =
+        new HashSet<>(queriedShardingKeysMap.get(MetadataStoreRoutingConstants.SHARDING_KEYS));
+    Set<String> expectedShardingKeys = new HashSet<>();
+    expectedShardingKeys.addAll(TEST_SHARDING_KEYS_1);
+    expectedShardingKeys.addAll(TEST_SHARDING_KEYS_2);
+
+    Assert.assertEquals(queriedShardingKeys, expectedShardingKeys);
+  }
+
+  /*
+   * Tests REST endpoint: "/sharding-keys?realm={realmName}"
+   */
+  @Test
+  public void testGetShardingKeysInRealm() throws IOException {
+    // Test NOT_FOUND response for a non existed realm.
+    new JerseyUriRequestBuilder("/sharding-keys?realm=nonExistedRealm")
+        .expectedReturnStatusCode(Response.Status.NOT_FOUND.getStatusCode()).get(this);
+
+    // Query param realm is set to empty, so NOT_FOUND response is returned.
+    new JerseyUriRequestBuilder("/sharding-keys?realm=")
+        .expectedReturnStatusCode(Response.Status.NOT_FOUND.getStatusCode()).get(this);
+
+    // Success response.
+    String responseBody = new JerseyUriRequestBuilder("/sharding-keys?realm=" + TEST_REALM_1)
+        .isBodyReturnExpected(true).get(this);
+    // It is safe to cast the object and suppress warnings.
+    @SuppressWarnings("unchecked")
+    Map<String, Object> queriedShardingKeysMap = OBJECT_MAPPER.readValue(responseBody,
Map.class);
+
+    // Check realm name in json response.
+    Assert.assertTrue(queriedShardingKeysMap
+        .containsKey(MetadataStoreRoutingConstants.SINGLE_METADATA_STORE_REALM));
+    Assert.assertEquals(
+        queriedShardingKeysMap.get(MetadataStoreRoutingConstants.SINGLE_METADATA_STORE_REALM),
+        TEST_REALM_1);
+    Assert.assertTrue(
+        queriedShardingKeysMap.containsKey(MetadataStoreRoutingConstants.SHARDING_KEYS));
+
+    // It is safe to cast the object and suppress warnings.
+    @SuppressWarnings("unchecked")
+    Set<String> queriedShardingKeys = new HashSet<>((Collection<String>)
queriedShardingKeysMap
+        .get(MetadataStoreRoutingConstants.SHARDING_KEYS));
+    Set<String> expectedShardingKeys = new HashSet<>(TEST_SHARDING_KEYS_1);
+
+    Assert.assertEquals(queriedShardingKeys, expectedShardingKeys);
+  }
+
+  @AfterClass
+  public void afterClass() {
+    _zkList.forEach(zk -> ZK_SERVER_MAP.get(zk).getZkClient()
+        .deleteRecursive(MetadataStoreRoutingConstants.ROUTING_DATA_PATH));
+  }
+}


Mime
View raw message