helix-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jiajunw...@apache.org
Subject [helix] branch customizeView updated: Replace customized view cache with property cache (#869)
Date Mon, 30 Mar 2020 23:47:20 GMT
This is an automated email from the ASF dual-hosted git repository.

jiajunwang pushed a commit to branch customizeView
in repository https://gitbox.apache.org/repos/asf/helix.git


The following commit(s) were added to refs/heads/customizeView by this push:
     new e367004  Replace customized view cache with property cache (#869)
e367004 is described below

commit e367004175ab04887657eff0864b7f188aeda5ff
Author: Ali Reza Zamani Zadeh Najari <anajari@linkedin.com>
AuthorDate: Mon Mar 30 16:47:13 2020 -0700

    Replace customized view cache with property cache (#869)
    
    In this commit, the custom implementation of customized view cache
    has been replaced with property cache implementation.
---
 .../helix/common/caches/CustomizedViewCache.java   | 116 +++++----------------
 .../spectator/TestRoutingTableProvider.java        |  69 ++++++++++--
 2 files changed, 86 insertions(+), 99 deletions(-)

diff --git a/helix-core/src/main/java/org/apache/helix/common/caches/CustomizedViewCache.java
b/helix-core/src/main/java/org/apache/helix/common/caches/CustomizedViewCache.java
index d46be64..cdbc0f8 100644
--- a/helix-core/src/main/java/org/apache/helix/common/caches/CustomizedViewCache.java
+++ b/helix-core/src/main/java/org/apache/helix/common/caches/CustomizedViewCache.java
@@ -19,23 +19,16 @@ package org.apache.helix.common.caches;
  * under the License.
  */
 
-import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
 import java.util.Map;
-import java.util.Set;
 
 import org.apache.helix.HelixDataAccessor;
-import org.apache.helix.HelixException;
 import org.apache.helix.PropertyKey;
 import org.apache.helix.PropertyType;
 import org.apache.helix.model.CustomizedView;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.common.collect.Maps;
 
 /**
  * Cache to hold all CustomizedView of a specific type.
@@ -43,8 +36,7 @@ import com.google.common.collect.Maps;
 public class CustomizedViewCache extends AbstractDataCache<CustomizedView> {
   private static final Logger LOG = LoggerFactory.getLogger(CustomizedViewCache.class.getName());
 
-  protected Map<String, CustomizedView> _customizedViewMap;
-  protected Map<String, CustomizedView> _customizedViewCache;
+  private final PropertyCache<CustomizedView> _customizedViewCache;
   protected String _clusterName;
   private PropertyType _propertyType;
   private String _customizedStateType;
@@ -54,13 +46,27 @@ public class CustomizedViewCache extends AbstractDataCache<CustomizedView>
{
   }
 
   protected CustomizedViewCache(String clusterName, PropertyType propertyType, String customizedStateType)
{
-    super(createDefaultControlContextProvider(clusterName));
-    _clusterName = clusterName;
-    _customizedViewMap = Collections.emptyMap();
-    _customizedViewCache = Collections.emptyMap();
-    _propertyType = propertyType;
-    _customizedStateType = customizedStateType;
-  }
+      super(createDefaultControlContextProvider(clusterName));
+      _clusterName = clusterName;
+      _propertyType = propertyType;
+      _customizedStateType = customizedStateType;
+      _customizedViewCache = new PropertyCache<>(AbstractDataCache.createDefaultControlContextProvider(clusterName),
"CustomizedView", new PropertyCache.PropertyCacheKeyFuncs<CustomizedView>() {
+        @Override
+        public PropertyKey getRootKey(HelixDataAccessor accessor) {
+          return accessor.keyBuilder().customizedView(_customizedStateType);
+        }
+
+        @Override
+        public PropertyKey getObjPropertyKey(HelixDataAccessor accessor, String objName)
{
+          return accessor.keyBuilder().customizedView(_customizedStateType, objName);
+        }
+
+        @Override
+        public String getObjName(CustomizedView obj) {
+          return obj.getResourceName();
+        }
+      }, true);
+    }
 
 
   /**
@@ -70,65 +76,7 @@ public class CustomizedViewCache extends AbstractDataCache<CustomizedView>
{
    * @return
    */
   public void refresh(HelixDataAccessor accessor) {
-    long startTime = System.currentTimeMillis();
-    PropertyKey.Builder keyBuilder = accessor.keyBuilder();
-    Set<PropertyKey> currentPropertyKeys = new HashSet<>();
-
-    List<String> resources = accessor.getChildNames(customizedViewsKey(keyBuilder));
-
-    for (String resource : resources) {
-      currentPropertyKeys.add(customizedViewKey(keyBuilder, resource));
-    }
-
-    Set<PropertyKey> cachedKeys = new HashSet<>();
-    Map<PropertyKey, CustomizedView> cachedCustomizedViewMap = Maps.newHashMap();
-    for (String resource : _customizedViewCache.keySet()) {
-      PropertyKey key = customizedViewKey(keyBuilder, resource);
-      cachedKeys.add(key);
-      cachedCustomizedViewMap.put(key, _customizedViewCache.get(resource));
-    }
-    cachedKeys.retainAll(currentPropertyKeys);
-
-    Set<PropertyKey> reloadKeys = new HashSet<>(currentPropertyKeys);
-    reloadKeys.removeAll(cachedKeys);
-
-    Map<PropertyKey, CustomizedView> updatedMap =
-        refreshProperties(accessor, reloadKeys, new ArrayList<>(cachedKeys),
-            cachedCustomizedViewMap, new HashSet<>());
-
-    Map<String, CustomizedView> newCustomizedViewMap = Maps.newHashMap();
-    for (CustomizedView customizedView : updatedMap.values()) {
-      newCustomizedViewMap.put(customizedView.getResourceName(), customizedView);
-    }
-
-    _customizedViewCache = new HashMap<>(newCustomizedViewMap);
-    _customizedViewMap = new HashMap<>(newCustomizedViewMap);
-
-    long endTime = System.currentTimeMillis();
-    LOG.info("Refresh " + _customizedViewMap.size() + " CustomizedViews of type " + _customizedStateType
-        + " for cluster " + _clusterName + ", took " + (endTime - startTime) + " ms");
-  }
-
-  private PropertyKey customizedViewsKey(PropertyKey.Builder keyBuilder) {
-    PropertyKey customizedViewPropertyKey;
-    if (_propertyType.equals(PropertyType.CUSTOMIZEDVIEW)){
-      customizedViewPropertyKey = keyBuilder.customizedView(_customizedStateType);
-    } else {
-      throw new HelixException(
-          "Failed to refresh CustomizedViewCache, Wrong property type " + _propertyType +
"!");
-    }
-    return customizedViewPropertyKey;
-  }
-
-  private PropertyKey customizedViewKey(PropertyKey.Builder keyBuilder, String resource)
{
-    PropertyKey customizedViewPropertyKey;
-    if (_propertyType.equals(PropertyType.CUSTOMIZEDVIEW)) {
-      customizedViewPropertyKey = keyBuilder.customizedView(_customizedStateType, resource);
-    } else {
-      throw new HelixException(
-          "Failed to refresh CustomizedViewCache, Wrong property type " + _propertyType +
"!");
-    }
-    return customizedViewPropertyKey;
+    _customizedViewCache.refresh(accessor);
   }
 
   /**
@@ -136,22 +84,6 @@ public class CustomizedViewCache extends AbstractDataCache<CustomizedView>
{
    * @return
    */
   public Map<String, CustomizedView> getCustomizedViewMap() {
-    return Collections.unmodifiableMap(_customizedViewMap);
-  }
-
-  /**
-   * Remove dead customized views from map
-   * @param resourceNames
-   */
-
-  public synchronized void removeCustomizedView(List<String> resourceNames) {
-    for (String resourceName : resourceNames) {
-      _customizedViewCache.remove(resourceName);
-    }
-  }
-
-  public void clear() {
-    _customizedViewCache.clear();
-    _customizedViewMap.clear();
+    return Collections.unmodifiableMap(_customizedViewCache.getPropertyMap());
   }
 }
diff --git a/helix-core/src/test/java/org/apache/helix/integration/spectator/TestRoutingTableProvider.java
b/helix-core/src/test/java/org/apache/helix/integration/spectator/TestRoutingTableProvider.java
index d1a2b4c..43ccdc9 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/spectator/TestRoutingTableProvider.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/spectator/TestRoutingTableProvider.java
@@ -1,5 +1,24 @@
 package org.apache.helix.integration.spectator;
 
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -47,6 +66,7 @@ public class TestRoutingTableProvider extends ZkTestBase {
   static final String CLUSTER_NAME = CLUSTER_PREFIX + "_" + CLASS_NAME;
   static final int PARTICIPANT_NUMBER = 3;
   static final int PARTICIPANT_START_PORT = 12918;
+  static final long WAIT_DURATION = 5 * 1000L; // 5 seconds
 
   static final int PARTITION_NUMBER = 20;
   static final int REPLICA_NUMBER = 3;
@@ -269,7 +289,7 @@ public class TestRoutingTableProvider extends ZkTestBase {
         customizedView.getRecord(), AccessOption.PERSISTENT);
 
     boolean onCustomizedViewChangeCalled =
-        TestHelper.verify(() -> customizedViewChangeCalled.get(), TestHelper.WAIT_DURATION);
+        TestHelper.verify(() -> customizedViewChangeCalled.get(), WAIT_DURATION);
     Assert.assertTrue(onCustomizedViewChangeCalled);
 
     _spectator.getHelixDataAccessor().getBaseDataAccessor().remove(customizedViewPath,
@@ -278,7 +298,7 @@ public class TestRoutingTableProvider extends ZkTestBase {
   }
 
   @Test(dependsOnMethods = "testCustomizedViewCorrectConstructor")
-  public void testGetRoutingTableSnapshot() {
+  public void testGetRoutingTableSnapshot() throws Exception {
     Map<PropertyType, List<String>> sourceDataTypes = new HashMap<>();
     sourceDataTypes.put(PropertyType.CUSTOMIZEDVIEW, Arrays.asList("typeA", "typeB"));
     sourceDataTypes.put(PropertyType.EXTERNALVIEW, Collections.emptyList());
@@ -308,19 +328,54 @@ public class TestRoutingTableProvider extends ZkTestBase {
         routingTableProvider.getRoutingTableSnapshot(PropertyType.CUSTOMIZEDVIEW, "typeA");
     Assert.assertEquals(routingTableSnapshot.getPropertyType(), PropertyType.CUSTOMIZEDVIEW);
     Assert.assertEquals(routingTableSnapshot.getCustomizedStateType(), "typeA");
+
     routingTableSnapshot =
         routingTableProvider.getRoutingTableSnapshot(PropertyType.CUSTOMIZEDVIEW, "typeB");
     Assert.assertEquals(routingTableSnapshot.getPropertyType(), PropertyType.CUSTOMIZEDVIEW);
     Assert.assertEquals(routingTableSnapshot.getCustomizedStateType(), "typeB");
 
-    Map<String, Map<String, RoutingTableSnapshot>> routingTableSnapshots =
-        routingTableProvider.getRoutingTableSnapshots();
-    Assert.assertEquals(routingTableSnapshots.size(), 2);
-    Assert.assertEquals(routingTableSnapshots.get(PropertyType.CUSTOMIZEDVIEW.name()).size(),
2);
+    // Make sure snapshot information is correct
+    // Check resources are in a correct state
+    boolean isRoutingTableUpdatedProperly = TestHelper.verify(() -> {
+      Map<String, Map<String, RoutingTableSnapshot>> routingTableSnapshots =
+          routingTableProvider.getRoutingTableSnapshots();
+      RoutingTableSnapshot routingTableSnapshotTypeA =
+          routingTableSnapshots.get(PropertyType.CUSTOMIZEDVIEW.name()).get("typeA");
+      RoutingTableSnapshot routingTableSnapshotTypeB =
+          routingTableSnapshots.get(PropertyType.CUSTOMIZEDVIEW.name()).get("typeB");
+      String typeAp1h1 = "noState";
+      String typeAp1h2 = "noState";
+      String typeAp2h1 = "noState";
+      String typeAp3h2 = "noState";
+      String typeBp1h2 = "noState";
+      String typeBp1h4 = "noState";
+      try {
+        typeAp1h1 = routingTableSnapshotTypeA.getCustomizeViews().iterator().next()
+            .getStateMap("p1").get("h1");
+        typeAp1h2 = routingTableSnapshotTypeA.getCustomizeViews().iterator().next()
+            .getStateMap("p1").get("h2");
+        typeAp2h1 = routingTableSnapshotTypeA.getCustomizeViews().iterator().next()
+            .getStateMap("p2").get("h1");
+        typeAp3h2 = routingTableSnapshotTypeA.getCustomizeViews().iterator().next()
+            .getStateMap("p3").get("h2");
+        typeBp1h2 = routingTableSnapshotTypeB.getCustomizeViews().iterator().next()
+            .getStateMap("p1").get("h3");
+        typeBp1h4 = routingTableSnapshotTypeB.getCustomizeViews().iterator().next()
+            .getStateMap("p1").get("h4");
+      } catch (Exception e) {
+        // ok because RoutingTable has not been updated yet
+        return false;
+      }
+      return (routingTableSnapshots.size() == 2
+          && routingTableSnapshots.get(PropertyType.CUSTOMIZEDVIEW.name()).size()
== 2
+          && typeAp1h1.equals("testState1") && typeAp1h2.equals("testState1")
+          && typeAp2h1.equals("testState2") && typeAp3h2.equals("testState3")
+          && typeBp1h2.equals("testState3") && typeBp1h4.equals("testState2"));
+    }, WAIT_DURATION);
+    Assert.assertTrue(isRoutingTableUpdatedProperly, "RoutingTable has been updated properly");
     routingTableProvider.shutdown();
   }
 
-
   private void validateRoutingTable(RoutingTableProvider routingTableProvider,
       Set<String> masterNodes, Set<String> slaveNodes) {
     IdealState is =


Mime
View raw message