helix-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From l...@apache.org
Subject helix git commit: Fix performance issue in ClusterDataCache by using set instead of list.
Date Tue, 03 Apr 2018 22:22:38 GMT
Repository: helix
Updated Branches:
  refs/heads/master 84ce019e0 -> 3561845fa


Fix performance issue in ClusterDataCache by using set instead of list.


Project: http://git-wip-us.apache.org/repos/asf/helix/repo
Commit: http://git-wip-us.apache.org/repos/asf/helix/commit/3561845f
Tree: http://git-wip-us.apache.org/repos/asf/helix/tree/3561845f
Diff: http://git-wip-us.apache.org/repos/asf/helix/diff/3561845f

Branch: refs/heads/master
Commit: 3561845fafbda75055e22f4393c939645d515a4e
Parents: 84ce019
Author: Lei Xia <lxia@linkedin.com>
Authored: Wed Mar 28 13:38:23 2018 -0700
Committer: Lei Xia <lxia@linkedin.com>
Committed: Mon Apr 2 11:55:26 2018 -0700

----------------------------------------------------------------------
 .../helix/common/caches/CurrentStateCache.java  | 29 +++++++++-----------
 .../controller/GenericHelixController.java      |  8 +++---
 .../helix/controller/pipeline/Pipeline.java     |  3 --
 .../controller/stages/ClusterDataCache.java     | 27 +++++++++---------
 4 files changed, 31 insertions(+), 36 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/helix/blob/3561845f/helix-core/src/main/java/org/apache/helix/common/caches/CurrentStateCache.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/common/caches/CurrentStateCache.java
b/helix-core/src/main/java/org/apache/helix/common/caches/CurrentStateCache.java
index abf84b4..d28a859 100644
--- a/helix-core/src/main/java/org/apache/helix/common/caches/CurrentStateCache.java
+++ b/helix-core/src/main/java/org/apache/helix/common/caches/CurrentStateCache.java
@@ -19,17 +19,16 @@ package org.apache.helix.common.caches;
  * under the License.
  */
 
-import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.Iterator;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import org.apache.helix.HelixDataAccessor;
-import org.apache.helix.HelixProperty;
 import org.apache.helix.PropertyKey;
-import org.apache.helix.controller.stages.ClusterDataCache;
 import org.apache.helix.model.CurrentState;
 import org.apache.helix.model.LiveInstance;
 import org.slf4j.Logger;
@@ -62,11 +61,9 @@ public class CurrentStateCache {
    */
   public boolean refresh(HelixDataAccessor accessor,
       Map<String, LiveInstance> liveInstanceMap) {
-    LOG.info("START: CurrentStateCache.refresh()");
     long startTime = System.currentTimeMillis();
 
     refreshCurrentStatesCache(accessor, liveInstanceMap);
-
     Map<String, Map<String, Map<String, CurrentState>>> allCurStateMap
= new HashMap<>();
     for (PropertyKey key : _currentStateCache.keySet()) {
       CurrentState currentState = _currentStateCache.get(key);
@@ -107,7 +104,7 @@ public class CurrentStateCache {
     long start = System.currentTimeMillis();
     PropertyKey.Builder keyBuilder = accessor.keyBuilder();
 
-    List<PropertyKey> currentStateKeys = Lists.newLinkedList();
+    Set<PropertyKey> currentStateKeys = new HashSet<>();
     for (String instanceName : liveInstanceMap.keySet()) {
       LiveInstance liveInstance = liveInstanceMap.get(instanceName);
       String sessionId = liveInstance.getSessionId();
@@ -117,22 +114,22 @@ public class CurrentStateCache {
         currentStateKeys.add(keyBuilder.currentState(instanceName, sessionId, currentStateName));
       }
     }
-
     // All new entries from zk not cached locally yet should be read from ZK.
-    List<PropertyKey> reloadKeys = Lists.newLinkedList(currentStateKeys);
+    Set<PropertyKey> reloadKeys = new HashSet<>(currentStateKeys);
     reloadKeys.removeAll(_currentStateCache.keySet());
 
-    List<PropertyKey> cachedKeys = Lists.newLinkedList(_currentStateCache.keySet());
+    Set<PropertyKey> cachedKeys = new HashSet<>(_currentStateCache.keySet());
     cachedKeys.retainAll(currentStateKeys);
 
     _currentStateCache = Collections.unmodifiableMap(BasicClusterDataCache
-        .updateReloadProperties(accessor, reloadKeys, cachedKeys, _currentStateCache));
+        .updateReloadProperties(accessor, new ArrayList<>(reloadKeys), new ArrayList<>(cachedKeys),
+            _currentStateCache));
 
-    LOG.info(
-        "# of CurrentStates reload: " + reloadKeys.size() + ", skipped:" + (currentStateKeys.size()
-            - reloadKeys.size()));
-    LOG.info("Takes " + (System.currentTimeMillis() - start)
-        + " ms to reload new current states for cluster: " + _clusterName);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("# of CurrentStates reload: " + reloadKeys.size() + ", skipped:" + (
+          currentStateKeys.size() - reloadKeys.size()) + ". took " + (System.currentTimeMillis()
+          - start) + " ms to reload new current states for cluster: " + _clusterName);
+    }
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/helix/blob/3561845f/helix-core/src/main/java/org/apache/helix/controller/GenericHelixController.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/controller/GenericHelixController.java
b/helix-core/src/main/java/org/apache/helix/controller/GenericHelixController.java
index 971f482..6e2df23 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/GenericHelixController.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/GenericHelixController.java
@@ -416,10 +416,10 @@ public class GenericHelixController implements IdealStateChangeListener,
       _continousRebalanceFailureCount = 0;
     }
     long endTime = System.currentTimeMillis();
-    logger.info(
-        "END: Invoking " + getPipelineType(cache.isTaskCache()) + " controller pipeline for
event: "
-            + event.getEventType() + " for cluster " + manager.getClusterName() + ", took
" + (
-            endTime - startTime) + " ms");
+    logger.info(String
+        .format("END: Invoking %s controller pipeline for event: %s for cluster %s, took
%d ms",
+            getPipelineType(cache.isTaskCache()), event.getEventType(), manager.getClusterName(),
+            (endTime - startTime)));
 
     if (!cache.isTaskCache()) {
       // report event process durations

http://git-wip-us.apache.org/repos/asf/helix/blob/3561845f/helix-core/src/main/java/org/apache/helix/controller/pipeline/Pipeline.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/controller/pipeline/Pipeline.java b/helix-core/src/main/java/org/apache/helix/controller/pipeline/Pipeline.java
index 593b945..ac483f4 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/pipeline/Pipeline.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/pipeline/Pipeline.java
@@ -54,9 +54,6 @@ public class Pipeline {
     }
     for (Stage stage : _stages) {
       long startTime = System.currentTimeMillis();
-      logger.info(String
-          .format("START %s for %s pipeline for cluster %s", stage.getStageName(), _pipelineType,
-              event.getClusterName()));
 
       stage.preProcess();
       stage.process(event);

http://git-wip-us.apache.org/repos/asf/helix/blob/3561845f/helix-core/src/main/java/org/apache/helix/controller/stages/ClusterDataCache.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/controller/stages/ClusterDataCache.java
b/helix-core/src/main/java/org/apache/helix/controller/stages/ClusterDataCache.java
index 9f48529..386150d 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/stages/ClusterDataCache.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/stages/ClusterDataCache.java
@@ -137,29 +137,23 @@ public class ClusterDataCache {
    * @return
    */
   public synchronized boolean refresh(HelixDataAccessor accessor) {
-    LOG.info("START: ClusterDataCache.refresh()");
     long startTime = System.currentTimeMillis();
     Builder keyBuilder = accessor.keyBuilder();
 
     if (_propertyDataChangedMap.get(ChangeType.IDEAL_STATE)) {
-      long start = System.currentTimeMillis();
       _propertyDataChangedMap.put(ChangeType.IDEAL_STATE, Boolean.valueOf(false));
       clearCachedResourceAssignments();
       _idealStateCacheMap = refreshIdealStates(accessor);
-      if (LOG.isInfoEnabled()) {
-        LOG.info("Reload IdealStates: " + _idealStateCacheMap.keySet() + ". Takes " + (
-            System.currentTimeMillis() - start) + " ms");
-      }
     }
 
     if (_propertyDataChangedMap.get(ChangeType.LIVE_INSTANCE)) {
+      long start = System.currentTimeMillis();
       _propertyDataChangedMap.put(ChangeType.LIVE_INSTANCE, Boolean.valueOf(false));
       clearCachedResourceAssignments();
       _liveInstanceCacheMap = accessor.getChildValuesMap(keyBuilder.liveInstances(), true);
       _updateInstanceOfflineTime = true;
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Reload LiveInstances: " + _liveInstanceCacheMap.keySet());
-      }
+      LOG.info("Refresh LiveInstances for cluster " + _clusterName + ", took " + (
+          System.currentTimeMillis() - startTime) + " ms");
     }
 
     if (_propertyDataChangedMap.get(ChangeType.INSTANCE_CONFIG)) {
@@ -233,6 +227,7 @@ public class ClusterDataCache {
         for (LiveInstance instance : _liveInstanceMap.values()) {
           LOG.debug("live instance: " + instance.getInstanceName() + " " + instance.getSessionId());
         }
+      LOG.debug("IdealStates: " + _idealStateMap.keySet());
       LOG.debug("ResourceConfigs: " + _resourceConfigMap.keySet());
       LOG.debug("InstanceConfigs: " + _instanceConfigMap.keySet());
       LOG.debug("ClusterConfigs: " + _clusterConfig);
@@ -673,13 +668,14 @@ public class ClusterDataCache {
   }
 
   private Map<String, IdealState> refreshIdealStates(HelixDataAccessor accessor) {
+    long startTime = System.currentTimeMillis();
     PropertyKey.Builder keyBuilder = accessor.keyBuilder();
-    List<PropertyKey> currentIdealStateKeys = new ArrayList<>();
+    Set<PropertyKey> currentIdealStateKeys = new HashSet<>();
     for (String idealState : accessor.getChildNames(keyBuilder.idealStates())) {
       currentIdealStateKeys.add(keyBuilder.idealStates(idealState));
     }
 
-    List<PropertyKey> cachedKeys = new ArrayList<>();
+    Set<PropertyKey> cachedKeys = new HashSet<>();
     Map<PropertyKey, IdealState> cachedIdealStateMap = Maps.newHashMap();
     for (String idealState : _idealStateCacheMap.keySet()) {
       cachedKeys.add(keyBuilder.idealStates(idealState));
@@ -688,16 +684,21 @@ public class ClusterDataCache {
     }
     cachedKeys.retainAll(currentIdealStateKeys);
 
-    List<PropertyKey> reloadKeys = new LinkedList<>(currentIdealStateKeys);
+    Set<PropertyKey> reloadKeys = new HashSet<>(currentIdealStateKeys);
     reloadKeys.removeAll(cachedKeys);
 
     Map<PropertyKey, IdealState> updatedMap = BasicClusterDataCache
-        .updateReloadProperties(accessor, reloadKeys, cachedKeys, cachedIdealStateMap);
+        .updateReloadProperties(accessor, new LinkedList<>(reloadKeys),
+            new ArrayList<>(cachedKeys), cachedIdealStateMap);
     Map<String, IdealState> newIdealStateMap = Maps.newHashMap();
     for (IdealState idealState : updatedMap.values()) {
       newIdealStateMap.put(idealState.getResourceName(), idealState);
     }
 
+    long endTime = System.currentTimeMillis();
+    LOG.info("Refresh idealStates for cluster " + _clusterName + ", took " + (endTime
+        - startTime) + " ms");
+
     return Collections.unmodifiableMap(newIdealStateMap);
   }
 


Mime
View raw message