helix-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From h...@apache.org
Subject [helix] branch master updated: Enhance ZkHelixVerifier and its related subclass to take a default wait period (#1328)
Date Sun, 13 Sep 2020 02:49:09 GMT
This is an automated email from the ASF dual-hosted git repository.

hzlu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git


The following commit(s) were added to refs/heads/master by this push:
     new 35ac0ff  Enhance ZkHelixVerifier and its related subclass to take a default wait
period (#1328)
35ac0ff is described below

commit 35ac0ff58b14e6ecc51280f88ed7c82c126bd248
Author: kaisun2000 <52840222+kaisun2000@users.noreply.github.com>
AuthorDate: Sat Sep 12 19:48:58 2020 -0700

    Enhance ZkHelixVerifier and its related subclass to take a default wait period (#1328)
    
    We have random sleep() in the test before verifyByPolling().
    This causes test failure in different environment. Here we enhance ZKHelixVerifier
    family with an option in builder to add a waiting period. To keep it backward
    compatible, without enable this waiting option, the code would behave exactly the
    same as before.
---
 .../BestPossibleExternalViewVerifier.java          | 11 +++++----
 .../ClusterVerifiers/ClusterLiveNodesVerifier.java | 10 ++++----
 .../StrictMatchExternalViewVerifier.java           | 12 ++++++----
 .../ClusterVerifiers/ZkHelixClusterVerifier.java   | 27 +++++++++++++++++++++-
 4 files changed, 46 insertions(+), 14 deletions(-)

diff --git a/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/BestPossibleExternalViewVerifier.java
b/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/BestPossibleExternalViewVerifier.java
index 5e50b6f..744624d 100644
--- a/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/BestPossibleExternalViewVerifier.java
+++ b/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/BestPossibleExternalViewVerifier.java
@@ -98,7 +98,7 @@ public class BestPossibleExternalViewVerifier extends ZkHelixClusterVerifier
{
   public BestPossibleExternalViewVerifier(RealmAwareZkClient zkClient, String clusterName,
       Set<String> resources, Map<String, Map<String, String>> errStates,
       Set<String> expectLiveInstances) {
-    super(zkClient, clusterName);
+    super(zkClient, clusterName, 0);
     _errStates = errStates;
     _resources = resources;
     _expectLiveInstances = expectLiveInstances;
@@ -107,8 +107,8 @@ public class BestPossibleExternalViewVerifier extends ZkHelixClusterVerifier
{
 
   private BestPossibleExternalViewVerifier(RealmAwareZkClient zkClient, String clusterName,
       Map<String, Map<String, String>> errStates, Set<String> resources,
-      Set<String> expectLiveInstances) {
-    super(zkClient, clusterName);
+      Set<String> expectLiveInstances, int waitPeriodTillVerify) {
+    super(zkClient, clusterName, waitPeriodTillVerify);
     // Deep copy data from Builder
     _errStates = new HashMap<>();
     if (errStates != null) {
@@ -151,7 +151,7 @@ public class BestPossibleExternalViewVerifier extends ZkHelixClusterVerifier
{
       return new BestPossibleExternalViewVerifier(
           createZkClient(RealmAwareZkClient.RealmMode.SINGLE_REALM, _realmAwareZkConnectionConfig,
               _realmAwareZkClientConfig, _zkAddress), _clusterName, _errStates, _resources,
-          _expectLiveInstances);
+          _expectLiveInstances, _waitPeriodTillVerify);
     }
 
     public String getClusterName() {
@@ -202,6 +202,8 @@ public class BestPossibleExternalViewVerifier extends ZkHelixClusterVerifier
{
 
   @Override
   public boolean verifyByZkCallback(long timeout) {
+    waitTillVerify();
+
     List<ClusterVerifyTrigger> triggers = new ArrayList<ClusterVerifyTrigger>();
 
     // setup triggers
@@ -319,6 +321,7 @@ public class BestPossibleExternalViewVerifier extends ZkHelixClusterVerifier
{
         }
 
         boolean result = verifyExternalView(extView, bpStateMap, stateModelDef);
+
         if (!result) {
           if (LOG.isDebugEnabled()) {
             LOG.debug("verifyExternalView fails for " + resourceName + "! ExternalView: "
+ extView
diff --git a/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/ClusterLiveNodesVerifier.java
b/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/ClusterLiveNodesVerifier.java
index 50ea8ea..8e28f3e 100644
--- a/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/ClusterLiveNodesVerifier.java
+++ b/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/ClusterLiveNodesVerifier.java
@@ -34,18 +34,20 @@ public class ClusterLiveNodesVerifier extends ZkHelixClusterVerifier {
   @Deprecated
   public ClusterLiveNodesVerifier(RealmAwareZkClient zkclient, String clusterName,
       List<String> expectLiveNodes) {
-    super(zkclient, clusterName);
+    super(zkclient, clusterName, 0);
     _expectLiveNodes = new HashSet<>(expectLiveNodes);
   }
 
   private ClusterLiveNodesVerifier(RealmAwareZkClient zkClient, String clusterName,
-      Set<String> expectLiveNodes) {
-    super(zkClient, clusterName);
+      Set<String> expectLiveNodes, int waitPeriodTillVerify) {
+    super(zkClient, clusterName, waitPeriodTillVerify);
     _expectLiveNodes = expectLiveNodes == null ? new HashSet<>() : new HashSet<>(expectLiveNodes);
   }
 
   @Override
   public boolean verifyByZkCallback(long timeout) {
+    waitTillVerify();
+
     List<ClusterVerifyTrigger> triggers = new ArrayList<ClusterVerifyTrigger>();
     triggers.add(new ClusterVerifyTrigger(_keyBuilder.liveInstances(), false, true, true));
 
@@ -81,7 +83,7 @@ public class ClusterLiveNodesVerifier extends ZkHelixClusterVerifier {
       validate();
       return new ClusterLiveNodesVerifier(
           createZkClient(RealmAwareZkClient.RealmMode.SINGLE_REALM, _realmAwareZkConnectionConfig,
-              _realmAwareZkClientConfig, _zkAddress), _clusterName, _expectLiveNodes);
+              _realmAwareZkClientConfig, _zkAddress), _clusterName, _expectLiveNodes, _waitPeriodTillVerify);
     }
   }
 }
diff --git a/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/StrictMatchExternalViewVerifier.java
b/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/StrictMatchExternalViewVerifier.java
index 9f91160..1073dad 100644
--- a/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/StrictMatchExternalViewVerifier.java
+++ b/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/StrictMatchExternalViewVerifier.java
@@ -66,7 +66,7 @@ public class StrictMatchExternalViewVerifier extends ZkHelixClusterVerifier
{
   @Deprecated
   public StrictMatchExternalViewVerifier(RealmAwareZkClient zkClient, String clusterName,
       Set<String> resources, Set<String> expectLiveInstances) {
-    this(zkClient, clusterName, resources, expectLiveInstances, false);
+    this(zkClient, clusterName, resources, expectLiveInstances, false, 0);
   }
 
   @Deprecated
@@ -79,8 +79,8 @@ public class StrictMatchExternalViewVerifier extends ZkHelixClusterVerifier
{
   }
 
   private StrictMatchExternalViewVerifier(RealmAwareZkClient zkClient, String clusterName,
-      Set<String> resources, Set<String> expectLiveInstances, boolean isDeactivatedNodeAware)
{
-    super(zkClient, clusterName);
+      Set<String> resources, Set<String> expectLiveInstances, boolean isDeactivatedNodeAware,
int waitPeriodTillVerify) {
+    super(zkClient, clusterName, waitPeriodTillVerify);
     _resources = resources == null ? new HashSet<>() : new HashSet<>(resources);
     _expectLiveInstances =
         expectLiveInstances == null ? new HashSet<>() : new HashSet<>(expectLiveInstances);
@@ -102,7 +102,7 @@ public class StrictMatchExternalViewVerifier extends ZkHelixClusterVerifier
{
 
       if (_zkClient != null) {
         return new StrictMatchExternalViewVerifier(_zkClient, _clusterName, _resources,
-            _expectLiveInstances, _isDeactivatedNodeAware);
+            _expectLiveInstances, _isDeactivatedNodeAware, _waitPeriodTillVerify);
       }
 
       if (_realmAwareZkConnectionConfig == null || _realmAwareZkClientConfig == null) {
@@ -115,7 +115,7 @@ public class StrictMatchExternalViewVerifier extends ZkHelixClusterVerifier
{
       return new StrictMatchExternalViewVerifier(
           createZkClient(RealmAwareZkClient.RealmMode.SINGLE_REALM, _realmAwareZkConnectionConfig,
               _realmAwareZkClientConfig, _zkAddress), _clusterName, _resources,
-          _expectLiveInstances, _isDeactivatedNodeAware);
+          _expectLiveInstances, _isDeactivatedNodeAware, _waitPeriodTillVerify);
     }
 
     public Builder(String clusterName) {
@@ -181,6 +181,8 @@ public class StrictMatchExternalViewVerifier extends ZkHelixClusterVerifier
{
 
   @Override
   public boolean verifyByZkCallback(long timeout) {
+    waitTillVerify();
+
     List<ClusterVerifyTrigger> triggers = new ArrayList<ClusterVerifyTrigger>();
 
     // setup triggers
diff --git a/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/ZkHelixClusterVerifier.java
b/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/ZkHelixClusterVerifier.java
index fe24b3b..734065f 100644
--- a/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/ZkHelixClusterVerifier.java
+++ b/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/ZkHelixClusterVerifier.java
@@ -58,6 +58,7 @@ public abstract class ZkHelixClusterVerifier
   protected final HelixDataAccessor _accessor;
   protected final PropertyKey.Builder _keyBuilder;
   private CountDownLatch _countdown;
+  protected final int _waitPeriodTillVerify;
 
   private ExecutorService _verifyTaskThreadPool =
       Executors.newSingleThreadExecutor(r -> new Thread(r, "ZkHelixClusterVerifier-verify_thread"));
@@ -93,7 +94,7 @@ public abstract class ZkHelixClusterVerifier
     }
   }
 
-  protected ZkHelixClusterVerifier(RealmAwareZkClient zkClient, String clusterName) {
+  protected ZkHelixClusterVerifier(RealmAwareZkClient zkClient, String clusterName, int waitPeriodTillVerify)
{
     if (zkClient == null || clusterName == null) {
       throw new IllegalArgumentException("requires zkClient|clusterName");
     }
@@ -102,6 +103,7 @@ public abstract class ZkHelixClusterVerifier
     _clusterName = clusterName;
     _accessor = new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<>(_zkClient));
     _keyBuilder = _accessor.keyBuilder();
+    _waitPeriodTillVerify = waitPeriodTillVerify;
   }
 
   @Deprecated
@@ -136,6 +138,7 @@ public abstract class ZkHelixClusterVerifier
     _clusterName = clusterName;
     _accessor = new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<>(_zkClient));
     _keyBuilder = _accessor.keyBuilder();
+    _waitPeriodTillVerify = 0;
   }
 
   /**
@@ -178,6 +181,15 @@ public abstract class ZkHelixClusterVerifier
     return verifyByZkCallback(DEFAULT_TIMEOUT);
   }
 
+  protected void waitTillVerify() {
+    try {
+      if (_waitPeriodTillVerify != 0) {
+        Thread.sleep(_waitPeriodTillVerify);
+      }
+    } catch (InterruptedException e) {
+      LOG.error("cooldown in verifyByPolling interrupted");
+    }
+  }
   /**
    * Verify the cluster by periodically polling the cluster status and verify.
    * The method will be blocked at most {@code timeout}.
@@ -187,6 +199,8 @@ public abstract class ZkHelixClusterVerifier
    * @return
    */
   public boolean verifyByPolling(long timeout, long period) {
+    waitTillVerify();
+
     try {
       long start = System.currentTimeMillis();
       boolean success;
@@ -197,6 +211,7 @@ public abstract class ZkHelixClusterVerifier
         }
         TimeUnit.MILLISECONDS.sleep(period);
       } while ((System.currentTimeMillis() - start) <= timeout);
+      LOG.error("verifier timeout out with timeout {}", timeout);
     } catch (Exception e) {
       LOG.error("Exception in verifier", e);
     }
@@ -234,6 +249,9 @@ public abstract class ZkHelixClusterVerifier
         if (!success) {
           // make a final try if timeout
           success = verifyState();
+          if (!success) {
+            LOG.error("verifyByCallback failed due to timeout {}", timeout);
+          }
         }
       }
     } catch (Exception e) {
@@ -319,6 +337,8 @@ public abstract class ZkHelixClusterVerifier
   }
 
   protected abstract static class Builder<B extends Builder<B>> extends GenericZkHelixApiBuilder<B>
{
+    protected int _waitPeriodTillVerify;
+
     public Builder() {
       // Note: ZkHelixClusterVerifier is a single-realm API, so RealmMode is assumed to be
       // SINGLE-REALM
@@ -335,6 +355,11 @@ public abstract class ZkHelixClusterVerifier
       return setZkAddress(zkAddress);
     }
 
+    public B setWaitTillVerify(int waitPeriod) {
+      _waitPeriodTillVerify = waitPeriod;
+      return (B) this;
+    }
+
     public String getClusterName() {
       if (_realmAwareZkConnectionConfig != null && (
           _realmAwareZkConnectionConfig.getZkRealmShardingKey() != null


Mime
View raw message