helix-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From h...@apache.org
Subject [helix] branch master updated: HelixClusterVerifier verify() with default waitTillVerify time (#1450)
Date Thu, 08 Oct 2020 23:37:17 GMT
This is an automated email from the ASF dual-hosted git repository.

hzlu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git


The following commit(s) were added to refs/heads/master by this push:
     new 12bfbae  HelixClusterVerifier verify() with default waitTillVerify time (#1450)
12bfbae is described below

commit 12bfbae5024d2b24e453921002e11cf71f98d047
Author: kaisun2000 <52840222+kaisun2000@users.noreply.github.com>
AuthorDate: Thu Oct 8 16:37:08 2020 -0700

    HelixClusterVerifier verify() with default waitTillVerify time (#1450)
    
    HelixClusterVerifier verify() and related method may return
    pre-maturely. The reason is that the verify the converging stable
    condition too early before controller has a chance to make
    change. Basically the previous stable state is mistaken as the
    expected next stable state. Part two.
---
 .../src/test/java/org/apache/helix/TestHelper.java |  2 --
 .../controller/stages/TestRebalancePipeline.java   |  1 +
 .../WagedRebalancer/TestWagedNodeSwap.java         |  6 ++--
 .../WagedRebalancer/TestWagedRebalance.java        | 16 +++++++----
 .../TestWagedRebalanceFaultZone.java               | 10 +++++--
 .../spectator/TestRoutingTableProvider.java        |  4 ++-
 .../TestRoutingTableProviderFromCurrentStates.java | 11 ++++----
 .../TestRoutingTableProviderPeriodicRefresh.java   |  5 +++-
 .../spectator/TestRoutingTableSnapshot.java        |  6 ++--
 .../helix/integration/task/TestDeleteWorkflow.java |  6 ++--
 .../integration/task/TestRebalanceRunningTask.java |  4 ++-
 .../handling/TestResourceThreadpoolSize.java       |  6 +++-
 .../TestClusterStatusMonitorLifecycle.java         |  6 +++-
 .../mbeans/TestClusterAggregateMetrics.java        |  6 ++--
 .../mbeans/TestDisableResourceMbean.java           |  4 ++-
 .../helix/task/TaskSynchronizedTestBase.java       |  5 +++-
 .../apache/helix/tools/TestClusterVerifier.java    | 33 +++++++++++++---------
 17 files changed, 87 insertions(+), 44 deletions(-)

diff --git a/helix-core/src/test/java/org/apache/helix/TestHelper.java b/helix-core/src/test/java/org/apache/helix/TestHelper.java
index 7a6c122..c805158 100644
--- a/helix-core/src/test/java/org/apache/helix/TestHelper.java
+++ b/helix-core/src/test/java/org/apache/helix/TestHelper.java
@@ -76,10 +76,8 @@ import org.testng.Assert;
 
 public class TestHelper {
   private static final Logger LOG = LoggerFactory.getLogger(TestHelper.class);
-
   public static final long WAIT_DURATION = 60 * 1000L; // 60 seconds
   public static final int DEFAULT_REBALANCE_PROCESSING_WAIT_TIME = 1500;
-
   /**
    * Returns a unused random port.
    */
diff --git a/helix-core/src/test/java/org/apache/helix/controller/stages/TestRebalancePipeline.java
b/helix-core/src/test/java/org/apache/helix/controller/stages/TestRebalancePipeline.java
index 95b0bdd..66c1771 100644
--- a/helix-core/src/test/java/org/apache/helix/controller/stages/TestRebalancePipeline.java
+++ b/helix-core/src/test/java/org/apache/helix/controller/stages/TestRebalancePipeline.java
@@ -158,6 +158,7 @@ public class TestRebalancePipeline extends ZkUnitTestBase {
     Thread.sleep(2 * MessageGenerationPhase.DEFAULT_OBSELETE_MSG_PURGE_DELAY);
     runPipeline(event, dataRefresh, false);
     
+    // Verify the stale message should be deleted
     Assert.assertTrue(TestHelper.verify(() -> {
       if (dataCache.getStaleMessages().size() != 0) {
         return false;
diff --git a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestWagedNodeSwap.java
b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestWagedNodeSwap.java
index c34851e..e956896 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestWagedNodeSwap.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestWagedNodeSwap.java
@@ -29,6 +29,7 @@ import java.util.Map;
 import java.util.Set;
 
 import org.apache.helix.ConfigAccessor;
+import org.apache.helix.TestHelper;
 import org.apache.helix.common.ZkTestBase;
 import org.apache.helix.integration.manager.ClusterControllerManager;
 import org.apache.helix.integration.manager.MockParticipantManager;
@@ -120,10 +121,11 @@ public class TestWagedNodeSwap extends ZkTestBase {
       _gSetupTool.rebalanceStorageCluster(CLUSTER_NAME, db, _replica);
       _allDBs.add(db);
     }
-    Thread.sleep(1000);
 
     _clusterVerifier =
-        new BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkAddr(ZK_ADDR).build();
+        new BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkAddr(ZK_ADDR)
+            .setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
+            .build();
     Assert.assertTrue(_clusterVerifier.verify(5000));
   }
 
diff --git a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestWagedRebalance.java
b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestWagedRebalance.java
index b778b3c..e0714cd 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestWagedRebalance.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestWagedRebalance.java
@@ -707,8 +707,14 @@ public class TestWagedRebalance extends ZkTestBase {
   private void validate(int expectedReplica) {
     HelixClusterVerifier _clusterVerifier =
         new StrictMatchExternalViewVerifier.Builder(CLUSTER_NAME).setZkAddr(ZK_ADDR)
-            .setDeactivatedNodeAwareness(true).setResources(_allDBs).build();
-    Assert.assertTrue(_clusterVerifier.verify(5000));
+            .setDeactivatedNodeAwareness(true).setResources(_allDBs)
+            .setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
+            .build();
+    try {
+      Assert.assertTrue(_clusterVerifier.verify(5000));
+    } finally {
+      _clusterVerifier.close();
+    }
     for (String db : _allDBs) {
       IdealState is =
           _gSetupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME, db);
@@ -716,7 +722,6 @@ public class TestWagedRebalance extends ZkTestBase {
           _gSetupTool.getClusterManagementTool().getResourceExternalView(CLUSTER_NAME, db);
       validateIsolation(is, ev, expectedReplica);
     }
-    _clusterVerifier.close();
   }
 
   /**
@@ -745,10 +750,11 @@ public class TestWagedRebalance extends ZkTestBase {
     }
     _allDBs.clear();
     // waiting for all DB be dropped.
-    Thread.sleep(100);
     ZkHelixClusterVerifier _clusterVerifier =
         new StrictMatchExternalViewVerifier.Builder(CLUSTER_NAME).setZkAddr(ZK_ADDR)
-            .setDeactivatedNodeAwareness(true).setResources(_allDBs).build();
+            .setDeactivatedNodeAwareness(true).setResources(_allDBs)
+            .setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
+            .build();
     try {
       Assert.assertTrue(_clusterVerifier.verifyByPolling());
     } finally {
diff --git a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestWagedRebalanceFaultZone.java
b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestWagedRebalanceFaultZone.java
index 4a65ab8..8590a78 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestWagedRebalanceFaultZone.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestWagedRebalanceFaultZone.java
@@ -28,6 +28,7 @@ import java.util.Map;
 import java.util.Set;
 
 import org.apache.helix.ConfigAccessor;
+import org.apache.helix.TestHelper;
 import org.apache.helix.common.ZkTestBase;
 import org.apache.helix.integration.manager.ClusterControllerManager;
 import org.apache.helix.integration.manager.MockParticipantManager;
@@ -304,7 +305,9 @@ public class TestWagedRebalanceFaultZone extends ZkTestBase {
 
   private void validate(int expectedReplica) {
     ZkHelixClusterVerifier _clusterVerifier =
-        new BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkClient(_gZkClient).build();
+        new BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkClient(_gZkClient)
+            .setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
+            .build();
     Assert.assertTrue(_clusterVerifier.verifyByPolling());
 
     for (String db : _allDBs) {
@@ -347,9 +350,10 @@ public class TestWagedRebalanceFaultZone extends ZkTestBase {
     }
     _allDBs.clear();
     // waiting for all DB be dropped.
-    Thread.sleep(100);
     ZkHelixClusterVerifier _clusterVerifier =
-        new BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkClient(_gZkClient).build();
+        new BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkClient(_gZkClient)
+            .setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
+            .build();
     Assert.assertTrue(_clusterVerifier.verifyByPolling());
   }
 
diff --git a/helix-core/src/test/java/org/apache/helix/integration/spectator/TestRoutingTableProvider.java
b/helix-core/src/test/java/org/apache/helix/integration/spectator/TestRoutingTableProvider.java
index 6f22b19..93c73fb 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/spectator/TestRoutingTableProvider.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/spectator/TestRoutingTableProvider.java
@@ -163,7 +163,9 @@ public class TestRoutingTableProvider extends ZkTestBase {
     _routingTableProvider_ev = new RoutingTableProvider(_spectator);
     _routingTableProvider_cs = new RoutingTableProvider(_spectator, PropertyType.CURRENTSTATES);
 
-    _clusterVerifier = new BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkClient(_gZkClient).build();
+    _clusterVerifier = new BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkClient(_gZkClient)
+        .setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
+        .build();
     Assert.assertTrue(_clusterVerifier.verifyByPolling());
   }
 
diff --git a/helix-core/src/test/java/org/apache/helix/integration/spectator/TestRoutingTableProviderFromCurrentStates.java
b/helix-core/src/test/java/org/apache/helix/integration/spectator/TestRoutingTableProviderFromCurrentStates.java
index b051b89..5503067 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/spectator/TestRoutingTableProviderFromCurrentStates.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/spectator/TestRoutingTableProviderFromCurrentStates.java
@@ -130,9 +130,10 @@ public class TestRoutingTableProviderFromCurrentStates extends ZkTestBase
{
       long startTime = System.currentTimeMillis();
       _gSetupTool.rebalanceStorageCluster(CLUSTER_NAME, db1, NUM_REPLICAS);
 
-      Thread.sleep(1000L);
       ZkHelixClusterVerifier clusterVerifier =
-          new BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkClient(_gZkClient).build();
+          new BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkClient(_gZkClient)
+              .setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
+              .build();
       Assert.assertTrue(clusterVerifier.verifyByPolling());
       validatePropagationLatency(PropertyType.CURRENTSTATES,
           System.currentTimeMillis() - startTime);
@@ -148,7 +149,6 @@ public class TestRoutingTableProviderFromCurrentStates extends ZkTestBase
{
       startTime = System.currentTimeMillis();
       _gSetupTool.rebalanceStorageCluster(CLUSTER_NAME, db2, NUM_REPLICAS);
 
-      Thread.sleep(1000L);
       Assert.assertTrue(clusterVerifier.verifyByPolling());
       validatePropagationLatency(PropertyType.CURRENTSTATES,
           System.currentTimeMillis() - startTime);
@@ -160,7 +160,6 @@ public class TestRoutingTableProviderFromCurrentStates extends ZkTestBase
{
       // shutdown an instance
       startTime = System.currentTimeMillis();
       _participants[0].syncStop();
-      Thread.sleep(1000L);
       Assert.assertTrue(clusterVerifier.verifyByPolling());
       validatePropagationLatency(PropertyType.CURRENTSTATES,
           System.currentTimeMillis() - startTime);
@@ -222,7 +221,9 @@ public class TestRoutingTableProviderFromCurrentStates extends ZkTestBase
{
           IdealState.RebalanceMode.FULL_AUTO.name(), CrushEdRebalanceStrategy.class.getName());
       _gSetupTool.rebalanceStorageCluster(CLUSTER_NAME, db, NUM_REPLICAS);
       ZkHelixClusterVerifier clusterVerifier =
-          new BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkClient(_gZkClient).build();
+          new BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkClient(_gZkClient)
+              .setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
+              .build();
       Assert.assertTrue(clusterVerifier.verifyByPolling(5000, 500));
       // 2. Process one event, so the current state will be refreshed with the new DB partitions
       routingTableCS.proceedNewEventHandling();
diff --git a/helix-core/src/test/java/org/apache/helix/integration/spectator/TestRoutingTableProviderPeriodicRefresh.java
b/helix-core/src/test/java/org/apache/helix/integration/spectator/TestRoutingTableProviderPeriodicRefresh.java
index 78f9ee1..d16165c 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/spectator/TestRoutingTableProviderPeriodicRefresh.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/spectator/TestRoutingTableProviderPeriodicRefresh.java
@@ -31,6 +31,7 @@ import org.apache.helix.HelixManagerFactory;
 import org.apache.helix.InstanceType;
 import org.apache.helix.NotificationContext;
 import org.apache.helix.PropertyType;
+import org.apache.helix.TestHelper;
 import org.apache.helix.common.ZkTestBase;
 import org.apache.helix.integration.manager.ClusterControllerManager;
 import org.apache.helix.integration.manager.MockParticipantManager;
@@ -136,7 +137,9 @@ public class TestRoutingTableProviderPeriodicRefresh extends ZkTestBase
{
     _spectator_3.addInstanceConfigChangeListener(_routingTableProviderLongPeriodicRefresh);
 
     _clusterVerifier =
-        new BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkClient(_gZkClient).build();
+        new BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkClient(_gZkClient)
+            .setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
+            .build();
     Assert.assertTrue(_clusterVerifier.verifyByPolling());
 
   }
diff --git a/helix-core/src/test/java/org/apache/helix/integration/spectator/TestRoutingTableSnapshot.java
b/helix-core/src/test/java/org/apache/helix/integration/spectator/TestRoutingTableSnapshot.java
index 470fec2..538c389 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/spectator/TestRoutingTableSnapshot.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/spectator/TestRoutingTableSnapshot.java
@@ -26,6 +26,7 @@ import org.apache.helix.HelixManager;
 import org.apache.helix.HelixManagerFactory;
 import org.apache.helix.InstanceType;
 import org.apache.helix.PropertyType;
+import org.apache.helix.TestHelper;
 import org.apache.helix.common.ZkTestBase;
 import org.apache.helix.controller.rebalancer.strategy.CrushEdRebalanceStrategy;
 import org.apache.helix.integration.manager.ClusterControllerManager;
@@ -106,7 +107,9 @@ public class TestRoutingTableSnapshot extends ZkTestBase {
 
       Thread.sleep(200);
       ZkHelixClusterVerifier clusterVerifier =
-          new BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkClient(_gZkClient).build();
+          new BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkClient(_gZkClient)
+              .setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
+              .build();
       Assert.assertTrue(clusterVerifier.verifyByPolling());
 
       IdealState idealState1 =
@@ -128,7 +131,6 @@ public class TestRoutingTableSnapshot extends ZkTestBase {
 
       // shutdown an instance
       _participants[0].syncStop();
-      Thread.sleep(200);
       Assert.assertTrue(clusterVerifier.verifyByPolling());
 
       // the original snapshot should not change
diff --git a/helix-core/src/test/java/org/apache/helix/integration/task/TestDeleteWorkflow.java
b/helix-core/src/test/java/org/apache/helix/integration/task/TestDeleteWorkflow.java
index 8d11b12..05977c7 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/task/TestDeleteWorkflow.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/task/TestDeleteWorkflow.java
@@ -153,8 +153,10 @@ public class TestDeleteWorkflow extends TaskTestBase {
     accessor.removeProperty(keyBuild.workflowContext(jobQueueName));
 
     BestPossibleExternalViewVerifier verifier =
-        new BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkAddr(ZK_ADDR)
-            .setZkClient(_gZkClient).build();
+      new BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME)
+        .setZkClient(_gZkClient)
+        .setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
+        .build();
     Assert.assertTrue(verifier.verifyByPolling());
 
     // Sometimes it's a ZK write fail - delete one more time to lower test failure rate
diff --git a/helix-core/src/test/java/org/apache/helix/integration/task/TestRebalanceRunningTask.java
b/helix-core/src/test/java/org/apache/helix/integration/task/TestRebalanceRunningTask.java
index d8fe719..058e9b4 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/task/TestRebalanceRunningTask.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/task/TestRebalanceRunningTask.java
@@ -323,7 +323,9 @@ public final class TestRebalanceRunningTask extends TaskSynchronizedTestBase
{
     startParticipant(_initialNumNodes);
     ZkHelixClusterVerifier clusterVerifier =
         new BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkClient(_gZkClient)
-            .setResources(Sets.newHashSet(DATABASE)).build();
+            .setResources(Sets.newHashSet(DATABASE))
+            .setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
+            .build();
     Assert.assertTrue(clusterVerifier.verify(10 * 1000));
 
     // Running tasks are also rebalanced
diff --git a/helix-core/src/test/java/org/apache/helix/messaging/handling/TestResourceThreadpoolSize.java
b/helix-core/src/test/java/org/apache/helix/messaging/handling/TestResourceThreadpoolSize.java
index f4444de..d550679 100644
--- a/helix-core/src/test/java/org/apache/helix/messaging/handling/TestResourceThreadpoolSize.java
+++ b/helix-core/src/test/java/org/apache/helix/messaging/handling/TestResourceThreadpoolSize.java
@@ -27,6 +27,7 @@ import java.util.concurrent.ThreadPoolExecutor;
 
 import org.apache.helix.ConfigAccessor;
 import org.apache.helix.HelixManager;
+import org.apache.helix.TestHelper;
 import org.apache.helix.integration.common.ZkStandAloneCMTestBase;
 import org.apache.helix.integration.manager.MockParticipantManager;
 import org.apache.helix.integration.task.WorkflowGenerator;
@@ -205,7 +206,10 @@ public class TestResourceThreadpoolSize extends ZkStandAloneCMTestBase
{
     Assert.assertTrue(executor.getPoolSize() >= numberOfDbs);
 
     BestPossibleExternalViewVerifier verifier =
-        new BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkAddr(ZK_ADDR).build();
+        new BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME)
+            .setZkClient(_gZkClient)
+            .setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
+            .build();
     Assert.assertTrue(verifier.verifyByPolling());
   }
 
diff --git a/helix-core/src/test/java/org/apache/helix/monitoring/TestClusterStatusMonitorLifecycle.java
b/helix-core/src/test/java/org/apache/helix/monitoring/TestClusterStatusMonitorLifecycle.java
index 207be0e..8668f8a 100644
--- a/helix-core/src/test/java/org/apache/helix/monitoring/TestClusterStatusMonitorLifecycle.java
+++ b/helix-core/src/test/java/org/apache/helix/monitoring/TestClusterStatusMonitorLifecycle.java
@@ -100,6 +100,7 @@ public class TestClusterStatusMonitorLifecycle extends ZkTestBase {
 
     ZkHelixClusterVerifier controllerClusterVerifier =
         new BestPossibleExternalViewVerifier.Builder(_controllerClusterName).setZkClient(_gZkClient)
+            .setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
             .build();
 
     Assert.assertTrue(controllerClusterVerifier.verifyByPolling(),
@@ -116,6 +117,7 @@ public class TestClusterStatusMonitorLifecycle extends ZkTestBase {
 
     ZkHelixClusterVerifier firstClusterVerifier =
         new BestPossibleExternalViewVerifier.Builder(_firstClusterName).setZkClient(_gZkClient)
+            .setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
             .build();
     Assert.assertTrue(firstClusterVerifier.verifyByPolling(), "first cluster NOT in ideal
state");
 
@@ -142,7 +144,9 @@ public class TestClusterStatusMonitorLifecycle extends ZkTestBase {
     for (int i = 1; i < clusterNb; i++) {
       ZkHelixClusterVerifier clusterVerifier =
           new BestPossibleExternalViewVerifier.Builder(_clusterNamePrefix + "0_" + i)
-              .setZkClient(_gZkClient).build();
+              .setZkClient(_gZkClient)
+              .setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
+              .build();
       Assert.assertTrue(clusterVerifier.verifyByPolling(), "Cluster NOT in ideal state.");
     }
   }
diff --git a/helix-core/src/test/java/org/apache/helix/monitoring/mbeans/TestClusterAggregateMetrics.java
b/helix-core/src/test/java/org/apache/helix/monitoring/mbeans/TestClusterAggregateMetrics.java
index 6b671ae..ba52601 100644
--- a/helix-core/src/test/java/org/apache/helix/monitoring/mbeans/TestClusterAggregateMetrics.java
+++ b/helix-core/src/test/java/org/apache/helix/monitoring/mbeans/TestClusterAggregateMetrics.java
@@ -145,8 +145,10 @@ public class TestClusterAggregateMetrics extends ZkTestBase {
   @Test
   public void testAggregateMetrics() throws Exception {
     BestPossibleExternalViewVerifier verifier =
-        new BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkAddr(ZK_ADDR)
-            .setZkClient(_gZkClient).build();
+        new BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME)
+            .setZkClient(_gZkClient)
+            .setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
+            .build();
 
     // Everything should be up and running initially with 5 total partitions
     updateMetrics();
diff --git a/helix-core/src/test/java/org/apache/helix/monitoring/mbeans/TestDisableResourceMbean.java
b/helix-core/src/test/java/org/apache/helix/monitoring/mbeans/TestDisableResourceMbean.java
index 8cf12f4..63e1a81 100644
--- a/helix-core/src/test/java/org/apache/helix/monitoring/mbeans/TestDisableResourceMbean.java
+++ b/helix-core/src/test/java/org/apache/helix/monitoring/mbeans/TestDisableResourceMbean.java
@@ -85,7 +85,9 @@ public class TestDisableResourceMbean extends ZkUnitTestBase {
     controller.syncStart();
 
     ZkHelixClusterVerifier clusterVerifier =
-        new BestPossibleExternalViewVerifier.Builder(clusterName).setZkClient(_gZkClient).build();
+        new BestPossibleExternalViewVerifier.Builder(clusterName).setZkClient(_gZkClient)
+            .setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
+            .build();
     Assert.assertTrue(clusterVerifier.verifyByPolling());
 
     // Verify the bean was created for TestDB0, but not for TestDB1.
diff --git a/helix-core/src/test/java/org/apache/helix/task/TaskSynchronizedTestBase.java
b/helix-core/src/test/java/org/apache/helix/task/TaskSynchronizedTestBase.java
index 191065d..026ee3c 100644
--- a/helix-core/src/test/java/org/apache/helix/task/TaskSynchronizedTestBase.java
+++ b/helix-core/src/test/java/org/apache/helix/task/TaskSynchronizedTestBase.java
@@ -28,6 +28,7 @@ import org.apache.helix.HelixException;
 import org.apache.helix.HelixManager;
 import org.apache.helix.HelixManagerFactory;
 import org.apache.helix.InstanceType;
+import org.apache.helix.TestHelper;
 import org.apache.helix.common.ZkTestBase;
 import org.apache.helix.controller.rebalancer.strategy.CrushEdRebalanceStrategy;
 import org.apache.helix.integration.manager.ClusterControllerManager;
@@ -76,7 +77,9 @@ public class TaskSynchronizedTestBase extends ZkTestBase {
     startParticipants();
     createManagers();
     _clusterVerifier =
-        new BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkClient(_gZkClient).build();
+        new BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkClient(_gZkClient)
+            .setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
+            .build();
   }
 
   @AfterClass
diff --git a/helix-core/src/test/java/org/apache/helix/tools/TestClusterVerifier.java b/helix-core/src/test/java/org/apache/helix/tools/TestClusterVerifier.java
index 59c695a..bbafe22 100644
--- a/helix-core/src/test/java/org/apache/helix/tools/TestClusterVerifier.java
+++ b/helix-core/src/test/java/org/apache/helix/tools/TestClusterVerifier.java
@@ -126,24 +126,23 @@ public class TestClusterVerifier extends ZkUnitTestBase {
     // Just ensure that the entire cluster passes
     // ensure that the external view coalesces
     HelixClusterVerifier bestPossibleVerifier =
-        new BestPossibleExternalViewVerifier.Builder(_clusterName).setZkClient(_gZkClient).build();
+        new BestPossibleExternalViewVerifier.Builder(_clusterName).setZkClient(_gZkClient)
+            .setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
+            .build();
     Assert.assertTrue(bestPossibleVerifier.verify(10000));
 
     // Disable partition for 1 instance, then Full-Auto ExternalView should match IdealState.
     _admin.enablePartition(false, _clusterName, _participants[0].getInstanceName(),
         FULL_AUTO_RESOURCES[0], Lists.newArrayList(FULL_AUTO_RESOURCES[0] + "_0"));
-    Thread.sleep(1000);
     Assert.assertTrue(bestPossibleVerifier.verify(3000));
 
     // Enable the partition back
     _admin.enablePartition(true, _clusterName, _participants[0].getInstanceName(),
         FULL_AUTO_RESOURCES[0], Lists.newArrayList(FULL_AUTO_RESOURCES[0] + "_0"));
-    Thread.sleep(1000);
     Assert.assertTrue(bestPossibleVerifier.verify(10000));
 
     // Make 1 instance non-live
     _participants[0].syncStop();
-    Thread.sleep(1000);
     Assert.assertTrue(bestPossibleVerifier.verify(10000));
 
     // Recover the participant before next test
@@ -154,31 +153,32 @@ public class TestClusterVerifier extends ZkUnitTestBase {
     HelixClusterVerifier strictMatchVerifier =
         new StrictMatchExternalViewVerifier.Builder(_clusterName)
             .setResources(Sets.newHashSet(RESOURCES)).setZkClient(_gZkClient)
-            .setDeactivatedNodeAwareness(true).build();
+            .setDeactivatedNodeAwareness(true)
+            .setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
+            .build();
     Assert.assertTrue(strictMatchVerifier.verify(10000));
 
     // Disable partition for 1 instance, then Full-Auto ExternalView should match IdealState.
     _admin.enablePartition(false, _clusterName, _participants[0].getInstanceName(),
         FULL_AUTO_RESOURCES[0], Lists.newArrayList(FULL_AUTO_RESOURCES[0] + "_0"));
-    Thread.sleep(1000);
     Assert.assertTrue(strictMatchVerifier.verify(3000));
 
     // Enable the partition back
     _admin.enablePartition(true, _clusterName, _participants[0].getInstanceName(),
         FULL_AUTO_RESOURCES[0], Lists.newArrayList(FULL_AUTO_RESOURCES[0] + "_0"));
-    Thread.sleep(1000);
     Assert.assertTrue(strictMatchVerifier.verify(10000));
 
     // Make 1 instance non-live
     _participants[0].syncStop();
-    Thread.sleep(1000);
 
     // Semi-Auto ExternalView matching
     for (String resource : SEMI_AUTO_RESOURCES) {
       System.out.println("Verify resource: " + resource);
       strictMatchVerifier =
           new StrictMatchExternalViewVerifier.Builder(_clusterName).setZkClient(_gZkClient)
-              .setResources(Sets.newHashSet(resource)).setDeactivatedNodeAwareness(true).build();
+              .setResources(Sets.newHashSet(resource)).setDeactivatedNodeAwareness(true)
+              .setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
+              .build();
       Assert.assertTrue(strictMatchVerifier.verify(3000));
     }
 
@@ -186,6 +186,7 @@ public class TestClusterVerifier extends ZkUnitTestBase {
     strictMatchVerifier =
         new StrictMatchExternalViewVerifier.Builder(_clusterName).setZkClient(_gZkClient)
             .setResources(Sets.newHashSet(FULL_AUTO_RESOURCES)).setDeactivatedNodeAwareness(true)
+            .setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
             .build();
     Assert.assertTrue(strictMatchVerifier.verify(10000));
   }
@@ -204,10 +205,11 @@ public class TestClusterVerifier extends ZkUnitTestBase {
 
     // Ensure that this passes even when one resource is down
     _admin.enableInstance(_clusterName, "localhost_12918", false);
-    Thread.sleep(1000);
     ZkHelixClusterVerifier verifier =
         new BestPossibleExternalViewVerifier.Builder(_clusterName).setZkClient(_gZkClient)
-            .setResources(Sets.newHashSet(testDB)).build();
+            .setResources(Sets.newHashSet(testDB))
+            .setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
+            .build();
     Assert.assertTrue(verifier.verifyByPolling());
 
     _admin.enableCluster(_clusterName, false);
@@ -238,12 +240,16 @@ public class TestClusterVerifier extends ZkUnitTestBase {
   public void testSleepTransition() throws InterruptedException {
 
     HelixClusterVerifier bestPossibleVerifier =
-        new BestPossibleExternalViewVerifier.Builder(_clusterName).setZkClient(_gZkClient).build();
+        new BestPossibleExternalViewVerifier.Builder(_clusterName).setZkClient(_gZkClient)
+            .setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
+            .build();
     Assert.assertTrue(bestPossibleVerifier.verify(10000));
 
     HelixClusterVerifier strictMatchVerifier =
         new StrictMatchExternalViewVerifier.Builder(_clusterName).setZkClient(_gZkClient)
-            .setDeactivatedNodeAwareness(true).build();
+            .setDeactivatedNodeAwareness(true)
+            .setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
+            .build();
     Assert.assertTrue(strictMatchVerifier.verify(10000));
 
     // Re-start a new participant with sleeping transition(all state model transition cannot
finish)
@@ -255,7 +261,6 @@ public class TestClusterVerifier extends ZkUnitTestBase {
     _participants[0].syncStart();
 
     // The new participant causes rebalance, but the state transitions are all stuck
-    Thread.sleep(1000);
     Assert.assertFalse(strictMatchVerifier.verify(3000));
   }
 }


Mime
View raw message