helix-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hu...@apache.org
Subject [helix] 34/43: Add rerunFailingTestsCount config to surefire-plugin (#865)
Date Tue, 17 Mar 2020 00:04:05 GMT
This is an automated email from the ASF dual-hosted git repository.

hulee pushed a commit to branch zooscalability
in repository https://gitbox.apache.org/repos/asf/helix.git

commit cb27e260b35049d11a5214d8c2c3d569d9f14527
Author: Hunter Lee <hulee@linkedin.com>
AuthorDate: Thu Mar 5 18:55:27 2020 -0800

    Add rerunFailingTestsCount config to surefire-plugin (#865)
    
    It was observed that if build fails (if there is a test failure), then not all of the
test goals are executed. There are currently two goals: default-test (single ZK) and multi-zk.
If default-test has any test failures, we won't ever see multi-zk get executed, and this is
a problem because we don't get to run the test suite in a multi-zk setup.
    
    This config change is a workaround for this - we allow failing tests to be retried up
to 3 times to make them pass. If they still fail, we will consider them as flaky tests and
have to fix them moving forward.
    
    This PR also address minor comments and fixes a test.
---
 .../helix/integration/TestEnableCompression.java   | 53 +++++++++++++---------
 pom.xml                                            |  6 +++
 .../zookeeper/util/HttpRoutingDataReader.java      |  6 +--
 3 files changed, 40 insertions(+), 25 deletions(-)

diff --git a/helix-core/src/test/java/org/apache/helix/integration/TestEnableCompression.java
b/helix-core/src/test/java/org/apache/helix/integration/TestEnableCompression.java
index 39e412c..f1da2e6 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/TestEnableCompression.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/TestEnableCompression.java
@@ -1,25 +1,5 @@
 package org.apache.helix.integration;
 
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-
-import org.apache.helix.PropertyPathBuilder;
-import org.apache.helix.TestHelper;
-import org.apache.helix.common.ZkTestBase;
-import org.apache.helix.integration.manager.ClusterControllerManager;
-import org.apache.helix.integration.manager.MockParticipantManager;
-import org.apache.helix.zookeeper.api.client.HelixZkClient;
-import org.apache.helix.zookeeper.impl.factory.SharedZkClientFactory;
-import org.apache.helix.model.IdealState;
-import org.apache.helix.model.builder.CustomModeISBuilder;
-import org.apache.helix.tools.ClusterStateVerifier;
-import org.apache.helix.tools.ClusterStateVerifier.BestPossAndExtViewZkVerifier;
-import org.apache.helix.util.GZipCompressionUtil;
-import org.apache.helix.zookeeper.zkclient.serialize.BytesPushThroughSerializer;
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
 /*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
@@ -38,6 +18,29 @@ import org.testng.annotations.Test;
  * specific language governing permissions and limitations
  * under the License.
  */
+
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.helix.PropertyPathBuilder;
+import org.apache.helix.TestHelper;
+import org.apache.helix.common.ZkTestBase;
+import org.apache.helix.integration.manager.ClusterControllerManager;
+import org.apache.helix.integration.manager.MockParticipantManager;
+import org.apache.helix.model.IdealState;
+import org.apache.helix.model.builder.CustomModeISBuilder;
+import org.apache.helix.tools.ClusterVerifiers.BestPossibleExternalViewVerifier;
+import org.apache.helix.util.GZipCompressionUtil;
+import org.apache.helix.zookeeper.api.client.HelixZkClient;
+import org.apache.helix.zookeeper.impl.factory.SharedZkClientFactory;
+import org.apache.helix.zookeeper.zkclient.serialize.BytesPushThroughSerializer;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+
 /**
  * Test controller, spectator and participant roles when compression is enabled.
  * Compression can be enabled for a specific resource by setting enableCompression=true in
the
@@ -66,6 +69,8 @@ public class TestEnableCompression extends ZkTestBase {
     List<String> instancesInCluster =
         _gSetupTool.getClusterManagementTool().getInstancesInCluster(clusterName);
     String resourceName = "TestResource";
+    Set<String> expectedResources = new HashSet<>();
+    expectedResources.add(resourceName);
     CustomModeISBuilder customModeISBuilder = new CustomModeISBuilder(resourceName);
 
     int numPartitions = 10000;
@@ -96,15 +101,19 @@ public class TestEnableCompression extends ZkTestBase {
         new ClusterControllerManager(ZK_ADDR, clusterName, "controller_0");
     controller.syncStart();
 
+    Set<String> expectedLiveInstances = new HashSet<>();
     // start participants
     for (int i = 0; i < 5; i++) {
       String instanceName = "localhost_" + (12918 + i);
       participants[i] = new MockParticipantManager(ZK_ADDR, clusterName, instanceName);
       participants[i].syncStart();
+      expectedLiveInstances.add(instanceName);
     }
 
-    boolean result = ClusterStateVerifier
-        .verifyByPolling(new BestPossAndExtViewZkVerifier(ZK_ADDR, clusterName), 120000L);
+    BestPossibleExternalViewVerifier verifier =
+        new BestPossibleExternalViewVerifier.Builder(clusterName).setZkAddr(ZK_ADDR)
+            .setExpectLiveInstances(expectedLiveInstances).setResources(expectedResources).build();
+    boolean result = verifier.verify(120000L);
     Assert.assertTrue(result);
 
     List<String> compressedPaths = new ArrayList<>();
diff --git a/pom.xml b/pom.xml
index 1681850..e8c9bd3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -643,6 +643,10 @@ under the License.
               </goals>
               <id>default-test</id>
               <phase>test</phase>
+              <configuration>
+                <rerunFailingTestsCount>3</rerunFailingTestsCount>
+                <skipAfterFailureCount>10</skipAfterFailureCount>
+              </configuration>
             </execution>
             <execution>
               <goals>
@@ -655,6 +659,8 @@ under the License.
                   <multiZk>true</multiZk>
                   <numZk>3</numZk>
                 </systemPropertyVariables>
+                <rerunFailingTestsCount>3</rerunFailingTestsCount>
+                <skipAfterFailureCount>10</skipAfterFailureCount>
               </configuration>
             </execution>
           </executions>
diff --git a/zookeeper-api/src/main/java/org/apache/helix/zookeeper/util/HttpRoutingDataReader.java
b/zookeeper-api/src/main/java/org/apache/helix/zookeeper/util/HttpRoutingDataReader.java
index b4c1f9c..f2f907a 100644
--- a/zookeeper-api/src/main/java/org/apache/helix/zookeeper/util/HttpRoutingDataReader.java
+++ b/zookeeper-api/src/main/java/org/apache/helix/zookeeper/util/HttpRoutingDataReader.java
@@ -21,9 +21,9 @@ package org.apache.helix.zookeeper.util;
 
 import java.io.IOException;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.stream.Collectors;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -49,10 +49,10 @@ public class HttpRoutingDataReader {
   /** Double-checked locking requires that the following fields be volatile */
   // The following map stands for (MSDS endpoint, Raw Routing Data)
   private static volatile Map<String, Map<String, List<String>>> _rawRoutingDataMap
=
-      new HashMap<>();
+      new ConcurrentHashMap<>();
   // The following map stands for (MSDS endpoint, MetadataStoreRoutingData)
   private static volatile Map<String, MetadataStoreRoutingData> _metadataStoreRoutingDataMap
=
-      new HashMap<>();
+      new ConcurrentHashMap<>();
 
   /**
    * This class is a Singleton.


Mime
View raw message