helix-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hu...@apache.org
Subject [helix] 10/44: Fix test failures and fix logic check stable state
Date Sat, 25 May 2019 01:19:44 GMT
This is an automated email from the ASF dual-hosted git repository.

hulee pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git

commit 9526ffdc610049d34539d698da3a04f21cbaf9c5
Author: Junkai Xue <jxue@linkedin.com>
AuthorDate: Mon Apr 1 16:56:03 2019 -0700

    Fix test failures and fix logic check stable state
    
    Fix test failures:
    1. Add logic to skip task framework idealstates
    2. fix logic for test failure.
    
    RB=1615941
    BUG=HELIX-1725
    G=helix-reviewers
    A=lxia
    
    Signed-off-by: Hunter Lee <hulee@linkedin.com>
---
 .../model/builder/HelixConfigScopeBuilder.java     |  5 ++++
 .../apache/helix/util/InstanceValidationUtil.java  |  5 +++-
 .../helix/rest/server/AbstractTestClass.java       | 28 ++++++++++++++++++----
 .../helix/rest/server/TestClusterAccessor.java     |  4 ++--
 .../apache/helix/rest/server/TestJobAccessor.java  |  7 +++---
 5 files changed, 39 insertions(+), 10 deletions(-)

diff --git a/helix-core/src/main/java/org/apache/helix/model/builder/HelixConfigScopeBuilder.java
b/helix-core/src/main/java/org/apache/helix/model/builder/HelixConfigScopeBuilder.java
index d65ffd3..b1a7cf6 100644
--- a/helix-core/src/main/java/org/apache/helix/model/builder/HelixConfigScopeBuilder.java
+++ b/helix-core/src/main/java/org/apache/helix/model/builder/HelixConfigScopeBuilder.java
@@ -48,6 +48,8 @@ public class HelixConfigScopeBuilder {
     switch (type) {
     case CLUSTER:
       break;
+    case REST:
+      break;
     case PARTICIPANT:
       if (keys.length > 1) {
         _participantName = keys[1];
@@ -121,6 +123,9 @@ public class HelixConfigScopeBuilder {
             new HelixConfigScope(_type, Arrays.asList(_clusterName, _resourceName), _partitionName);
       }
       break;
+    case REST:
+      scope = new HelixConfigScope(_type, Arrays.asList(_clusterName, _clusterName), null);
+      break;
     default:
       break;
     }
diff --git a/helix-core/src/main/java/org/apache/helix/util/InstanceValidationUtil.java b/helix-core/src/main/java/org/apache/helix/util/InstanceValidationUtil.java
index 112ae1b..385920f 100644
--- a/helix-core/src/main/java/org/apache/helix/util/InstanceValidationUtil.java
+++ b/helix-core/src/main/java/org/apache/helix/util/InstanceValidationUtil.java
@@ -35,6 +35,7 @@ import org.apache.helix.model.IdealState;
 import org.apache.helix.model.InstanceConfig;
 import org.apache.helix.model.LiveInstance;
 import org.apache.helix.model.RESTConfig;
+import org.apache.helix.task.TaskConstants;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -270,9 +271,11 @@ public class InstanceValidationUtil {
     List<String> idealStateNames = dataAccessor.getChildNames(keyBuilder.idealStates());
     for (String idealStateName : idealStateNames) {
       IdealState idealState = dataAccessor.getProperty(keyBuilder.idealStates(idealStateName));
-      if (idealState == null || !idealState.isEnabled()) {
+      if (idealState == null || !idealState.isEnabled() || !idealState.isValid()
+          || TaskConstants.STATE_MODEL_NAME.equals(idealState.getStateModelDefRef())) {
         continue;
       }
+
       ExternalView externalView = dataAccessor.getProperty(keyBuilder.externalView(idealStateName));
       if (externalView == null) {
         throw new HelixException(
diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/AbstractTestClass.java
b/helix-rest/src/test/java/org/apache/helix/rest/server/AbstractTestClass.java
index 2003d2b..ab30419 100644
--- a/helix-rest/src/test/java/org/apache/helix/rest/server/AbstractTestClass.java
+++ b/helix-rest/src/test/java/org/apache/helix/rest/server/AbstractTestClass.java
@@ -47,6 +47,7 @@ import org.apache.helix.TestHelper;
 import org.apache.helix.ZNRecord;
 import org.apache.helix.integration.manager.ClusterControllerManager;
 import org.apache.helix.integration.manager.MockParticipantManager;
+import org.apache.helix.integration.task.MockTask;
 import org.apache.helix.integration.task.TaskTestUtil;
 import org.apache.helix.manager.zk.ZNRecordSerializer;
 import org.apache.helix.manager.zk.ZkBaseDataAccessor;
@@ -54,6 +55,7 @@ import org.apache.helix.manager.zk.client.DedicatedZkClientFactory;
 import org.apache.helix.manager.zk.client.HelixZkClient;
 import org.apache.helix.model.ClusterConfig;
 import org.apache.helix.model.InstanceConfig;
+import org.apache.helix.participant.StateMachineEngine;
 import org.apache.helix.rest.common.ContextPropertyKeys;
 import org.apache.helix.rest.common.HelixRestNamespace;
 import org.apache.helix.rest.server.auditlog.AuditLog;
@@ -64,10 +66,14 @@ import org.apache.helix.store.HelixPropertyStore;
 import org.apache.helix.store.zk.ZkHelixPropertyStore;
 import org.apache.helix.task.JobConfig;
 import org.apache.helix.task.JobContext;
+import org.apache.helix.task.Task;
+import org.apache.helix.task.TaskCallbackContext;
 import org.apache.helix.task.TaskConstants;
 import org.apache.helix.task.TaskDriver;
+import org.apache.helix.task.TaskFactory;
 import org.apache.helix.task.TaskPartitionState;
 import org.apache.helix.task.TaskState;
+import org.apache.helix.task.TaskStateModelFactory;
 import org.apache.helix.task.TaskUtil;
 import org.apache.helix.task.Workflow;
 import org.apache.helix.task.WorkflowContext;
@@ -108,6 +114,7 @@ public class AbstractTestClass extends JerseyTestNg.ContainerPerClassTest
{
   protected static HelixZkClient _gZkClientTestNS;
   protected static BaseDataAccessor<ZNRecord> _baseAccessorTestNS;
   protected static final String STOPPABLE_CLUSTER = "StoppableTestCluster";
+  protected static final String TASK_TEST_CLUSTER = "TaskTestCluster";
   protected static final List<String> STOPPABLE_INSTANCES =
       Arrays.asList("instance0", "instance1", "instance2", "instance3", "instance4", "instance5");
 
@@ -118,10 +125,10 @@ public class AbstractTestClass extends JerseyTestNg.ContainerPerClassTest
{
   protected static Map<String, Set<String>> _resourcesMap = new HashMap<>();
   protected static Map<String, Map<String, Workflow>> _workflowMap = new HashMap<>();
 
-  protected MockAuditLogger _auditLogger = new MockAuditLogger();
+  protected static MockAuditLogger _auditLogger = new MockAuditLogger();
   protected static HelixRestServer _helixRestServer;
 
-  protected class MockAuditLogger implements AuditLogger {
+  protected static class MockAuditLogger implements AuditLogger {
     List<AuditLog> _auditLogList = new ArrayList<>();
 
     @Override
@@ -197,7 +204,7 @@ public class AbstractTestClass extends JerseyTestNg.ContainerPerClassTest
{
               try {
                 _helixRestServer =
                     new HelixRestServer(namespaces, baseUri.getPort(), baseUri.getPath(),
-                        Arrays.<AuditLogger>asList(new MockAuditLogger()));
+                        Arrays.<AuditLogger>asList(_auditLogger));
                 _helixRestServer.start();
               } catch (Exception ex) {
                 throw new TestContainerException(ex);
@@ -274,7 +281,9 @@ public class AbstractTestClass extends JerseyTestNg.ContainerPerClassTest
{
   protected void setup() throws Exception {
     _clusters = createClusters(3);
     _gSetupTool.addCluster(_superCluster, true);
+    _gSetupTool.addCluster(TASK_TEST_CLUSTER, true);
     _clusters.add(_superCluster);
+    _clusters.add(TASK_TEST_CLUSTER);
     for (String cluster : _clusters) {
       Set<String> instances = createInstances(cluster, 10);
       Set<String> liveInstances = startInstances(cluster, instances, 6);
@@ -290,7 +299,6 @@ public class AbstractTestClass extends JerseyTestNg.ContainerPerClassTest
{
       startController(cluster);
     }
     preSetupForParallelInstancesStoppableTest(STOPPABLE_CLUSTER, STOPPABLE_INSTANCES);
-    _clusters.add(STOPPABLE_CLUSTER);
   }
 
   protected Set<String> createInstances(String cluster, int numInstances) throws Exception
{
@@ -333,6 +341,15 @@ public class AbstractTestClass extends JerseyTestNg.ContainerPerClassTest
{
     int i = 0;
     for (String instance : instances) {
       MockParticipantManager participant = new MockParticipantManager(ZK_ADDR, cluster, instance);
+      Map<String, TaskFactory> taskFactoryReg = new HashMap<String, TaskFactory>();
+      taskFactoryReg.put(MockTask.TASK_COMMAND, new TaskFactory() {
+        @Override public Task createNewTask(TaskCallbackContext context) {
+          return new MockTask(context);
+        }
+      });
+      StateMachineEngine stateMachineEngine = participant.getStateMachineEngine();
+      stateMachineEngine.registerStateModelFactory("Task",
+          new TaskStateModelFactory(participant, taskFactoryReg));
       participant.syncStart();
       liveInstances.add(instance);
       if (++i > numLiveinstances) {
@@ -511,5 +528,8 @@ public class AbstractTestClass extends JerseyTestNg.ContainerPerClassTest
{
     startInstances(clusterName, new TreeSet<>(instances), 3);
     createResources(clusterName, 1);
     startController(clusterName);
+
+    _clusters.add(STOPPABLE_CLUSTER);
+    _workflowMap.put(STOPPABLE_CLUSTER, createWorkflows(STOPPABLE_CLUSTER, 3));
   }
 }
diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
index fc5d94a..2bfa4ce 100644
--- a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
+++ b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
@@ -307,7 +307,7 @@ public class TestClusterAccessor extends AbstractTestClass {
         Response.Status.NOT_FOUND.getStatusCode(), false);
   }
 
-  @Test
+  @Test(dependsOnMethods = "testEnableDisableMaintenanceModeWithCustomFields")
   public void testGetControllerLeadershipHistory() throws IOException {
     System.out.println("Start test :" + TestHelper.getTestMethodName());
     String cluster = _clusters.iterator().next();
@@ -340,7 +340,7 @@ public class TestClusterAccessor extends AbstractTestClass {
     Assert.assertTrue(lastLeaderEntry.contains(leader));
   }
 
-  @Test
+  @Test(dependsOnMethods = "testGetControllerLeadershipHistory")
   public void testGetMaintenanceHistory() throws IOException {
     System.out.println("Start test :" + TestHelper.getTestMethodName());
     String cluster = _clusters.iterator().next();
diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/TestJobAccessor.java b/helix-rest/src/test/java/org/apache/helix/rest/server/TestJobAccessor.java
index 82416fb..0c5d862 100644
--- a/helix-rest/src/test/java/org/apache/helix/rest/server/TestJobAccessor.java
+++ b/helix-rest/src/test/java/org/apache/helix/rest/server/TestJobAccessor.java
@@ -46,7 +46,7 @@ import org.testng.annotations.Test;
 import com.google.common.collect.ImmutableMap;
 
 public class TestJobAccessor extends AbstractTestClass {
-  private final static String CLUSTER_NAME = "TestCluster_0";
+  private final static String CLUSTER_NAME = TASK_TEST_CLUSTER;
   private final static String WORKFLOW_NAME = WORKFLOW_PREFIX + 0;
   private final static String TEST_QUEUE_NAME = "TestQueue";
   private final static String JOB_NAME = WORKFLOW_NAME + "_" + JOB_PREFIX + 0;
@@ -194,15 +194,16 @@ public class TestJobAccessor extends AbstractTestClass {
   }
 
   @Test(dependsOnMethods = "testInvalidGetAndUpdateJobContentStore")
-  public void testDeleteJob() {
+  public void testDeleteJob() throws InterruptedException {
     System.out.println("Start test :" + TestHelper.getTestMethodName());
     TaskDriver driver = getTaskDriver(CLUSTER_NAME);
-
+    driver.waitToStop(TEST_QUEUE_NAME, 5000);
     delete("clusters/" + CLUSTER_NAME + "/workflows/" + TEST_QUEUE_NAME + "/jobs/" + TEST_JOB_NAME,
         Response.Status.OK.getStatusCode());
 
     String jobName = TaskUtil.getNamespacedJobName(TEST_QUEUE_NAME, TEST_JOB_NAME);
     JobConfig jobConfig = driver.getJobConfig(jobName);
+
     Assert.assertNull(jobConfig);
 
     WorkflowConfig workflowConfig = driver.getWorkflowConfig(TEST_QUEUE_NAME);


Mime
View raw message