helix-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From j...@apache.org
Subject [1/4] helix git commit: Persist preference list into IdealState in full-auto mode and allow user to choose persisting either bestpossible or intermediate state mapping into the mapfield of IS.
Date Mon, 09 Oct 2017 18:58:33 GMT
Repository: helix
Updated Branches:
  refs/heads/master d2c3ebb48 -> 188969926


Persist preference list into IdealState in full-auto mode and allow user to choose persisting
either bestpossible or intermediate state mapping into the mapfield of IS.


Project: http://git-wip-us.apache.org/repos/asf/helix/repo
Commit: http://git-wip-us.apache.org/repos/asf/helix/commit/18896992
Tree: http://git-wip-us.apache.org/repos/asf/helix/tree/18896992
Diff: http://git-wip-us.apache.org/repos/asf/helix/diff/18896992

Branch: refs/heads/master
Commit: 1889699269597dd5cb8b10e2eb90dcbcf6c64622
Parents: 0aeb557
Author: Lei Xia <lxia@linkedin.com>
Authored: Fri Feb 10 17:28:40 2017 -0800
Committer: Lei Xia <lxia@linkedin.com>
Committed: Fri Oct 6 12:23:47 2017 -0700

----------------------------------------------------------------------
 .../stages/BestPossibleStateCalcStage.java      | 10 +++----
 .../stages/PersistAssignmentStage.java          |  2 +-
 .../org/apache/helix/model/ClusterConfig.java   |  5 +++-
 .../java/org/apache/helix/model/IdealState.java | 29 +++++++++++---------
 .../apache/helix/task/WorkflowRebalancer.java   |  6 ----
 .../TestRebalancerPersistAssignments.java       |  2 ++
 .../integration/task/TestJobQueueCleanUp.java   |  4 +--
 .../integration/task/TestRecurringJobQueue.java |  1 +
 8 files changed, 31 insertions(+), 28 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/helix/blob/18896992/helix-core/src/main/java/org/apache/helix/controller/stages/BestPossibleStateCalcStage.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/controller/stages/BestPossibleStateCalcStage.java
b/helix-core/src/main/java/org/apache/helix/controller/stages/BestPossibleStateCalcStage.java
index 0a13a8d..5307e1a 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/stages/BestPossibleStateCalcStage.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/stages/BestPossibleStateCalcStage.java
@@ -144,23 +144,23 @@ public class BestPossibleStateCalcStage extends AbstractBaseStage {
     }
 
     if (rebalancer != null && mappingCalculator != null) {
-
       if (rebalancer instanceof TaskRebalancer) {
         TaskRebalancer taskRebalancer = TaskRebalancer.class.cast(rebalancer);
         taskRebalancer.setClusterStatusMonitor(
             (ClusterStatusMonitor) event.getAttribute("clusterStatusMonitor"));
       }
-
       try {
         HelixManager manager = event.getAttribute("helixmanager");
         rebalancer.init(manager);
-        idealState = rebalancer.computeNewIdealState(resourceName, idealState, currentStateOutput,
cache);
+        idealState =
+            rebalancer.computeNewIdealState(resourceName, idealState, currentStateOutput,
cache);
+
         output.setPreferenceLists(resourceName, idealState.getPreferenceLists());
 
         // Use the internal MappingCalculator interface to compute the final assignment
         // The next release will support rebalancers that compute the mapping from start
to finish
-        ResourceAssignment partitionStateAssignment =
-            mappingCalculator.computeBestPossiblePartitionState(cache, idealState, resource,
currentStateOutput);
+        ResourceAssignment partitionStateAssignment = mappingCalculator
+            .computeBestPossiblePartitionState(cache, idealState, resource, currentStateOutput);
         for (Partition partition : resource.getPartitions()) {
           Map<String, String> newStateMap = partitionStateAssignment.getReplicaMap(partition);
           output.setState(resourceName, partition, newStateMap);

http://git-wip-us.apache.org/repos/asf/helix/blob/18896992/helix-core/src/main/java/org/apache/helix/controller/stages/PersistAssignmentStage.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/controller/stages/PersistAssignmentStage.java
b/helix-core/src/main/java/org/apache/helix/controller/stages/PersistAssignmentStage.java
index cd320a4..fd4c706 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/stages/PersistAssignmentStage.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/stages/PersistAssignmentStage.java
@@ -24,6 +24,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import org.I0Itec.zkclient.DataUpdater;
+
 import org.apache.helix.HelixDataAccessor;
 import org.apache.helix.HelixManager;
 import org.apache.helix.PropertyKey;
@@ -87,7 +88,6 @@ public class PersistAssignmentStage extends AbstractBaseStage {
             needPersist = true;
           }
         }
-
         PartitionStateMap partitionStateMap =
             bestPossibleAssignment.getPartitionStateMap(resourceId);
         if (clusterConfig.isPersistIntermediateAssignment()) {

http://git-wip-us.apache.org/repos/asf/helix/blob/18896992/helix-core/src/main/java/org/apache/helix/model/ClusterConfig.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/model/ClusterConfig.java b/helix-core/src/main/java/org/apache/helix/model/ClusterConfig.java
index 4b217cd..8ee3a79 100644
--- a/helix-core/src/main/java/org/apache/helix/model/ClusterConfig.java
+++ b/helix-core/src/main/java/org/apache/helix/model/ClusterConfig.java
@@ -79,7 +79,8 @@ public class ClusterConfig extends HelixProperty {
 
   /**
    * Enable/Disable persist best possible assignment in a resource's idealstate.
-   *
+   * CAUTION: if both {@link #setPersistBestPossibleAssignment(Boolean)} and {@link #setPersistIntermediateAssignment(Boolean)}
+   * are set to true, the IntermediateAssignment will be persisted into IdealState's map
field.
    * @return
    */
   public void setPersistBestPossibleAssignment(Boolean enable) {
@@ -104,6 +105,8 @@ public class ClusterConfig extends HelixProperty {
 
   /**
    * Enable/Disable persist IntermediateAssignment in a resource's idealstate.
+   * CAUTION: if both {@link #setPersistBestPossibleAssignment(Boolean)} and {@link #setPersistIntermediateAssignment(Boolean)}
+   * are set to true, the IntermediateAssignment will be persisted into IdealState's map
field.
    *
    * @return
    */

http://git-wip-us.apache.org/repos/asf/helix/blob/18896992/helix-core/src/main/java/org/apache/helix/model/IdealState.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/model/IdealState.java b/helix-core/src/main/java/org/apache/helix/model/IdealState.java
index 48e43d6..ccc8a0a 100644
--- a/helix-core/src/main/java/org/apache/helix/model/IdealState.java
+++ b/helix-core/src/main/java/org/apache/helix/model/IdealState.java
@@ -351,9 +351,11 @@ public class IdealState extends HelixProperty {
   }
 
   /**
-   * Get the current mapping of a partition
-   * CAUTION: In FULL-AUTO mode, this method could return empty map if
-   * {@link ClusterConfig#setPersistBestPossibleAssignment(Boolean)} is set to true.
+   * Get the current mapping of a partition.
+   *
+   * CAUTION: In FULL-AUTO mode, this method
+   * could return empty map if neither {@link ClusterConfig#setPersistBestPossibleAssignment(Boolean)}
+   * nor {@link ClusterConfig#setPersistIntermediateAssignment(Boolean)} is set to true.
    *
    * @param partitionName the name of the partition
    * @return the instances where the replicas live and the state of each
@@ -375,25 +377,25 @@ public class IdealState extends HelixProperty {
 
   /**
    * Get the instances who host replicas of a partition.
-   * CAUTION: In FULL-AUTO mode, this method could return empty map if
-   * {@link ClusterConfig#setPersistBestPossibleAssignment(Boolean)} is set to true.
-   +
-   * @param partitionName the partition to look up
+   * CAUTION: In FULL-AUTO mode, this method
+   * could return empty set if neither {@link ClusterConfig#setPersistBestPossibleAssignment(Boolean)}
+   * nor {@link ClusterConfig#setPersistIntermediateAssignment(Boolean)} is set to true.
+   *
    * @return set of instance names
    */
   public Set<String> getInstanceSet(String partitionName) {
-    switch (getRebalanceMode()) {
+    switch(getRebalanceMode()) {
     case FULL_AUTO:
     case SEMI_AUTO:
     case USER_DEFINED:
     case TASK:
       List<String> prefList = _record.getListField(partitionName);
       if (prefList != null && !prefList.isEmpty()) {
-        return new TreeSet<String>(prefList);
+        return new TreeSet<>(prefList);
       } else {
         Map<String, String> stateMap = _record.getMapField(partitionName);
         if (stateMap != null && !stateMap.isEmpty()) {
-          return new TreeSet<String>(stateMap.keySet());
+          return new TreeSet<>(stateMap.keySet());
         } else {
           logger.warn(partitionName + " does NOT exist");
         }
@@ -402,21 +404,22 @@ public class IdealState extends HelixProperty {
     case CUSTOMIZED:
       Map<String, String> stateMap = _record.getMapField(partitionName);
       if (stateMap != null) {
-        return new TreeSet<String>(stateMap.keySet());
+        return new TreeSet<>(stateMap.keySet());
       } else {
         logger.warn(partitionName + " does NOT exist");
       }
       break;
     case NONE:
     default:
-      logger.error("Invalid ideal state mode: " + getResourceName());
+      logger.warn("Invalid ideal state mode: " + getResourceName());
       break;
     }
 
     return Collections.emptySet();
   }
 
-  /** Set the preference list of a partition
+  /**
+   * Set the preference list of a partition
    * @param partitionName the name of the partition
    * @param instanceList the instance preference list
    */

http://git-wip-us.apache.org/repos/asf/helix/blob/18896992/helix-core/src/main/java/org/apache/helix/task/WorkflowRebalancer.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/task/WorkflowRebalancer.java b/helix-core/src/main/java/org/apache/helix/task/WorkflowRebalancer.java
index ac2ac87..6b95658 100644
--- a/helix-core/src/main/java/org/apache/helix/task/WorkflowRebalancer.java
+++ b/helix-core/src/main/java/org/apache/helix/task/WorkflowRebalancer.java
@@ -443,14 +443,8 @@ public class WorkflowRebalancer extends TaskRebalancer {
   }
 
   /**
-<<<<<<< HEAD
-   * Clean up a workflow. This removes the workflow config, idealstate, externalview and
workflow
-   * contexts associated with this workflow, and all jobs information, including their configs,
-   * context, IS and EV.
-=======
    * Cleans up workflow configs and workflow contexts associated with this workflow, including
all
    * job-level configs and context, plus workflow-level information.
->>>>>>> Support configurable job purge interval for a queue.
    */
   private void cleanupWorkflow(String workflow, WorkflowConfig workflowcfg) {
     LOG.info("Cleaning up workflow: " + workflow);

http://git-wip-us.apache.org/repos/asf/helix/blob/18896992/helix-core/src/test/java/org/apache/helix/integration/TestRebalancerPersistAssignments.java
----------------------------------------------------------------------
diff --git a/helix-core/src/test/java/org/apache/helix/integration/TestRebalancerPersistAssignments.java
b/helix-core/src/test/java/org/apache/helix/integration/TestRebalancerPersistAssignments.java
index 2a9dc69..01aec14 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/TestRebalancerPersistAssignments.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/TestRebalancerPersistAssignments.java
@@ -24,8 +24,10 @@ import org.apache.helix.integration.manager.MockParticipantManager;
 import org.apache.helix.model.BuiltInStateModelDefinitions;
 import org.apache.helix.model.IdealState;
 import org.apache.helix.model.IdealState.RebalanceMode;
+import org.apache.helix.model.MasterSlaveSMD;
 import org.apache.helix.tools.ClusterSetup;
 import org.apache.helix.tools.ClusterVerifiers.BestPossibleExternalViewVerifier;
+import org.apache.helix.tools.ClusterVerifiers.HelixClusterVerifier;
 import org.testng.Assert;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.DataProvider;

http://git-wip-us.apache.org/repos/asf/helix/blob/18896992/helix-core/src/test/java/org/apache/helix/integration/task/TestJobQueueCleanUp.java
----------------------------------------------------------------------
diff --git a/helix-core/src/test/java/org/apache/helix/integration/task/TestJobQueueCleanUp.java
b/helix-core/src/test/java/org/apache/helix/integration/task/TestJobQueueCleanUp.java
index 6eecf20..018d071 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/task/TestJobQueueCleanUp.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/task/TestJobQueueCleanUp.java
@@ -86,11 +86,11 @@ public class TestJobQueueCleanUp extends TaskTestBase {
     String queueName = TestHelper.getTestMethodName();
     JobQueue.Builder builder = TaskTestUtil.buildJobQueue(queueName, capacity);
     WorkflowConfig.Builder cfgBuilder = new WorkflowConfig.Builder(builder.getWorkflowConfig());
-    cfgBuilder.setJobPurgeInterval(1000);
+    cfgBuilder.setJobPurgeInterval(500);
     builder.setWorkflowConfig(cfgBuilder.build());
 
     JobConfig.Builder jobBuilder =
-        new JobConfig.Builder().setTargetResource(WorkflowGenerator.DEFAULT_TGT_DB)
+        new JobConfig.Builder().setNumberOfTasks(1)
             .setCommand(MockTask.TASK_COMMAND).setMaxAttemptsPerTask(2).setJobCommandConfigMap(
             ImmutableMap.of(MockTask.SUCCESS_COUNT_BEFORE_FAIL, String.valueOf(capacity /
2)))
             .setExpiry(200L);

http://git-wip-us.apache.org/repos/asf/helix/blob/18896992/helix-core/src/test/java/org/apache/helix/integration/task/TestRecurringJobQueue.java
----------------------------------------------------------------------
diff --git a/helix-core/src/test/java/org/apache/helix/integration/task/TestRecurringJobQueue.java
b/helix-core/src/test/java/org/apache/helix/integration/task/TestRecurringJobQueue.java
index a1070d8..8253e18 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/task/TestRecurringJobQueue.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/task/TestRecurringJobQueue.java
@@ -276,6 +276,7 @@ public class TestRecurringJobQueue extends TaskTestBase {
     // Record all scheduled workflows
     wCtx = TaskTestUtil.pollForWorkflowContext(_driver, queueName);
     final List<String> scheduledWorkflows = new ArrayList<>(wCtx.getScheduledWorkflows());
+    Assert.assertFalse(scheduledWorkflows.size() > 2);
 
     // Delete recurrent workflow
     _driver.delete(queueName);


Mime
View raw message