helix-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From h...@apache.org
Subject [helix] 02/13: Add model to record history and status of management mode (#1771)
Date Fri, 16 Jul 2021 21:03:09 GMT
This is an automated email from the ASF dual-hosted git repository.

hzlu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git

commit 88dae34522e3dc75497ea6d8cb5c19b288ee3212
Author: Huizhi Lu <5187721+huizhilu@users.noreply.github.com>
AuthorDate: Wed Jun 9 14:54:14 2021 -0700

    Add model to record history and status of management mode (#1771)
    
    Management mode operation history needs to be persisted to the controller history znode.
The status of IN_PROGRESS or COMPLETED is recorded in the temporary status znode: /{clusterName}/STATUS/CLUSTER/{clusterName}.
    This commit adds data model and methods to record the status and history for management
mode.
---
 .../main/java/org/apache/helix/PropertyKey.java    |  12 ++-
 .../java/org/apache/helix/PropertyPathBuilder.java |   7 ++
 .../main/java/org/apache/helix/PropertyType.java   |   1 +
 .../java/org/apache/helix/model/ClusterStatus.java |  78 ++++++++++++++
 .../org/apache/helix/model/ControllerHistory.java  | 112 +++++++++++++++------
 .../org/apache/helix/TestPropertyPathBuilder.java  |   3 +
 .../helix/model/TestControllerHistoryModel.java    |  93 +++++++++++++++++
 7 files changed, 276 insertions(+), 30 deletions(-)

diff --git a/helix-core/src/main/java/org/apache/helix/PropertyKey.java b/helix-core/src/main/java/org/apache/helix/PropertyKey.java
index 2cf1168..254cb95 100644
--- a/helix-core/src/main/java/org/apache/helix/PropertyKey.java
+++ b/helix-core/src/main/java/org/apache/helix/PropertyKey.java
@@ -25,10 +25,11 @@ import java.util.Objects;
 import org.apache.helix.model.CloudConfig;
 import org.apache.helix.model.ClusterConfig;
 import org.apache.helix.model.ClusterConstraints;
+import org.apache.helix.model.ClusterStatus;
 import org.apache.helix.model.ControllerHistory;
 import org.apache.helix.model.CurrentState;
-import org.apache.helix.model.CustomizedStateConfig;
 import org.apache.helix.model.CustomizedState;
+import org.apache.helix.model.CustomizedStateConfig;
 import org.apache.helix.model.CustomizedView;
 import org.apache.helix.model.Error;
 import org.apache.helix.model.ExternalView;
@@ -56,10 +57,10 @@ import static org.apache.helix.PropertyType.CONFIGS;
 import static org.apache.helix.PropertyType.CONTROLLER;
 import static org.apache.helix.PropertyType.CURRENTSTATES;
 import static org.apache.helix.PropertyType.CUSTOMIZEDSTATES;
+import static org.apache.helix.PropertyType.CUSTOMIZEDVIEW;
 import static org.apache.helix.PropertyType.ERRORS;
 import static org.apache.helix.PropertyType.ERRORS_CONTROLLER;
 import static org.apache.helix.PropertyType.EXTERNALVIEW;
-import static org.apache.helix.PropertyType.CUSTOMIZEDVIEW;
 import static org.apache.helix.PropertyType.HISTORY;
 import static org.apache.helix.PropertyType.IDEALSTATES;
 import static org.apache.helix.PropertyType.INSTANCE_HISTORY;
@@ -241,6 +242,13 @@ public class PropertyKey {
           _clusterName, ConfigScopeProperty.CLUSTER.toString(), _clusterName);
     }
 
+    /**
+     * Get a property key associated with this cluster status
+     * @return {@link PropertyKey}
+     */
+    public PropertyKey clusterStatus() {
+      return new PropertyKey(PropertyType.STATUS, ClusterStatus.class, _clusterName);
+    }
 
     /**
      * Get a property key associated with this Cloud configuration
diff --git a/helix-core/src/main/java/org/apache/helix/PropertyPathBuilder.java b/helix-core/src/main/java/org/apache/helix/PropertyPathBuilder.java
index 2ba1ebd..34efd29 100644
--- a/helix-core/src/main/java/org/apache/helix/PropertyPathBuilder.java
+++ b/helix-core/src/main/java/org/apache/helix/PropertyPathBuilder.java
@@ -25,6 +25,7 @@ import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import org.apache.helix.model.ClusterStatus;
 import org.apache.helix.model.ControllerHistory;
 import org.apache.helix.model.CurrentState;
 import org.apache.helix.model.CustomizedView;
@@ -66,6 +67,7 @@ public class PropertyPathBuilder {
     typeToClassMapping.put(PropertyType.HISTORY, ControllerHistory.class);
     typeToClassMapping.put(PropertyType.PAUSE, PauseSignal.class);
     typeToClassMapping.put(PropertyType.MAINTENANCE, MaintenanceSignal.class);
+    typeToClassMapping.put(PropertyType.STATUS, ClusterStatus.class);
     // TODO: Below must handle the case for future versions of Task Framework with a different
path
     // structure
     typeToClassMapping.put(PropertyType.WORKFLOWCONTEXT, WorkflowContext.class);
@@ -86,6 +88,7 @@ public class PropertyPathBuilder {
     addEntry(PropertyType.CUSTOMIZEDVIEW, 1, "/{clusterName}/CUSTOMIZEDVIEW");
     addEntry(PropertyType.CUSTOMIZEDVIEW, 2, "/{clusterName}/CUSTOMIZEDVIEW/{customizedStateType}");
     addEntry(PropertyType.CUSTOMIZEDVIEW, 3, "/{clusterName}/CUSTOMIZEDVIEW/{customizedStateType}/{resourceName}");
+    addEntry(PropertyType.STATUS, 1, "/{clusterName}/STATUS");
 
     addEntry(PropertyType.TARGETEXTERNALVIEW, 1, "/{clusterName}/TARGETEXTERNALVIEW");
     addEntry(PropertyType.TARGETEXTERNALVIEW, 2,
@@ -452,4 +455,8 @@ public class PropertyPathBuilder {
   public static String maintenance(String clusterName) {
     return String.format("/%s/CONTROLLER/MAINTENANCE", clusterName);
   }
+
+  public static String clusterStatus(String clusterName) {
+    return String.format("/%s/STATUS/CLUSTER/%s", clusterName, clusterName);
+  }
 }
diff --git a/helix-core/src/main/java/org/apache/helix/PropertyType.java b/helix-core/src/main/java/org/apache/helix/PropertyType.java
index bedf79e..474ea05 100644
--- a/helix-core/src/main/java/org/apache/helix/PropertyType.java
+++ b/helix-core/src/main/java/org/apache/helix/PropertyType.java
@@ -48,6 +48,7 @@ public enum PropertyType {
   STATEMODELDEFS(Type.CLUSTER, true, false, false, false, true),
   CONTROLLER(Type.CLUSTER, true, false),
   PROPERTYSTORE(Type.CLUSTER, true, false),
+  STATUS(Type.CLUSTER, true, false, true),
 
   // INSTANCE PROPERTIES
   MESSAGES(Type.INSTANCE, true, true, true),
diff --git a/helix-core/src/main/java/org/apache/helix/model/ClusterStatus.java b/helix-core/src/main/java/org/apache/helix/model/ClusterStatus.java
new file mode 100644
index 0000000..6ed354c
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/model/ClusterStatus.java
@@ -0,0 +1,78 @@
+package org.apache.helix.model;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.helix.HelixProperty;
+import org.apache.helix.PropertyType;
+import org.apache.helix.api.status.ClusterManagementMode;
+
+/**
+ * Represents the cluster status. It can have fields for
+ * {@link ClusterManagementMode} type and status.
+ */
+public class ClusterStatus extends HelixProperty {
+  public ClusterStatus() {
+    super(PropertyType.STATUS.name());
+  }
+
+  public enum ClusterStatusProperty {
+    MANAGEMENT_MODE,
+    MANAGEMENT_MODE_STATUS
+  }
+
+  /**
+   * Sets the type of management mode
+   *
+   * @param mode {@link ClusterManagementMode.Type}
+   */
+  public void setManagementMode(ClusterManagementMode.Type mode) {
+    _record.setEnumField(ClusterStatusProperty.MANAGEMENT_MODE.name(), mode);
+  }
+
+  /**
+   * Gets the type of management mode
+   *
+   * @return {@link ClusterManagementMode.Type}
+   */
+  public ClusterManagementMode.Type getManagementMode() {
+    return _record.getEnumField(ClusterStatusProperty.MANAGEMENT_MODE.name(),
+        ClusterManagementMode.Type.class, null);
+  }
+
+  /**
+   * Sets the cluster management mode status.
+   *
+   * @param status {@link ClusterManagementMode.Status}
+   */
+  public void setManagementModeStatus(ClusterManagementMode.Status status) {
+    _record.setEnumField(ClusterStatusProperty.MANAGEMENT_MODE_STATUS.name(), status);
+  }
+
+  /**
+   * Gets the {@link ClusterManagementMode.Status} of cluster management mode.
+   *
+   * @return {@link ClusterManagementMode.Status} if status is valid; otherwise, return {@code
+   * null}.
+   */
+  public ClusterManagementMode.Status getManagementModeStatus() {
+    return _record.getEnumField(ClusterStatusProperty.MANAGEMENT_MODE_STATUS.name(),
+        ClusterManagementMode.Status.class, null);
+  }
+}
\ No newline at end of file
diff --git a/helix-core/src/main/java/org/apache/helix/model/ControllerHistory.java b/helix-core/src/main/java/org/apache/helix/model/ControllerHistory.java
index cc0da7f..4e418c3 100644
--- a/helix-core/src/main/java/org/apache/helix/model/ControllerHistory.java
+++ b/helix-core/src/main/java/org/apache/helix/model/ControllerHistory.java
@@ -22,7 +22,9 @@ package org.apache.helix.model;
 import java.io.IOException;
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
+import java.time.Instant;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
@@ -30,10 +32,11 @@ import java.util.Map;
 import java.util.TimeZone;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.helix.HelixException;
 import org.apache.helix.HelixProperty;
+import org.apache.helix.api.status.ClusterManagementMode;
 import org.apache.helix.zookeeper.datamodel.ZNRecord;
 
-
 /**
  * The history of instances that have served as the leader controller
  */
@@ -57,6 +60,12 @@ public class ControllerHistory extends HelixProperty {
 
   }
 
+  private enum ManagementModeConfigKey {
+    MANAGEMENT_MODE_HISTORY,
+    MODE,
+    STATUS
+  }
+
   private enum OperationType {
     // The following are options for OPERATION_TYPE in MaintenanceConfigKey
     ENTER,
@@ -65,7 +74,8 @@ public class ControllerHistory extends HelixProperty {
 
   public enum HistoryType {
     CONTROLLER_LEADERSHIP,
-    MAINTENANCE
+    MAINTENANCE,
+    MANAGEMENT_MODE
   }
 
   public ControllerHistory(String id) {
@@ -96,17 +106,6 @@ public class ControllerHistory extends HelixProperty {
     list.add(instanceName);
     // TODO: remove above in future when we confirmed no one consumes it */
 
-    List<String> historyList = _record.getListField(ConfigProperty.HISTORY.name());
-    if (historyList == null) {
-      historyList = new ArrayList<>();
-      _record.setListField(ConfigProperty.HISTORY.name(), historyList);
-    }
-
-    // Keep only the last HISTORY_SIZE entries
-    while (historyList.size() >= HISTORY_SIZE) {
-      historyList.remove(0);
-    }
-
     Map<String, String> historyEntry = new HashMap<>();
 
     long currentTime = System.currentTimeMillis();
@@ -119,8 +118,7 @@ public class ControllerHistory extends HelixProperty {
     historyEntry.put(ConfigProperty.DATE.name(), dateTime);
     historyEntry.put(ConfigProperty.VERSION.name(), version);
 
-    historyList.add(historyEntry.toString());
-    return _record;
+    return populateHistoryEntries(HistoryType.CONTROLLER_LEADERSHIP, historyEntry.toString());
   }
 
   /**
@@ -137,6 +135,40 @@ public class ControllerHistory extends HelixProperty {
   }
 
   /**
+   * Gets the management mode history.
+   *
+   * @return List of history strings.
+   */
+  public List<String> getManagementModeHistory() {
+    List<String> history =
+        _record.getListField(ManagementModeConfigKey.MANAGEMENT_MODE_HISTORY.name());
+    return history == null ? Collections.emptyList() : history;
+  }
+
+  /**
+   * Updates management mode and status history to controller history in FIFO order.
+   *
+   * @param controller controller name
+   * @param mode cluster management mode {@link ClusterManagementMode}
+   * @param fromHost the hostname that creates the management mode signal
+   * @param time time in millis
+   * @param reason reason to put the cluster in management mode
+   * @return updated history znrecord
+   */
+  public ZNRecord updateManagementModeHistory(String controller, ClusterManagementMode mode,
+      String fromHost, long time, String reason) {
+    Map<String, String> historyEntry = new HashMap<>();
+    historyEntry.put(ConfigProperty.CONTROLLER.name(), controller);
+    historyEntry.put(ConfigProperty.TIME.name(), Instant.ofEpochMilli(time).toString());
+    historyEntry.put(ManagementModeConfigKey.MODE.name(), mode.getMode().name());
+    historyEntry.put(ManagementModeConfigKey.STATUS.name(), mode.getStatus().name());
+    historyEntry.put(PauseSignal.PauseSignalProperty.FROM_HOST.name(), fromHost);
+    historyEntry.put(PauseSignal.PauseSignalProperty.REASON.name(), reason);
+
+    return populateHistoryEntries(HistoryType.MANAGEMENT_MODE, historyEntry.toString());
+  }
+
+  /**
    * Record up to MAINTENANCE_HISTORY_SIZE number of changes to MaintenanceSignal in FIFO
order.
    * @param enabled
    * @param reason
@@ -148,18 +180,6 @@ public class ControllerHistory extends HelixProperty {
   public ZNRecord updateMaintenanceHistory(boolean enabled, String reason, long currentTime,
       MaintenanceSignal.AutoTriggerReason internalReason, Map<String, String> customFields,
       MaintenanceSignal.TriggeringEntity triggeringEntity) throws IOException {
-    List<String> maintenanceHistoryList =
-        _record.getListField(MaintenanceConfigKey.MAINTENANCE_HISTORY.name());
-    if (maintenanceHistoryList == null) {
-      maintenanceHistoryList = new ArrayList<>();
-      _record.setListField(MaintenanceConfigKey.MAINTENANCE_HISTORY.name(), maintenanceHistoryList);
-    }
-
-    // Keep only the last MAINTENANCE_HISTORY_SIZE entries
-    while (maintenanceHistoryList.size() >= MAINTENANCE_HISTORY_SIZE) {
-      maintenanceHistoryList.remove(0);
-    }
-
     DateFormat df = new SimpleDateFormat("yyyy-MM-dd-HH:" + "mm:ss");
     df.setTimeZone(TimeZone.getTimeZone("UTC"));
     String dateTime = df.format(new Date(currentTime));
@@ -189,7 +209,43 @@ public class ControllerHistory extends HelixProperty {
         }
       }
     }
-    maintenanceHistoryList.add(new ObjectMapper().writeValueAsString(maintenanceEntry));
+
+    return populateHistoryEntries(HistoryType.MAINTENANCE,
+        new ObjectMapper().writeValueAsString(maintenanceEntry));
+  }
+
+  private ZNRecord populateHistoryEntries(HistoryType type, String entry) {
+    String configKey;
+    int historySize;
+    switch (type) {
+      case CONTROLLER_LEADERSHIP:
+        configKey = ConfigProperty.HISTORY.name();
+        historySize = HISTORY_SIZE;
+        break;
+      case MAINTENANCE:
+        configKey = MaintenanceConfigKey.MAINTENANCE_HISTORY.name();
+        historySize = MAINTENANCE_HISTORY_SIZE;
+        break;
+      case MANAGEMENT_MODE:
+        configKey = ManagementModeConfigKey.MANAGEMENT_MODE_HISTORY.name();
+        historySize = HISTORY_SIZE;
+        break;
+      default:
+        throw new HelixException("Unknown history type " + type.name());
+    }
+
+    List<String> historyList = _record.getListField(configKey);
+    if (historyList == null) {
+      historyList = new ArrayList<>();
+      _record.setListField(configKey, historyList);
+    }
+
+    while (historyList.size() >= historySize) {
+      historyList.remove(0);
+    }
+
+    historyList.add(entry);
+
     return _record;
   }
 
diff --git a/helix-core/src/test/java/org/apache/helix/TestPropertyPathBuilder.java b/helix-core/src/test/java/org/apache/helix/TestPropertyPathBuilder.java
index 422fb9c..9212568 100644
--- a/helix-core/src/test/java/org/apache/helix/TestPropertyPathBuilder.java
+++ b/helix-core/src/test/java/org/apache/helix/TestPropertyPathBuilder.java
@@ -19,6 +19,7 @@ package org.apache.helix;
  * under the License.
  */
 
+import org.testng.Assert;
 import org.testng.AssertJUnit;
 import org.testng.annotations.Test;
 
@@ -56,5 +57,7 @@ public class TestPropertyPathBuilder {
     actual = PropertyPathBuilder.controllerMessage("test_cluster");
     AssertJUnit.assertEquals(actual, "/test_cluster/CONTROLLER/MESSAGES");
 
+    actual = PropertyPathBuilder.clusterStatus("test_cluster");
+    Assert.assertEquals(actual, "/test_cluster/STATUS/CLUSTER/test_cluster");
   }
 }
diff --git a/helix-core/src/test/java/org/apache/helix/model/TestControllerHistoryModel.java
b/helix-core/src/test/java/org/apache/helix/model/TestControllerHistoryModel.java
new file mode 100644
index 0000000..2e0bab3
--- /dev/null
+++ b/helix-core/src/test/java/org/apache/helix/model/TestControllerHistoryModel.java
@@ -0,0 +1,93 @@
+package org.apache.helix.model;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import com.google.common.base.Splitter;
+import org.apache.helix.TestHelper;
+import org.apache.helix.api.status.ClusterManagementMode;
+import org.apache.helix.zookeeper.zkclient.NetworkUtil;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+public class TestControllerHistoryModel {
+  @Test
+  public void testManagementModeHistory() {
+    ControllerHistory controllerHistory = new ControllerHistory("HISTORY");
+    String controller = "controller-0";
+    ClusterManagementMode mode = new ClusterManagementMode(ClusterManagementMode.Type.CLUSTER_PAUSE,
+        ClusterManagementMode.Status.COMPLETED);
+    long time = System.currentTimeMillis();
+    String fromHost = NetworkUtil.getLocalhostName();
+    String reason = TestHelper.getTestMethodName();
+    controllerHistory.updateManagementModeHistory(controller, mode, fromHost, time, reason);
+
+    List<String> historyList = controllerHistory.getManagementModeHistory();
+    String lastHistory = historyList.get(historyList.size() - 1);
+    Map<String, String> historyMap = stringToMap(lastHistory);
+
+    Map<String, String> expectedMap = new HashMap<>();
+    expectedMap.put("CONTROLLER", controller);
+    expectedMap.put("TIME", Instant.ofEpochMilli(time).toString());
+    expectedMap.put("MODE", mode.getMode().name());
+    expectedMap.put("STATUS", mode.getStatus().name());
+    expectedMap.put(PauseSignal.PauseSignalProperty.FROM_HOST.name(), fromHost);
+    expectedMap.put(PauseSignal.PauseSignalProperty.REASON.name(), reason);
+
+    Assert.assertEquals(historyMap, expectedMap);
+
+    // Add more than 10 entries, it should only keep the latest 10.
+    List<String> reasonList = new ArrayList<>();
+    for (int i = 0; i < 15; i++) {
+      String reasonI = reason + "-" + i;
+      controllerHistory.updateManagementModeHistory(controller, mode, fromHost, time, reasonI);
+      reasonList.add(reasonI);
+    }
+
+    historyList = controllerHistory.getManagementModeHistory();
+
+    Assert.assertEquals(historyList.size(), 10);
+
+    // Assert the history is the latest 10 entries.
+    int i = 5;
+    for (String entry : historyList) {
+      Map<String, String> actual = stringToMap(entry);
+      Assert.assertEquals(actual.get(PauseSignal.PauseSignalProperty.REASON.name()),
+          reasonList.get(i++));
+    }
+  }
+
+  /**
+   * Performs conversion from a map string into a map. The string was converted by map's
toString().
+   *
+   * @param mapAsString A string that is converted by map's toString() method.
+   *                    Example: "{k1=v1, k2=v2}"
+   * @return Map<String, String>
+   */
+  private static Map<String, String> stringToMap(String mapAsString) {
+    return Splitter.on(", ").withKeyValueSeparator('=')
+        .split(mapAsString.substring(1, mapAsString.length() - 1));
+  }
+}

Mime
View raw message