helix-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From j...@apache.org
Subject [helix] branch helix-cloud updated: Change the cluster creation logic (#872)
Date Fri, 06 Mar 2020 21:15:35 GMT
This is an automated email from the ASF dual-hosted git repository.

jxue pushed a commit to branch helix-cloud
in repository https://gitbox.apache.org/repos/asf/helix.git


The following commit(s) were added to refs/heads/helix-cloud by this push:
     new ba7ced1  Change the cluster creation logic (#872)
ba7ced1 is described below

commit ba7ced120d15a194ad8a11fc713fb425e1c0f760
Author: Ali Reza Zamani Zadeh Najari <anajari@linkedin.com>
AuthorDate: Fri Mar 6 13:15:26 2020 -0800

    Change the cluster creation logic (#872)
    
    * Change the cluster creation logic
    
    In this commit, minor modifications have been added which:
    1- Update topology information in the Cluster Config
    if cloud is enabled for Azure environment.
    2- Tests have been change accordingly.
    3- The deprecated APIs have been changed in the test to
    follow new APIs.
---
 .../org/apache/helix/cloud/azure/AzureConstants.java     |  6 ++++++
 .../main/java/org/apache/helix/tools/ClusterSetup.java   | 13 ++++++++++++-
 .../java/org/apache/helix/tools/TestClusterSetup.java    | 14 +++++++++++---
 .../apache/helix/rest/server/TestClusterAccessor.java    | 16 +++++++++++-----
 4 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureConstants.java b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureConstants.java
new file mode 100644
index 0000000..b93f345
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureConstants.java
@@ -0,0 +1,6 @@
+package org.apache.helix.cloud.azure;
+
+public class AzureConstants {
+  public static final String AZURE_TOPOLOGY = "/faultDomain/hostname";
+  public static final String AZURE_FAULT_ZONE_TYPE = "faultDomain";
+}
\ No newline at end of file
diff --git a/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java b/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java
index d1dc6d3..a8969bc 100644
--- a/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java
+++ b/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java
@@ -37,12 +37,13 @@ import org.apache.commons.cli.OptionBuilder;
 import org.apache.commons.cli.OptionGroup;
 import org.apache.commons.cli.Options;
 import org.apache.commons.cli.ParseException;
+import org.apache.helix.ConfigAccessor;
 import org.apache.helix.HelixAdmin;
 import org.apache.helix.HelixConstants;
 import org.apache.helix.HelixException;
 import org.apache.helix.PropertyKey.Builder;
-import org.apache.helix.SystemPropertyKeys;
 import org.apache.helix.ZNRecord;
+import org.apache.helix.cloud.azure.AzureConstants;
 import org.apache.helix.cloud.constants.CloudProvider;
 import org.apache.helix.manager.zk.ZKHelixAdmin;
 import org.apache.helix.manager.zk.ZKHelixDataAccessor;
@@ -189,6 +190,16 @@ public class ClusterSetup {
 
     if (cloudConfig != null) {
       _admin.addCloudConfig(clusterName, cloudConfig);
+      // If cloud is enabled and Cloud Provider is Azure, populated the Topology information
in cluster config
+      if (cloudConfig.isCloudEnabled()
+          && cloudConfig.getCloudProvider().equals(CloudProvider.AZURE.name())) {
+        ConfigAccessor configAccessor = new ConfigAccessor(_zkServerAddress);
+        ClusterConfig clusterConfig = new ClusterConfig(clusterName);
+        clusterConfig.setTopology(AzureConstants.AZURE_TOPOLOGY);
+        clusterConfig.setTopologyAwareEnabled(true);
+        clusterConfig.setFaultZoneType(AzureConstants.AZURE_FAULT_ZONE_TYPE);
+        configAccessor.updateClusterConfig(clusterName, clusterConfig);
+      }
     }
   }
 
diff --git a/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java b/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java
index 68195bd..0a1664b 100644
--- a/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java
+++ b/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java
@@ -34,12 +34,14 @@ import org.apache.helix.PropertyPathBuilder;
 import org.apache.helix.TestHelper;
 import org.apache.helix.ZNRecord;
 import org.apache.helix.ZkUnitTestBase;
+import org.apache.helix.cloud.azure.AzureConstants;
 import org.apache.helix.cloud.constants.CloudProvider;
 import org.apache.helix.manager.zk.ZKHelixAdmin;
 import org.apache.helix.manager.zk.ZKHelixDataAccessor;
 import org.apache.helix.manager.zk.ZNRecordSerializer;
 import org.apache.helix.manager.zk.ZkBaseDataAccessor;
 import org.apache.helix.model.CloudConfig;
+import org.apache.helix.model.ClusterConfig;
 import org.apache.helix.model.HelixConfigScope.ConfigScopeProperty;
 import org.apache.helix.model.IdealState;
 import org.apache.helix.model.LiveInstance;
@@ -355,7 +357,7 @@ public class TestClusterSetup extends ZkUnitTestBase {
 
     // add fake liveInstance
     ZKHelixDataAccessor accessor =
-        new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_gZkClient));
+        new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(ZK_ADDR));
     Builder keyBuilder = new Builder(clusterName);
     LiveInstance liveInstance = new LiveInstance("localhost_12918");
     liveInstance.setSessionId("session_0");
@@ -421,7 +423,7 @@ public class TestClusterSetup extends ZkUnitTestBase {
     ClusterSetup.processCommandLineArgs(new String[] {
         "--zkSvr", ZK_ADDR, "--enableResource", clusterName, "TestDB0", "false"
     });
-    BaseDataAccessor<ZNRecord> baseAccessor = new ZkBaseDataAccessor<ZNRecord>(_gZkClient);
+    BaseDataAccessor<ZNRecord> baseAccessor = new ZkBaseDataAccessor<ZNRecord>(ZK_ADDR);
     HelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, baseAccessor);
     PropertyKey.Builder keyBuilder = accessor.keyBuilder();
     IdealState idealState = accessor.getProperty(keyBuilder.idealStates("TestDB0"));
@@ -505,12 +507,18 @@ public class TestClusterSetup extends ZkUnitTestBase {
     _clusterSetup.addCluster(clusterName, false, cloudConfigInit);
 
     // Read CloudConfig from Zookeeper and check the content
-    ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+    ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
     CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
     Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
     Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestID");
     List<String> listUrlFromZk = cloudConfigFromZk.getCloudInfoSources();
 
+    // Since it is Azure, topology information should have been populated.
+    ClusterConfig clusterConfig = _configAccessor.getClusterConfig(clusterName);
+    Assert.assertEquals(clusterConfig.getTopology(), AzureConstants.AZURE_TOPOLOGY);
+    Assert.assertEquals(clusterConfig.getFaultZoneType(), AzureConstants.AZURE_FAULT_ZONE_TYPE);
+    Assert.assertTrue(clusterConfig.isTopologyAwareEnabled());
+
     // Since provider is not customized, CloudInfoSources and CloudInfoProcessorName will
be null.
     Assert.assertNull(listUrlFromZk);
     Assert.assertNull(cloudConfigFromZk.getCloudInfoProcessorName());
diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
index 4ea2d79..21a2853 100644
--- a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
+++ b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
@@ -38,6 +38,7 @@ import org.apache.helix.HelixDataAccessor;
 import org.apache.helix.PropertyKey;
 import org.apache.helix.TestHelper;
 import org.apache.helix.ZNRecord;
+import org.apache.helix.cloud.azure.AzureConstants;
 import org.apache.helix.controller.rebalancer.DelayedAutoRebalancer;
 import org.apache.helix.controller.rebalancer.strategy.CrushEdRebalanceStrategy;
 import org.apache.helix.integration.manager.ClusterDistributedController;
@@ -587,11 +588,16 @@ public class TestClusterAccessor extends AbstractTestClass {
         Response.Status.CREATED.getStatusCode());
 
     // Read CloudConfig from Zookeeper and check the content
-    ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+    ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
     CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
     Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
     Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
     Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.AZURE.name());
+
+    ClusterConfig clusterConfigFromZk = _configAccessor.getClusterConfig(clusterName);
+    Assert.assertEquals(clusterConfigFromZk.getTopology(), AzureConstants.AZURE_TOPOLOGY);
+    Assert.assertEquals(clusterConfigFromZk.getFaultZoneType(), AzureConstants.AZURE_FAULT_ZONE_TYPE);
+    Assert.assertTrue(clusterConfigFromZk.isTopologyAwareEnabled());
   }
 
   @Test(dependsOnMethods = "testAddClusterWithCloudConfig")
@@ -662,7 +668,7 @@ public class TestClusterAccessor extends AbstractTestClass {
         Response.Status.CREATED.getStatusCode());
 
     // Read CloudConfig from Zookeeper and check the content
-    ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+    ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
     CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
     Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
     Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
@@ -692,7 +698,7 @@ public class TestClusterAccessor extends AbstractTestClass {
         Response.Status.CREATED.getStatusCode());
 
     // Read CloudConfig from Zookeeper and check the content
-    ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+    ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
     CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
     Assert.assertFalse(cloudConfigFromZk.isCloudEnabled());
     Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
@@ -751,7 +757,7 @@ public class TestClusterAccessor extends AbstractTestClass {
         Response.Status.OK.getStatusCode());
 
     // Read CloudConfig from Zookeeper and check the content
-    ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+    ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
     CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig("TestCloud");
     Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
     Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
@@ -795,7 +801,7 @@ public class TestClusterAccessor extends AbstractTestClass {
         Response.Status.OK.getStatusCode());
 
     // Read CloudConfig from Zookeeper and make sure it has been removed
-    ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+    ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
     CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig("TestCloud");
     Assert.assertNull(cloudConfigFromZk);
 


Mime
View raw message