helix-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kisho...@apache.org
Subject git commit: [HELIX-73] Making the instanceId change backward compatible, if instanceName matches host[:_]port format, extract host and port automatically
Date Sat, 13 Apr 2013 18:08:05 GMT
Updated Branches:
  refs/heads/master c50a1ae8d -> 6b70b7783


[HELIX-73] Making the instanceId change backward compatible, if instanceName matches host[:_]port
format, extract host and port automatically


Project: http://git-wip-us.apache.org/repos/asf/incubator-helix/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-helix/commit/6b70b778
Tree: http://git-wip-us.apache.org/repos/asf/incubator-helix/tree/6b70b778
Diff: http://git-wip-us.apache.org/repos/asf/incubator-helix/diff/6b70b778

Branch: refs/heads/master
Commit: 6b70b77838287dadc2bfc6717f8043bd87a2c0ef
Parents: c50a1ae
Author: kishoreg <kishoreg@apache.org>
Authored: Sat Apr 13 11:07:42 2013 -0700
Committer: kishoreg <kishoreg@apache.org>
Committed: Sat Apr 13 11:07:42 2013 -0700

----------------------------------------------------------------------
 .../helix/webapp/resources/IdealStateResource.java |    2 +-
 .../org/apache/helix/model/InstanceConfig.java     |    4 +-
 .../java/org/apache/helix/tools/ClusterSetup.java  |  138 ++++++++-------
 .../java/org/apache/helix/TestConfigAccessor.java  |    2 +
 .../org/apache/helix/TestHelixConfigAccessor.java  |   12 +-
 .../helix/integration/TestAutoRebalance.java       |    2 +-
 .../apache/helix/filestore/IntegrationTest.java    |   10 +-
 7 files changed, 88 insertions(+), 82 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/6b70b778/helix-admin-webapp/src/main/java/org/apache/helix/webapp/resources/IdealStateResource.java
----------------------------------------------------------------------
diff --git a/helix-admin-webapp/src/main/java/org/apache/helix/webapp/resources/IdealStateResource.java
b/helix-admin-webapp/src/main/java/org/apache/helix/webapp/resources/IdealStateResource.java
index 80645b9..966dc14 100644
--- a/helix-admin-webapp/src/main/java/org/apache/helix/webapp/resources/IdealStateResource.java
+++ b/helix-admin-webapp/src/main/java/org/apache/helix/webapp/resources/IdealStateResource.java
@@ -152,7 +152,7 @@ public class IdealStateResource extends Resource
         String keyPrefix = jsonParameters.getParameter(JsonParameters.RESOURCE_KEY_PREFIX);
         String groupTag = jsonParameters.getParameter(ClusterSetup.instanceGroupTag);
         
-          setupTool.rebalanceStorageCluster(clusterName,
+          setupTool.rebalanceCluster(clusterName,
                                             resourceName,
                                             replicas,
                                             keyPrefix,

http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/6b70b778/helix-core/src/main/java/org/apache/helix/model/InstanceConfig.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/model/InstanceConfig.java b/helix-core/src/main/java/org/apache/helix/model/InstanceConfig.java
index e719357..9ac594a 100644
--- a/helix-core/src/main/java/org/apache/helix/model/InstanceConfig.java
+++ b/helix-core/src/main/java/org/apache/helix/model/InstanceConfig.java
@@ -46,9 +46,9 @@ public class InstanceConfig extends HelixProperty
   }
   private static final Logger _logger = Logger.getLogger(InstanceConfig.class.getName());
 
-  public InstanceConfig(String id)
+  public InstanceConfig(String instanceId)
   {
-    super(id);
+    super(instanceId);
   }
 
   public InstanceConfig(ZNRecord record)

http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/6b70b778/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java b/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java
index 5ccf68d..9356c88 100644
--- a/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java
+++ b/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java
@@ -62,8 +62,6 @@ import org.apache.helix.model.builder.HelixConfigScopeBuilder;
 import org.apache.helix.util.HelixUtil;
 import org.apache.helix.util.ZKClientPool;
 import org.apache.log4j.Logger;
-import org.codehaus.jackson.JsonGenerationException;
-import org.codehaus.jackson.map.JsonMappingException;
 
 
 public class ClusterSetup
@@ -195,46 +193,73 @@ public class ClusterSetup
     _admin.dropCluster(clusterName);
   }
 
-  public void addInstancesToCluster(String clusterName, String[] InstanceInfoArray)
+  public void addInstancesToCluster(String clusterName, String[] instanceInfoArray)
   {
-    for (String InstanceInfo : InstanceInfoArray)
+    for (String instanceInfo : instanceInfoArray)
     {
-      if (InstanceInfo.length() > 0)
+      if (instanceInfo.length() > 0)
       {
-        addInstanceToCluster(clusterName, InstanceInfo);
+        addInstanceToCluster(clusterName, instanceInfo);
       }
     }
   }
 
-  public void addInstanceToCluster(String clusterName, String instanceId)
+  private InstanceConfig toInstanceConfig(String instanceId)
   {
-    if(instanceId.contains(":"))
-    {
-      int lastPos = instanceId.lastIndexOf(":");
-      
-      String host = instanceId.substring(0, lastPos);
-      String portStr = instanceId.substring(lastPos + 1);
-      try
+    String host = null;
+    int port = -1;
+    // to maintain backward compatibility we parse string of format host:port
+    // and host_port, where host port must be of type string and int
+    char[] delims = new char[]
+    { ':', '_' };
+    for (char delim : delims)
+    {
+      String regex = String.format("(.*)[%c]([\\d]+)", delim);
+      if (instanceId.matches(regex))
       {
-        int port = Integer.parseInt(portStr);
-        instanceId = host + "_" + portStr;
+        int lastIndexOf = instanceId.lastIndexOf(delim);
+        try
+        {
+          port = Integer.parseInt(instanceId.substring(lastIndexOf + 1));
+          host = instanceId.substring(0, lastIndexOf);
+        } catch (Exception e)
+        {
+          _logger.warn("Unable to extract host and port from instanceId:"
+              + instanceId);
+        }
+        break;
       }
-      catch(Exception e)
-      {}
+    }
+    if (host != null && port > 0)
+    {
+      instanceId = host + "_" + port;
     }
     InstanceConfig config = new InstanceConfig(instanceId);
+    if (host != null && port > 0)
+    {
+      config.setHostName(host);
+      config.setPort(String.valueOf(port));
+
+    }
+
     config.setInstanceEnabled(true);
     config.setHostName(instanceId);
+    return config;
+  }
+
+  public void addInstanceToCluster(String clusterName, String instanceId)
+  {
+    InstanceConfig config = toInstanceConfig(instanceId);
     _admin.addInstance(clusterName, config);
   }
 
-  public void dropInstancesFromCluster(String clusterName, String[] InstanceInfoArray)
+  public void dropInstancesFromCluster(String clusterName, String[] instanceInfoArray)
   {
-    for (String InstanceInfo : InstanceInfoArray)
+    for (String instanceInfo : instanceInfoArray)
     {
-      if (InstanceInfo.length() > 0)
+      if (instanceInfo.length() > 0)
       {
-        dropInstanceFromCluster(clusterName, InstanceInfo);
+        dropInstanceFromCluster(clusterName, instanceInfo);
       }
     }
   }
@@ -245,20 +270,8 @@ public class ClusterSetup
         new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_zkClient));
     Builder keyBuilder = accessor.keyBuilder();
     
-    if(instanceId.contains(":"))
-    {
-      int lastPos = instanceId.lastIndexOf(":");
-      
-      String host = instanceId.substring(0, lastPos);
-      String portStr = instanceId.substring(lastPos + 1);
-      try
-      {
-        int port = Integer.parseInt(portStr);
-        instanceId = host + "_" + portStr;
-      }
-      catch(Exception e)
-      {}
-    }
+   InstanceConfig instanceConfig = toInstanceConfig(instanceId);
+   instanceId = instanceConfig.getInstanceName();
     
     // ensure node is stopped
     LiveInstance liveInstance = accessor.getProperty(keyBuilder.liveInstance(instanceId));
@@ -291,7 +304,7 @@ public class ClusterSetup
                            String newInstanceName)
   {
     ZKHelixDataAccessor accessor =
-        new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor(_zkClient));
+        new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_zkClient));
     Builder keyBuilder = accessor.keyBuilder();
 
     InstanceConfig oldConfig =
@@ -536,7 +549,7 @@ public class ClusterSetup
   }
   
   
-  public void rebalanceStorageCluster(String clusterName,
+  public void rebalanceCluster(String clusterName,
       String resourceName,
       int replica,
       String keyPrefix, String group)
@@ -674,20 +687,20 @@ public class ClusterSetup
   }
   
   /**
-   * Sets up a cluster with 6 Instances[localhost:8900 to localhost:8905], 1
-   * resource[EspressoDB] with a replication factor of 3
-   * 
+   * Sets up a cluster<br/>
+   * 6 Instances[localhost:8900 to localhost:8905], <br/>
+   * 1 resource[TestDB] with a replication factor of 3 and using MasterSlave state model<br/>
    * @param clusterName
    */
   public void setupTestCluster(String clusterName)
   {
     addCluster(clusterName, true);
-    String storageInstanceInfoArray[] = new String[6];
-    for (int i = 0; i < storageInstanceInfoArray.length; i++)
+    String instanceInfoArray[] = new String[6];
+    for (int i = 0; i < instanceInfoArray.length; i++)
     {
-      storageInstanceInfoArray[i] = "localhost_" + (8900 + i);
+      instanceInfoArray[i] = "localhost_" + (8900 + i);
     }
-    addInstancesToCluster(clusterName, storageInstanceInfoArray);
+    addInstancesToCluster(clusterName, instanceInfoArray);
     addResourceToCluster(clusterName, "TestDB", 10, "MasterSlave");
     rebalanceStorageCluster(clusterName, "TestDB", 3);
   }
@@ -1213,9 +1226,9 @@ public class ClusterSetup
     if (cmd.hasOption(addInstance))
     {
       String clusterName = cmd.getOptionValues(addInstance)[0];
-      String InstanceAddressInfo = cmd.getOptionValues(addInstance)[1];
-      String[] InstanceAddresses = InstanceAddressInfo.split(";");
-      setupTool.addInstancesToCluster(clusterName, InstanceAddresses);
+      String instanceAddressInfo = cmd.getOptionValues(addInstance)[1];
+      String[] instanceAddresses = instanceAddressInfo.split(";");
+      setupTool.addInstancesToCluster(clusterName, instanceAddresses);
       return 0;
     }
 
@@ -1267,7 +1280,7 @@ public class ClusterSetup
       {
         instanceGroupTagVal = cmd.getOptionValue(instanceGroupTag);
       }
-      setupTool.rebalanceStorageCluster(clusterName, resourceName, replicas, keyPrefixVal,
instanceGroupTagVal);
+      setupTool.rebalanceCluster(clusterName, resourceName, replicas, keyPrefixVal, instanceGroupTagVal);
       return 0;
     }
 
@@ -1290,9 +1303,9 @@ public class ClusterSetup
     if (cmd.hasOption(dropInstance))
     {
       String clusterName = cmd.getOptionValues(dropInstance)[0];
-      String InstanceAddressInfo = cmd.getOptionValues(dropInstance)[1];
-      String[] InstanceAddresses = InstanceAddressInfo.split(";");
-      setupTool.dropInstancesFromCluster(clusterName, InstanceAddresses);
+      String instanceAddressInfo = cmd.getOptionValues(dropInstance)[1];
+      String[] instanceAddresses = instanceAddressInfo.split(";");
+      setupTool.dropInstancesFromCluster(clusterName, instanceAddresses);
       return 0;
     }
 
@@ -1326,7 +1339,7 @@ public class ClusterSetup
       String clusterName = cmd.getOptionValue(listClusterInfo);
       List<String> resourceNames =
           setupTool.getClusterManagementTool().getResourcesInCluster(clusterName);
-      List<String> Instances =
+      List<String> instances =
           setupTool.getClusterManagementTool().getInstancesInCluster(clusterName);
 
       System.out.println("Existing resources in cluster " + clusterName + ":");
@@ -1336,7 +1349,7 @@ public class ClusterSetup
       }
 
       System.out.println("Instances in cluster " + clusterName + ":");
-      for (String InstanceName : Instances)
+      for (String InstanceName : instances)
       {
         System.out.println(InstanceName);
       }
@@ -1345,13 +1358,13 @@ public class ClusterSetup
     else if (cmd.hasOption(listInstances))
     {
       String clusterName = cmd.getOptionValue(listInstances);
-      List<String> Instances =
+      List<String> instances =
           setupTool.getClusterManagementTool().getInstancesInCluster(clusterName);
 
       System.out.println("Instances in cluster " + clusterName + ":");
-      for (String InstanceName : Instances)
+      for (String instanceName : instances)
       {
-        System.out.println(InstanceName);
+        System.out.println(instanceName);
       }
       return 0;
     }
@@ -1766,19 +1779,14 @@ public class ClusterSetup
   /**
    * @param args
    * @throws Exception
-   * @throws JsonMappingException
-   * @throws JsonGenerationException
    */
   public static void main(String[] args) throws Exception
   {
-    // debug
-    // System.out.println("args(" + args.length + "): " + Arrays.asList(args));
-
+    System.exit(1);
     if (args.length == 1 && args[0].equals("setup-test-cluster"))
     {
-      System.out.println("By default setting up ");
-      new ClusterSetup("localhost:2181").setupTestCluster("storage-integration-cluster");
-      new ClusterSetup("localhost:2181").setupTestCluster("relay-integration-cluster");
+      System.out.println("By default setting up TestCluster with 6 instances, 10 partitions,
Each partition will have 3 replicas");
+      new ClusterSetup("localhost:2181").setupTestCluster("TestCluster");
       System.exit(0);
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/6b70b778/helix-core/src/test/java/org/apache/helix/TestConfigAccessor.java
----------------------------------------------------------------------
diff --git a/helix-core/src/test/java/org/apache/helix/TestConfigAccessor.java b/helix-core/src/test/java/org/apache/helix/TestConfigAccessor.java
index 882a5bb..e03620a 100644
--- a/helix-core/src/test/java/org/apache/helix/TestConfigAccessor.java
+++ b/helix-core/src/test/java/org/apache/helix/TestConfigAccessor.java
@@ -19,6 +19,7 @@ package org.apache.helix;
  * under the License.
  */
 
+import java.util.Arrays;
 import java.util.Date;
 import java.util.List;
 
@@ -104,6 +105,7 @@ public class TestConfigAccessor extends ZkUnitTestBase
     Assert.assertEquals(keys.get(0), "clusterConfigKey");
 
     keys = configAccessor.getKeys(ConfigScopeProperty.PARTICIPANT, clusterName, "localhost_12918");
+    System.out.println((keys));
     Assert.assertEquals(keys.size(), 4, "should be [HELIX_ENABLED, HELIX_HOST, HELIX_PORT,
participantConfigKey]");
     Assert.assertEquals(keys.get(3), "participantConfigKey");
 

http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/6b70b778/helix-core/src/test/java/org/apache/helix/TestHelixConfigAccessor.java
----------------------------------------------------------------------
diff --git a/helix-core/src/test/java/org/apache/helix/TestHelixConfigAccessor.java b/helix-core/src/test/java/org/apache/helix/TestHelixConfigAccessor.java
index 7672726..3bbc4d3 100644
--- a/helix-core/src/test/java/org/apache/helix/TestHelixConfigAccessor.java
+++ b/helix-core/src/test/java/org/apache/helix/TestHelixConfigAccessor.java
@@ -110,8 +110,8 @@ public class TestHelixConfigAccessor extends ZkUnitTestBase {
                                     .forCluster(clusterName)
                                     .build());
     Assert.assertEquals(keys.size(), 5, "should be [localhost_12918~22] sorted");
-    Assert.assertEquals(keys.get(0), "localhost_12918");
-    Assert.assertEquals(keys.get(4), "localhost_12922");
+    Assert.assertTrue(keys.contains("localhost_12918"));
+    Assert.assertTrue(keys.contains("localhost_12922"));
 
     keys = configAccessor.getKeys(new HelixConfigScopeBuilder(ConfigScopeProperty.PARTITION)
                                       .forCluster(clusterName)
@@ -125,20 +125,20 @@ public class TestHelixConfigAccessor extends ZkUnitTestBase {
                                       .forResource("testResource")
                                       .build());
     Assert.assertEquals(keys.size(), 1, "should be [resourceConfigKey]");
-    Assert.assertEquals(keys.get(0), "resourceConfigKey");
+    Assert.assertTrue(keys.contains("resourceConfigKey"));
 
     keys = configAccessor.getKeys(new HelixConfigScopeBuilder(ConfigScopeProperty.CLUSTER)
                                       .forCluster(clusterName)
                                       .build());
     Assert.assertEquals(keys.size(), 1, "should be [clusterConfigKey]");
-    Assert.assertEquals(keys.get(0), "clusterConfigKey");
+    Assert.assertTrue(keys.contains("clusterConfigKey"));
 
     keys = configAccessor.getKeys(new HelixConfigScopeBuilder(ConfigScopeProperty.PARTICIPANT)
                                       .forCluster(clusterName)
                                       .forParticipant("localhost_12918")
                                       .build());
-    Assert.assertEquals(keys.size(), 3, "should be [HELIX_ENABLED, HELIX_HOST, participantConfigKey]");
-    Assert.assertEquals(keys.get(2), "participantConfigKey");
+    Assert.assertEquals(keys.size(), 4, "should be [HELIX_ENABLED, HELIX_PORT, HELIX_HOST,
participantConfigKey]");
+    Assert.assertTrue(keys.contains("participantConfigKey"));
 
     keys = configAccessor.getKeys(new HelixConfigScopeBuilder(ConfigScopeProperty.PARTITION)
                                       .forCluster(clusterName)

http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/6b70b778/helix-core/src/test/java/org/apache/helix/integration/TestAutoRebalance.java
----------------------------------------------------------------------
diff --git a/helix-core/src/test/java/org/apache/helix/integration/TestAutoRebalance.java
b/helix-core/src/test/java/org/apache/helix/integration/TestAutoRebalance.java
index 7cf0477..0671971 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/TestAutoRebalance.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/TestAutoRebalance.java
@@ -89,7 +89,7 @@ public class TestAutoRebalance extends ZkStandAloneCMTestBaseWithPropertyServerC
       _setupTool.getClusterManagementTool().addInstanceTag(CLUSTER_NAME, storageNodeName,
_tag);
     }
 
-    _setupTool.rebalanceStorageCluster(CLUSTER_NAME, db2, 1, "ucpx",_tag);
+    _setupTool.rebalanceCluster(CLUSTER_NAME, db2, 1, "ucpx",_tag);
     
     // start dummy participants
     for (int i = 0; i < NODE_NR; i++)

http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/6b70b778/recipes/rsync-replicated-file-system/src/main/java/org/apache/helix/filestore/IntegrationTest.java
----------------------------------------------------------------------
diff --git a/recipes/rsync-replicated-file-system/src/main/java/org/apache/helix/filestore/IntegrationTest.java
b/recipes/rsync-replicated-file-system/src/main/java/org/apache/helix/filestore/IntegrationTest.java
index d9192c2..37b1376 100644
--- a/recipes/rsync-replicated-file-system/src/main/java/org/apache/helix/filestore/IntegrationTest.java
+++ b/recipes/rsync-replicated-file-system/src/main/java/org/apache/helix/filestore/IntegrationTest.java
@@ -33,14 +33,10 @@ import org.apache.helix.HelixDataAccessor;
 import org.apache.helix.HelixManager;
 import org.apache.helix.PropertyKey.Builder;
 import org.apache.helix.controller.HelixControllerMain;
-import org.apache.helix.model.ConfigScope;
 import org.apache.helix.model.HelixConfigScope;
 import org.apache.helix.model.HelixConfigScope.ConfigScopeProperty;
-import org.apache.helix.model.StateModelDefinition;
-import org.apache.helix.model.builder.ConfigScopeBuilder;
 import org.apache.helix.model.builder.HelixConfigScopeBuilder;
 import org.apache.helix.tools.ClusterSetup;
-import org.apache.helix.tools.StateModelConfigGenerator;
 
 public class IntegrationTest
 {
@@ -75,9 +71,9 @@ public class IntegrationTest
       final String clusterName = "file-store-test";
       setup.deleteCluster(clusterName);
       setup.addCluster(clusterName, true);
-      setup.addInstanceToCluster(clusterName, "localhost", 12001);
-      setup.addInstanceToCluster(clusterName, "localhost", 12002);
-      setup.addInstanceToCluster(clusterName, "localhost", 12003);
+      setup.addInstanceToCluster(clusterName, "localhost_12001");
+      setup.addInstanceToCluster(clusterName, "localhost_12002");
+      setup.addInstanceToCluster(clusterName, "localhost_12003");
       setup.addResourceToCluster(clusterName, "repository", 1, "MasterSlave");
       setup.rebalanceResource(clusterName, "repository", 3);
       // Set the configuration


Mime
View raw message