fluo-notifications mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From GitBox <...@apache.org>
Subject [GitHub] mikewalch closed pull request #223: Several updates to Muchos
Date Fri, 13 Jul 2018 16:21:30 GMT
mikewalch closed pull request #223: Several updates to Muchos
URL: https://github.com/apache/fluo-muchos/pull/223
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/conf/muchos.props.example b/conf/muchos.props.example
index c23439a..b729350 100644
--- a/conf/muchos.props.example
+++ b/conf/muchos.props.example
@@ -49,8 +49,13 @@ fluo_yarn_sha256 = c6220d35cf23127272f3b5638c44586504dc17a46f5beecdfee5027b5ff87
 accumulo_sha256 = f9cebff3ff85cacb8c80263725663b047ef239916cb4490c93c62509d62e1e76
 
 [ec2]
+# AWS machine image to use. The default below is for a CentOS 7 image (in us-east-1).
+# You may need to change this value if a new image has been released or you are running in
a different region.
+# Before using this AMI, subscribe to it on the CentOS product page below or launching will
fail:
+#   https://aws.amazon.com/marketplace/pp/B00O7WM7QW
+aws_ami = ami-9887c6e7
 # Type of AWS instance launched by default
-default_instance_type = m3.large
+default_instance_type = m5d.large
 # Type of AWS instance launched for any node running 'worker' service
 # Leave default below to use same instance type set by 'default_instance_type' property
 worker_instance_type = %(default_instance_type)s
@@ -71,9 +76,6 @@ fstype = ext3
 force_format = no
 # Tags to add instances
 #instance_tags = key1:value1,key2:value2
-# Overrides the default CentOS 7 AWS AMIs used by Muchos
-# Only set if you want to use a custom AMI (which should be derived from a CentOS 7 AMI)
-#aws_ami = ami-xxxxxxxx
 # Nodes will be given public IP addresses if true
 associate_public_ip = true
 # Path to file containing user data that will be executed at launch
diff --git a/lib/muchos/config.py b/lib/muchos/config.py
index 96b5b6f..7196aa0 100644
--- a/lib/muchos/config.py
+++ b/lib/muchos/config.py
@@ -14,7 +14,7 @@
 
 from ConfigParser import ConfigParser
 from sys import exit
-from util import get_ephemeral_devices, get_arch, get_ami
+from util import get_ephemeral_devices, get_arch
 import os
 
 SERVICES = ['zookeeper', 'namenode', 'resourcemanager', 'accumulomaster', 'mesosmaster',
'worker', 'fluo', 'fluo_yarn', 'metrics']
@@ -52,9 +52,9 @@ def verify_config(self, action):
                     if not self.has_service(service):
                         exit("ERROR - Missing '{0}' service from [nodes] section of muchos.props".format(service))
 
-    def verify_launch(self, region):
-        self.get_image_id(self.get('ec2', 'default_instance_type'), region)
-        self.get_image_id(self.get('ec2', 'worker_instance_type'), region)
+    def verify_launch(self):
+        self.verify_instance_type(self.get('ec2', 'default_instance_type'))
+        self.verify_instance_type(self.get('ec2', 'worker_instance_type'))
 
     def init_nodes(self):
         self.node_d = {}
@@ -121,11 +121,10 @@ def version(self, software_id):
     def sha256(self, software_id):
         return self.get('general', software_id + '_sha256')
 
-    def get_image_id(self, instance_type, region):
+    def verify_instance_type(self, instance_type):
         if get_arch(instance_type) == 'pvm':
             exit("ERROR - Configuration contains instance type '{0}' that uses pvm architecture."
                  "Only hvm architecture is supported!".format(instance_type))
-        return get_ami(region)
 
     def instance_tags(self):
         retd = {}
diff --git a/lib/muchos/main.py b/lib/muchos/main.py
index 97d205a..26c7025 100644
--- a/lib/muchos/main.py
+++ b/lib/muchos/main.py
@@ -56,15 +56,13 @@ def launch_node(self, hostname, services, sg_id):
         request['InstanceType'] = instance_type
         request['InstanceInitiatedShutdownBehavior'] = self.config.get('ec2', 'shutdown_behavior')
 
-        if self.config.has_option('ec2', 'aws_ami'):
-            image_id = self.config.get('ec2', 'aws_ami')
-        else:
-            session = boto3.session.Session()
-            image_id = self.config.get_image_id(instance_type, session.region_name)
+        if not self.config.has_option('ec2', 'aws_ami'):
+            exit('aws_ami property must be set!')
+        image_id = self.config.get('ec2', 'aws_ami')
         if not image_id:
-            exit('ERROR - Image not found for instance type: '+instance_type)
-        request['ImageId'] = image_id
+            exit('aws_ami property was not properly')
 
+        request['ImageId'] = image_id
         request['BlockDeviceMappings'] = get_block_device_map(instance_type)
 
         if self.config.has_option('ec2', 'key_name'):
@@ -155,8 +153,7 @@ def launch(self):
             exit("ERROR - A hosts file already exists at {0}.  Please delete before running
launch again"
                  .format(self.config.hosts_path))
 
-        session = boto3.session.Session()
-        self.config.verify_launch(session.region_name)
+        self.config.verify_launch()
 
         print "Launching {0} cluster".format(self.config.cluster_name)
 
diff --git a/lib/muchos/util.py b/lib/muchos/util.py
index 86811cb..1ddea45 100644
--- a/lib/muchos/util.py
+++ b/lib/muchos/util.py
@@ -33,15 +33,10 @@ def __init__(self, arch, ephemeral=1, has_nvme=False):
 are no longer available to new users.  If you think this is the case, go to the CentOS 7
product
 page on AWS Marketplace at the URL below to find the latest AMI:
 
-https://aws.amazon.com/marketplace/ordering?productId=b7ee8a69-ee97-4a49-9e68-afaee216db2e
+    https://aws.amazon.com/marketplace/pp/B00O7WM7QW
 
-On the product page, click 'Manual Launch' to find the latest AMI ID for your EC2 region.
-This should be used to set the 'aws_ami' property in your muchos.props which will override
-the default AMI IDs used by Muchos.  After setting the 'aws_ami' property, run the launch
-command again.
-
-Also, let us know that this has occured by creating an issue on the Muchos's GitHub page
-and we'll upgrade the defaults AMIs used by Muchos to be the latest CentOS images.
+On the product page, find the latest AMI ID for your EC2 region. This should be used to set
the 'aws_ami'
+property in your muchos.props.  After setting the 'aws_ami' property, run the launch command
again.
 """
 
 instance_types = {
@@ -76,6 +71,12 @@ def __init__(self, arch, ephemeral=1, has_nvme=False):
     "m3.large": EC2Type("hvm"),
     "m3.medium": EC2Type("hvm"),
     "m3.xlarge": EC2Type("hvm", 2),
+    "m5d.large": EC2Type("hvm", 1, True),
+    "m5d.xlarge": EC2Type("hvm", 1, True),
+    "m5d.2xlarge": EC2Type("hvm", 1, True),
+    "m5d.4xlarge": EC2Type("hvm", 2, True),
+    "m5d.12xlarge": EC2Type("hvm", 2, True),
+    "m5d.24xlarge": EC2Type("hvm", 4, True),
     "r3.2xlarge": EC2Type("hvm", 1),
     "r3.4xlarge": EC2Type("hvm", 1),
     "r3.8xlarge": EC2Type("hvm", 2),
@@ -87,24 +88,6 @@ def __init__(self, arch, ephemeral=1, has_nvme=False):
     "d2.8xlarge": EC2Type("hvm", 24)
 }
 
-# AMI given region for HVM arch.  PVM arch is not supported.
-ami_lookup = {
-    "us-east-1": "ami-4bf3d731",
-    "us-east-2": "ami-e1496384",
-    "us-west-1": "ami-65e0e305",
-    "us-west-2": "ami-a042f4d8",
-    "ca-central-1": "ami-dcad28b8",
-    "eu-central-1": "ami-337be65c",
-    "eu-west-1": "ami-6e28b517",
-    "eu-west-2": "ami-ee6a718a",
-    "eu-west-3": "ami-bfff49c2",
-    "ap-northeast-1": "ami-25bd2743",
-    "ap-northeast-2": "ami-7248e81c",
-    "ap-southeast-1": "ami-d2fa88ae",
-    "ap-southeast-2": "ami-b6bb47d4",
-    "ap-south-1": "ami-5d99ce32",
-    "sa-east-1": "ami-f9adef95"
-}
 
 def verify_type(instance_type):
     if instance_type not in instance_types:
@@ -125,7 +108,11 @@ def get_ephemeral_devices(instance_type):
     devices = []
     ec2_type = instance_types.get(instance_type)
 
-    for i in range(0, ec2_type.ephemeral):
+    start = 0
+    if instance_type.startswith("m5d"):
+        start = 1
+
+    for i in range(start, ec2_type.ephemeral + start):
         if ec2_type.has_nvme:
             devices.append('/dev/nvme' + str(i) + 'n1')
         else:
@@ -148,9 +135,6 @@ def get_block_device_map(instance_type):
 
     return bdm
 
-def get_ami(region):
-    return ami_lookup.get(region)
-
 def parse_args(hosts_dir, input_args=None):
     parser = OptionParser(
               usage="muchos [options] <action>\n\n"
diff --git a/lib/tests/test_config.py b/lib/tests/test_config.py
index 717048e..6ef2f32 100644
--- a/lib/tests/test_config.py
+++ b/lib/tests/test_config.py
@@ -18,12 +18,13 @@
 def test_defaults():
     c = DeployConfig("muchos", '../conf/muchos.props.example', '../conf/hosts/example/example_cluster',
                      'mycluster')
-    assert c.get('ec2', 'default_instance_type') == 'm3.large'
-    assert c.get('ec2', 'worker_instance_type') == 'm3.large'
+    assert c.get('ec2', 'default_instance_type') == 'm5d.large'
+    assert c.get('ec2', 'worker_instance_type') == 'm5d.large'
+    assert c.get('ec2', 'aws_ami') == 'ami-9887c6e7'
     assert c.max_ephemeral() == 1
     assert c.mounts(2) == ['/media/ephemeral0', '/media/ephemeral1']
-    assert c.node_type_map() == {'default': {'mounts': ['/media/ephemeral0', ], 'devices':
['/dev/xvdb', ]},
-                                 'worker': {'mounts': ['/media/ephemeral0', ], 'devices':
['/dev/xvdb', ]}}
+    assert c.node_type_map() == {'default': {'mounts': ['/media/ephemeral0', ], 'devices':
['/dev/nvme1n1', ]},
+                                 'worker': {'mounts': ['/media/ephemeral0', ], 'devices':
['/dev/nvme1n1', ]}}
     assert c.node_type('worker1') == 'worker'
     assert c.node_type('leader1') == 'default'
     assert not c.has_option('ec2', 'vpc_id')
@@ -59,7 +60,6 @@ def test_defaults():
     assert c.get_host_services() == [('leader1', 'namenode zookeeper fluo fluo_yarn'), ('leader2',
'resourcemanager zookeeper'),
                                      ('leader3', 'accumulomaster zookeeper'), ('metrics',
'metrics'),
                                      ('worker1', 'worker'), ('worker2', 'worker'), ('worker3',
'worker')]
-    assert c.get_image_id('m3.large', 'us-east-1') == 'ami-4bf3d731'
 
 
 def test_case_sensitive():
diff --git a/lib/tests/test_util.py b/lib/tests/test_util.py
index 6c6015d..9fc3c7c 100644
--- a/lib/tests/test_util.py
+++ b/lib/tests/test_util.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from muchos.util import get_arch, parse_args, get_ami, get_ephemeral_devices
+from muchos.util import get_arch, parse_args, get_ephemeral_devices
 
 
 def test_util():
@@ -25,8 +25,6 @@ def test_util():
     assert get_arch('m1.large') == 'pvm'
     assert get_arch('m3.large') == 'hvm'
 
-    assert get_ami('us-east-1') == 'ami-4bf3d731'
-
     hosts_dir = '../conf/hosts'
     assert parse_args(hosts_dir, ['launch']) is None
     assert parse_args(hosts_dir, ['launch', 'mycluster']) is None


 

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services

Mime
View raw message