libcloud-notifications mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From to...@apache.org
Subject [2/5] git commit: Add script for scraping and updating EC2 pricing data.
Date Thu, 10 Apr 2014 19:01:27 GMT
Add script for scraping and updating EC2 pricing data.


Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo
Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/5521b87a
Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/5521b87a
Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/5521b87a

Branch: refs/heads/trunk
Commit: 5521b87afa2ab3df426de797992b8765a6b2806f
Parents: c842f35
Author: Tomaz Muraus <tomaz@apache.org>
Authored: Thu Apr 10 12:04:45 2014 -0400
Committer: Tomaz Muraus <tomaz@apache.org>
Committed: Thu Apr 10 14:34:07 2014 -0400

----------------------------------------------------------------------
 contrib/scrape-ec2-prices.py | 144 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 144 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/libcloud/blob/5521b87a/contrib/scrape-ec2-prices.py
----------------------------------------------------------------------
diff --git a/contrib/scrape-ec2-prices.py b/contrib/scrape-ec2-prices.py
new file mode 100755
index 0000000..bf543f4
--- /dev/null
+++ b/contrib/scrape-ec2-prices.py
@@ -0,0 +1,144 @@
+#!/usr/bin/env python
+#
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing,
+#  software distributed under the License is distributed on an
+#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#  KIND, either express or implied.  See the License for the
+#  specific language governing permissions and limitations
+#  under the License.
+
+import os
+import json
+import time
+from collections import defaultdict, OrderedDict
+
+import requests
+
+ON_DEMAND_LINUX_URL = 'http://aws.amazon.com/ec2/pricing/json/linux-od.json'
+
+EC2_REGIONS = [
+    'us-east-1',
+    'us-west-1',
+    'us-west-2',
+    'eu-west-1',
+    'ap-southeast-1',
+    'ap-southeast-2',
+    'ap-northeast-1',
+    'sa-east-1'
+]
+
+EC2_INSTANCE_TYPES = [
+    't1.micro',
+    'm1.small',
+    'm1.medium',
+    'm1.large',
+    'm1.xlarge',
+    'm2.xlarge',
+    'm2.2xlarge',
+    'm2.4xlarge',
+    'm3.medium',
+    'm3.large',
+    'm3.xlarge',
+    'm3.2xlarge',
+    'c1.medium',
+    'c1.xlarge',
+    'cc1.4xlarge',
+    'cc2.8xlarge',
+    'c3.large',
+    'c3.xlarge',
+    'c3.2xlarge',
+    'c3.4xlarge',
+    'c3.8xlarge',
+    'cg1.4xlarge',
+    'g2.2xlarge',
+    'cr1.8xlarge',
+    'hs1.4xlarge',
+    'hs1.8xlarge',
+    'i2.xlarge',
+    'i2.2xlarge',
+    'i2.4xlarge',
+    'i2.8xlarge',
+]
+
+# Maps EC2 region name to region name used in the pricing file
+REGION_NAME_MAP = {
+    'us-east': 'ec2_us_east',
+    'us-west': 'ec2_us_west',
+    'us-west-2': 'ec2_us_west_oregon',
+    'eu-ireland': 'ec2_eu_west',
+    'apac-sin': 'ec2_ap_southeast',
+    'apac-syd': 'ec2_ap_southeast_2',
+    'apac-tokyo': 'ec2_ap_northeast',
+    'sa-east-1': 'ec2_sa_east',
+}
+
+BASE_PATH = os.path.dirname(os.path.abspath(__file__))
+PRICING_FILE_PATH = os.path.join(BASE_PATH, '../libcloud/data/pricing.json')
+PRICING_FILE_PATH = os.path.abspath(PRICING_FILE_PATH)
+
+
+def scrape_ec2_pricing():
+    response = requests.get(ON_DEMAND_LINUX_URL)
+    data = response.json()
+
+    regions = data['config']['regions']
+
+    result = defaultdict(OrderedDict)
+    for region_data in regions:
+        region_name = region_data['region']
+        libcloud_region_name = REGION_NAME_MAP[region_name]
+        instance_types = region_data['instanceTypes']
+
+        for instance_type in instance_types:
+            sizes = instance_type['sizes']
+
+            for size in sizes:
+                price = size['valueColumns'][0]['prices']['USD']
+                result[libcloud_region_name][size['size']] = price
+
+    return result
+
+
+def update_pricing_file(pricing_file_path, pricing_data):
+    with open(pricing_file_path, 'r') as fp:
+        content = fp.read()
+
+    data = json.loads(content)
+    data['updated'] = int(time.time())
+    data['compute'].update(pricing_data)
+
+    # Always sort the pricing info
+    data = OrderedDict(sorted(data.items()))
+
+    content = json.dumps(data, indent=4)
+    lines = content.splitlines()
+    print lines
+    lines = [line.rstrip() for line in lines]
+    content = '\n'.join(lines)
+
+    with open(pricing_file_path, 'w') as fp:
+        fp.write(content)
+
+
+def main():
+    print('Scraping EC2 pricing data')
+
+    pricing_data = scrape_ec2_pricing()
+    update_pricing_file(pricing_file_path=PRICING_FILE_PATH,
+                        pricing_data=pricing_data)
+
+    print('Pricing data updated')
+
+
+if __name__ == '__main__':
+    main()


Mime
View raw message