libcloud-notifications mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From anthonys...@apache.org
Subject [4/5] libcloud git commit: Retry requests that have been rate-limited in Vultr compute driver
Date Sun, 18 Jun 2017 02:25:51 GMT
Retry requests that have been rate-limited in Vultr compute driver

Closes #1058


Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo
Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/d19574e3
Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/d19574e3
Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/d19574e3

Branch: refs/heads/trunk
Commit: d19574e3464d86c1770ec0b08b4ca520d80a9e66
Parents: e597775
Author: Francisco Ros <fjros@doalitic.com>
Authored: Wed May 24 09:03:10 2017 +0200
Committer: Anthony Shaw <anthonyshaw@apache.org>
Committed: Sun Jun 18 12:24:13 2017 +1000

----------------------------------------------------------------------
 libcloud/compute/drivers/vultr.py               | 51 ++++++++++++++++++--
 .../compute/fixtures/vultr/error_rate_limit.txt |  1 +
 libcloud/test/compute/test_vultr.py             | 11 +++++
 3 files changed, 60 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/libcloud/blob/d19574e3/libcloud/compute/drivers/vultr.py
----------------------------------------------------------------------
diff --git a/libcloud/compute/drivers/vultr.py b/libcloud/compute/drivers/vultr.py
index 7c31dfb..1cabd46 100644
--- a/libcloud/compute/drivers/vultr.py
+++ b/libcloud/compute/drivers/vultr.py
@@ -17,17 +17,58 @@ Vultr Driver
 """
 
 import time
+from functools import update_wrapper
 
 from libcloud.utils.py3 import httplib
 from libcloud.utils.py3 import urlencode
 
 from libcloud.common.base import ConnectionKey, JsonResponse
 from libcloud.compute.types import Provider, NodeState
-from libcloud.common.types import LibcloudError, InvalidCredsError
+from libcloud.common.types import InvalidCredsError
+from libcloud.common.types import LibcloudError
+from libcloud.common.types import ServiceUnavailableError
 from libcloud.compute.base import NodeDriver
 from libcloud.compute.base import Node, NodeImage, NodeSize, NodeLocation
 
 
+class rate_limited:
+    """
+    Decorator for retrying Vultr calls that are rate-limited.
+
+    :param int sleep: Seconds to sleep after being rate-limited.
+    :param int retries: Number of retries.
+    """
+
+    def __init__(self, sleep=1, retries=1):
+        self.sleep = sleep
+        self.retries = retries
+
+    def __call__(self, call):
+        """
+        Run ``call`` method until it's not rate-limited.
+
+        The method is invoked while it returns 503 Service Unavailable or the
+        allowed number of retries is reached.
+
+        :param callable call: Method to be decorated.
+        """
+
+        def wrapper(*args, **kwargs):
+            last_exception = None
+
+            for i in range(self.retries + 1):
+                try:
+                    return call(*args, **kwargs)
+                except ServiceUnavailableError as e:
+                    last_exception = e
+                    time.sleep(self.sleep)  # hit by rate limit, let's sleep
+
+            raise last_exception
+
+        update_wrapper(wrapper, call)
+        return wrapper
+
+
 class VultrResponse(JsonResponse):
     def parse_error(self):
         if self.status == httplib.OK:
@@ -35,6 +76,8 @@ class VultrResponse(JsonResponse):
             return body
         elif self.status == httplib.FORBIDDEN:
             raise InvalidCredsError(self.body)
+        elif self.status == httplib.SERVICE_UNAVAILABLE:
+            raise ServiceUnavailableError(self.body)
         else:
             raise LibcloudError(self.body)
 
@@ -57,7 +100,7 @@ class VultrConnection(ConnectionKey):
 
     host = 'api.vultr.com'
     responseCls = VultrResponse
-    unauthenticated_endpoints = {  # {path: actions}
+    unauthenticated_endpoints = {  # {action: methods}
         '/v1/app/list': ['GET'],
         '/v1/os/list': ['GET'],
         '/v1/plans/list': ['GET'],
@@ -82,9 +125,11 @@ class VultrConnection(ConnectionKey):
     def encode_data(self, data):
         return urlencode(data)
 
+    @rate_limited()
     def get(self, url):
         return self.request(url)
 
+    @rate_limited()
     def post(self, url, data):
         headers = {'Content-Type': 'application/x-www-form-urlencoded'}
         return self.request(url, data=data, headers=headers, method='POST')
@@ -99,7 +144,7 @@ class VultrConnection(ConnectionKey):
 
         try:
             return self.method \
-                   not in self.unauthenticated_endpoints[self.action]
+                not in self.unauthenticated_endpoints[self.action]
         except KeyError:
             return True
 

http://git-wip-us.apache.org/repos/asf/libcloud/blob/d19574e3/libcloud/test/compute/fixtures/vultr/error_rate_limit.txt
----------------------------------------------------------------------
diff --git a/libcloud/test/compute/fixtures/vultr/error_rate_limit.txt b/libcloud/test/compute/fixtures/vultr/error_rate_limit.txt
new file mode 100644
index 0000000..27def76
--- /dev/null
+++ b/libcloud/test/compute/fixtures/vultr/error_rate_limit.txt
@@ -0,0 +1 @@
+Rate limit reached - please try your request again later.  Current rate limit: 2 requests/sec
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/libcloud/blob/d19574e3/libcloud/test/compute/test_vultr.py
----------------------------------------------------------------------
diff --git a/libcloud/test/compute/test_vultr.py b/libcloud/test/compute/test_vultr.py
index 303f206..9d94fb4 100644
--- a/libcloud/test/compute/test_vultr.py
+++ b/libcloud/test/compute/test_vultr.py
@@ -22,6 +22,8 @@ except ImportError:
 
 from libcloud.utils.py3 import httplib
 
+from libcloud.common.types import ServiceUnavailableError
+
 from libcloud.compute.drivers.vultr import VultrNodeDriver
 
 from libcloud.test import LibcloudTestCase, MockHttp
@@ -116,6 +118,10 @@ class VultrTests(LibcloudTestCase):
         res = self.driver.delete_key_pair(key_pair)
         self.assertTrue(res)
 
+    def test_rate_limit(self):
+        VultrMockHttp.type = 'SERVICE_UNAVAILABLE'
+        self.assertRaises(ServiceUnavailableError, self.driver.list_nodes)
+
 
 class VultrMockHttp(MockHttp):
     fixtures = ComputeFileFixtures('vultr')
@@ -136,6 +142,11 @@ class VultrMockHttp(MockHttp):
         body = self.fixtures.load('list_nodes.json')
         return (httplib.OK, body, {}, httplib.responses[httplib.OK])
 
+    def _v1_server_list_SERVICE_UNAVAILABLE(self, method, url, body, headers):
+        body = self.fixtures.load('error_rate_limit.txt')
+        return (httplib.SERVICE_UNAVAILABLE, body, {},
+                httplib.responses[httplib.SERVICE_UNAVAILABLE])
+
     def _v1_server_create(self, method, url, body, headers):
         body = self.fixtures.load('create_node.json')
         return (httplib.OK, body, {}, httplib.responses[httplib.OK])


Mime
View raw message