libcloud-notifications mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From anthonys...@apache.org
Subject [39/51] [abbrv] libcloud git commit: switch back to raw for _upload_object, encode strings for StringIO hashing, fix various tests
Date Mon, 09 Jan 2017 04:52:59 GMT
switch back to raw for _upload_object, encode strings for StringIO hashing, fix various tests


Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo
Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/cd399d91
Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/cd399d91
Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/cd399d91

Branch: refs/heads/trunk
Commit: cd399d9101a94956cf3e04ee5d5c488510b9103b
Parents: 6e0040d
Author: Anthony Shaw <anthonyshaw@apache.org>
Authored: Sat Jan 7 12:24:18 2017 +1100
Committer: Anthony Shaw <anthonyshaw@apache.org>
Committed: Sat Jan 7 12:24:18 2017 +1100

----------------------------------------------------------------------
 libcloud/common/base.py             |  14 +++
 libcloud/storage/base.py            |  16 ++-
 libcloud/test/__init__.py           | 188 ++++++++++++++++++-------------
 libcloud/test/storage/test_atmos.py |   3 +-
 libcloud/test/storage/test_base.py  | 101 +++++------------
 5 files changed, 165 insertions(+), 157 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/libcloud/blob/cd399d91/libcloud/common/base.py
----------------------------------------------------------------------
diff --git a/libcloud/common/base.py b/libcloud/common/base.py
index 5266163..180cc11 100644
--- a/libcloud/common/base.py
+++ b/libcloud/common/base.py
@@ -284,6 +284,20 @@ class RawResponse(Response):
         self._reason = None
         self.connection = connection
 
+    def success(self):
+        """
+        Determine if our request was successful.
+
+        The meaning of this can be arbitrary; did we receive OK status? Did
+        the node get created? Were we authenticated?
+
+        :rtype: ``bool``
+        :return: ``True`` or ``False``
+        """
+        # pylint: disable=E1101
+        return self.status in [requests.codes.ok, requests.codes.created,
+                               httplib.OK, httplib.CREATED, httplib.ACCEPTED]
+
     @property
     def response(self):
         if not self._response:

http://git-wip-us.apache.org/repos/asf/libcloud/blob/cd399d91/libcloud/storage/base.py
----------------------------------------------------------------------
diff --git a/libcloud/storage/base.py b/libcloud/storage/base.py
index c290ffd..29b84fb 100644
--- a/libcloud/storage/base.py
+++ b/libcloud/storage/base.py
@@ -629,7 +629,7 @@ class StorageDriver(BaseDriver):
             response = self.connection.request(
                 request_path,
                 method=request_method, data=stream,
-                headers=headers, raw=False)
+                headers=headers, raw=True)
             stream_hash, stream_length = self._hash_buffered_stream(
                 stream,
                 self._get_hash_function())
@@ -638,7 +638,7 @@ class StorageDriver(BaseDriver):
                 response = self.connection.request(
                     request_path,
                     method=request_method, data=file_stream,
-                    headers=headers, raw=False)
+                    headers=headers, raw=True)
             with open(file_path, 'rb') as file_stream:
                 stream_hash, stream_length = self._hash_buffered_stream(
                     file_stream,
@@ -656,12 +656,20 @@ class StorageDriver(BaseDriver):
                 'data_hash': stream_hash}
 
     def _hash_buffered_stream(self, stream, hasher, blocksize=65536):
+        total_len = 0
+        if not hasattr(stream, '__exit__'):
+            for s in iter(stream):
+                hasher.update(s)
+                total_len = total_len + len(s)
+            return (hasher.hexdigest(), total_len)
         with stream:
             buf = stream.read(blocksize)
-            total_len = 0
             while len(buf) > 0:
                 total_len = total_len + len(buf)
-                hasher.update(buf)
+                if isinstance(buf, str):
+                    hasher.update(buf.encode())
+                else:
+                    hasher.update(buf)
                 buf = stream.read(blocksize)
         return (hasher.hexdigest(), total_len)
 

http://git-wip-us.apache.org/repos/asf/libcloud/blob/cd399d91/libcloud/test/__init__.py
----------------------------------------------------------------------
diff --git a/libcloud/test/__init__.py b/libcloud/test/__init__.py
index ba911a0..3909d2e 100644
--- a/libcloud/test/__init__.py
+++ b/libcloud/test/__init__.py
@@ -150,6 +150,93 @@ class BaseMockHttpObject(object):
         return meth_name
 
 
+class MockRawResponse(BaseMockHttpObject):
+    """
+    Mock RawResponse object suitable for testing.
+    """
+
+    type = None
+    responseCls = MockResponse
+
+    def __init__(self, connection, path=None):
+        super(MockRawResponse, self).__init__()
+        self._data = []
+        self._current_item = 0
+        self._response = None
+        self._status = None
+        self._headers = None
+        self._reason = None
+        self._path = path
+        self.connection = connection
+
+    def next(self):
+        if self._current_item == len(self._data):
+            raise StopIteration
+
+        value = self._data[self._current_item]
+        self._current_item += 1
+        return value
+
+    def __next__(self):
+        return self.next()
+
+    def _generate_random_data(self, size):
+        data = ''
+        current_size = 0
+        while current_size < size:
+            value = str(random.randint(0, 9))
+            value_size = len(value)
+            data += value
+            current_size += value_size
+
+        return data
+
+    @property
+    def response(self):
+        return self._get_response_if_not_available()
+
+    @property
+    def status(self):
+        self._get_response_if_not_available()
+        return self._status
+
+    @property
+    def status_code(self):
+        self._get_response_if_not_available()
+        return self._status
+
+    def success(self):
+        self._get_response_if_not_available()
+        return self._status in [httplib.OK, httplib.CREATED, httplib.ACCEPTED]
+
+    @property
+    def headers(self):
+        self._get_response_if_not_available()
+        return self._headers
+
+    @property
+    def reason(self):
+        self._get_response_if_not_available()
+        return self._reason
+
+    def _get_response_if_not_available(self):
+        if not self._response:
+            meth_name = self._get_method_name(type=self.type,
+                                              use_param=False, qs=None,
+                                              path=self._path if self._path else self.connection.action)
+            meth = getattr(self, meth_name.replace('%', '_'))
+            result = meth(self.connection.method, None, None, None)
+            self._status, self._body, self._headers, self._reason = result
+            self._response = self.responseCls(self._status, self._body,
+                                              self._headers, self._reason)
+        return self._response
+
+    @property
+    def text(self):
+        self._get_response_if_not_available()
+        return self._body
+
+
 class MockHttp(BaseMockHttpObject):
     """
     A mock HTTP client/server suitable for testing purposes. This replaces
@@ -181,6 +268,7 @@ class MockHttp(BaseMockHttpObject):
 
     """
     responseCls = MockResponse
+    rawResponseCls = MockRawResponse
     host = None
     port = None
     response = None
@@ -284,10 +372,31 @@ class MockHttpTestCase(MockHttp, unittest.TestCase):
                 self.assertEqual(params[key], value)
 
 
+class MockConnection(object):
+    def __init__(self, action):
+        self.action = action
+
+
 class StorageMockHttp(MockHttp):
     def prepared_request(self, method, url, body=None, headers=None, raw=False,
                          stream=False):
-        pass
+         # Find a method we can use for this request
+        parsed = urlparse.urlparse(url)
+        scheme, netloc, path, params, query, fragment = parsed
+        qs = parse_qs(query)
+        if path.endswith('/'):
+            path = path[:-1]
+        meth_name = self._get_method_name(type=self.type,
+                                          use_param=self.use_param,
+                                          qs=qs, path=path)
+        meth = getattr(self, meth_name.replace('%', '_'))
+
+        if self.test and isinstance(self.test, LibcloudTestCase):
+            self.test._add_visited_url(url=url)
+            self.test._add_executed_mock_method(method_name=meth_name)
+
+        status, body, headers, reason = meth(method, url, body, headers)
+        self.response = self.rawResponseCls(MockConnection(action=method), path=path)
 
     def putrequest(self, method, action, skip_host=0, skip_accept_encoding=0):
         pass
@@ -302,83 +411,6 @@ class StorageMockHttp(MockHttp):
         pass
 
 
-class MockRawResponse(BaseMockHttpObject):
-    """
-    Mock RawResponse object suitable for testing.
-    """
-
-    type = None
-    responseCls = MockResponse
-
-    def __init__(self, connection):
-        super(MockRawResponse, self).__init__()
-        self._data = []
-        self._current_item = 0
-
-        self._status = None
-        self._response = None
-        self._headers = None
-        self._reason = None
-        self.connection = connection
-
-    def next(self):
-        if self._current_item == len(self._data):
-            raise StopIteration
-
-        value = self._data[self._current_item]
-        self._current_item += 1
-        return value
-
-    def __next__(self):
-        return self.next()
-
-    def _generate_random_data(self, size):
-        data = ''
-        current_size = 0
-        while current_size < size:
-            value = str(random.randint(0, 9))
-            value_size = len(value)
-            data += value
-            current_size += value_size
-
-        return data
-
-    @property
-    def response(self):
-        return self._get_response_if_not_available()
-
-    @property
-    def status(self):
-        self._get_response_if_not_available()
-        return self._status
-
-    @property
-    def status_code(self):
-        self._get_response_if_not_availale()
-        return self._status
-
-    @property
-    def headers(self):
-        self._get_response_if_not_available()
-        return self._headers
-
-    @property
-    def reason(self):
-        self._get_response_if_not_available()
-        return self._reason
-
-    def _get_response_if_not_available(self):
-        if not self._response:
-            meth_name = self._get_method_name(type=self.type,
-                                              use_param=False, qs=None,
-                                              path=self.connection.action)
-            meth = getattr(self, meth_name.replace('%', '_'))
-            result = meth(self.connection.method, None, None, None)
-            self._status, self._body, self._headers, self._reason = result
-            self._response = self.responseCls(self._status, self._body,
-                                              self._headers, self._reason)
-        return self._response
-
 if __name__ == "__main__":
     import doctest
     doctest.testmod()

http://git-wip-us.apache.org/repos/asf/libcloud/blob/cd399d91/libcloud/test/storage/test_atmos.py
----------------------------------------------------------------------
diff --git a/libcloud/test/storage/test_atmos.py b/libcloud/test/storage/test_atmos.py
index d847fc0..1371b5a 100644
--- a/libcloud/test/storage/test_atmos.py
+++ b/libcloud/test/storage/test_atmos.py
@@ -491,7 +491,8 @@ class AtmosMockHttp(StorageMockHttp, unittest.TestCase):
     def runTest(self):
         pass
 
-    def request(self, method, url, body=None, headers=None, raw=False):
+    def request(self, method, url, body=None, headers=None, raw=False,
+                stream=False):
         headers = headers or {}
         parsed = urlparse.urlparse(url)
         if parsed.query.startswith('metadata/'):

http://git-wip-us.apache.org/repos/asf/libcloud/blob/cd399d91/libcloud/test/storage/test_base.py
----------------------------------------------------------------------
diff --git a/libcloud/test/storage/test_base.py b/libcloud/test/storage/test_base.py
index 91bbe2f..2a6316d 100644
--- a/libcloud/test/storage/test_base.py
+++ b/libcloud/test/storage/test_base.py
@@ -15,12 +15,14 @@
 
 import sys
 import hashlib
+from libcloud.utils.py3 import httplib
+from io import BytesIO
 
 from mock import Mock
 
 from libcloud.utils.py3 import StringIO
 from libcloud.utils.py3 import PY3
-from libcloud.utils.py3 import b
+from libcloud.utils.py3 import b, u
 
 if PY3:
     from io import FileIO as file
@@ -30,16 +32,33 @@ from libcloud.storage.base import DEFAULT_CONTENT_TYPE
 
 from libcloud.test import unittest
 from libcloud.test import StorageMockHttp
+from libcloud.test import MockRawResponse
 
 
+class BaseMockHttp(StorageMockHttp):
+    def __init__(self, *args, **kwargs):
+        self.text = ''
+        self.status_code = 200
+
+    def root(self, method, url, body, headers):
+        body = ''
+        return (httplib.OK, body, {}, httplib.responses[httplib.OK])
+
+    def _(self, method, url, body, headers):
+        body = ''
+        return (httplib.OK, body, {}, httplib.responses[httplib.OK])
+
 class BaseStorageTests(unittest.TestCase):
 
     def setUp(self):
         self.send_called = 0
-        StorageDriver.connectionCls.conn_class = StorageMockHttp
-
+        StorageDriver.connectionCls.conn_class = BaseMockHttp
+        BaseMockHttp.rawResponseCls = BaseMockHttp
         self.driver1 = StorageDriver('username', 'key', host='localhost')
         self.driver1.supports_chunked_encoding = True
+
+        #self.driver1.connectionCls.rawResponseCls = BaseMockHttp
+
         self.driver2 = StorageDriver('username', 'key', host='localhost')
         self.driver2.supports_chunked_encoding = False
 
@@ -47,20 +66,8 @@ class BaseStorageTests(unittest.TestCase):
         self.driver1.strict_mode = False
 
     def test__upload_object_iterator_must_have_next_method(self):
-        class Iterator(object):
-
-            def next(self):
-                pass
-
-        class Iterator2(file):
-
-            def __init__(self):
-                pass
-
-        class SomeClass(object):
-            pass
 
-        valid_iterators = [Iterator(), Iterator2(), StringIO('bar')]
+        valid_iterators = [BytesIO(b('134')), StringIO('bar')]
         invalid_iterators = ['foobar', '', False, True, 1, object()]
 
         def upload_func(*args, **kwargs):
@@ -71,11 +78,11 @@ class BaseStorageTests(unittest.TestCase):
                   'request_path': '/', 'headers': {}}
 
         for value in valid_iterators:
-            kwargs['iterator'] = value
+            kwargs['stream'] = value
             self.driver1._upload_object(**kwargs)
 
         for value in invalid_iterators:
-            kwargs['iterator'] = value
+            kwargs['stream'] = value
 
             try:
                 self.driver1._upload_object(**kwargs)
@@ -84,60 +91,6 @@ class BaseStorageTests(unittest.TestCase):
             else:
                 self.fail('Exception was not thrown')
 
-    def test_upload_zero_bytes_long_object_via_stream(self):
-        iterator = Mock()
-
-        if PY3:
-            iterator.__next__ = Mock()
-            iterator.__next__.side_effect = StopIteration()
-        else:
-            iterator.next.side_effect = StopIteration()
-
-        def mock_send(data):
-            self.send_called += 1
-
-        response = Mock()
-        response.connection.connection.send = mock_send
-
-        # Normal
-        success, data_hash, bytes_transferred = \
-            self.driver1._stream_data(response=response,
-                                      iterator=iterator,
-                                      chunked=False, calculate_hash=True)
-
-        self.assertTrue(success)
-        self.assertEqual(data_hash, hashlib.md5(b('')).hexdigest())
-        self.assertEqual(bytes_transferred, 0)
-        self.assertEqual(self.send_called, 1)
-
-        # Chunked
-        success, data_hash, bytes_transferred = \
-            self.driver1._stream_data(response=response,
-                                      iterator=iterator,
-                                      chunked=True, calculate_hash=True)
-
-        self.assertTrue(success)
-        self.assertEqual(data_hash, hashlib.md5(b('')).hexdigest())
-        self.assertEqual(bytes_transferred, 0)
-        self.assertEqual(self.send_called, 5)
-
-    def test__upload_data(self):
-        def mock_send(data):
-            self.send_called += 1
-
-        response = Mock()
-        response.connection.connection.send = mock_send
-
-        data = '123456789901234567'
-        success, data_hash, bytes_transferred = \
-            self.driver1._upload_data(response=response, data=data,
-                                      calculate_hash=True)
-
-        self.assertTrue(success)
-        self.assertEqual(data_hash, hashlib.md5(b(data)).hexdigest())
-        self.assertEqual(bytes_transferred, (len(data)))
-        self.assertEqual(self.send_called, 1)
-
     def test__get_hash_function(self):
         self.driver1.hash_type = 'md5'
         func = self.driver1._get_hash_function()
@@ -169,7 +122,7 @@ class BaseStorageTests(unittest.TestCase):
                                     upload_func=upload_func,
                                     upload_func_kwargs={},
                                     request_path='/',
-                                    iterator=iterator)
+                                    stream=iterator)
 
         headers = self.driver1.connection.request.call_args[-1]['headers']
         self.assertEqual(headers['Content-Type'], DEFAULT_CONTENT_TYPE)
@@ -186,7 +139,7 @@ class BaseStorageTests(unittest.TestCase):
                                 upload_func=upload_func,
                                 upload_func_kwargs={},
                                 request_path='/',
-                                iterator=iterator)
+                                stream=iterator)
 
 
 if __name__ == '__main__':


Mime
View raw message