libcloud-notifications mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From anthonys...@apache.org
Subject [35/51] [abbrv] libcloud git commit: update test interfaces to work with new modules
Date Mon, 09 Jan 2017 04:52:55 GMT
update test interfaces to work with new modules


Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo
Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/7ec75c19
Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/7ec75c19
Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/7ec75c19

Branch: refs/heads/trunk
Commit: 7ec75c19cd86f5fb46ee16ab8f5098778503c3c0
Parents: 610c624
Author: Anthony Shaw <anthonyshaw@apache.org>
Authored: Fri Jan 6 16:26:54 2017 +1100
Committer: Anthony Shaw <anthonyshaw@apache.org>
Committed: Fri Jan 6 16:26:54 2017 +1100

----------------------------------------------------------------------
 libcloud/common/base.py           |   3 +-
 libcloud/storage/drivers/dummy.py |   5 +
 libcloud/storage/drivers/s3.py    | 175 +--------------------------------
 libcloud/test/__init__.py         |   8 +-
 4 files changed, 18 insertions(+), 173 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/libcloud/blob/7ec75c19/libcloud/common/base.py
----------------------------------------------------------------------
diff --git a/libcloud/common/base.py b/libcloud/common/base.py
index d893cf0..15233a9 100644
--- a/libcloud/common/base.py
+++ b/libcloud/common/base.py
@@ -154,7 +154,8 @@ class Response(object):
         self.request = response.request
 
         self.body = response.text.strip() \
-            if response.text is not None else ''
+            if response.text is not None and hasattr(response.text, 'strip') \
+            else ''
 
         if not self.success():
             raise exception_from_message(code=self.status,

http://git-wip-us.apache.org/repos/asf/libcloud/blob/7ec75c19/libcloud/storage/drivers/dummy.py
----------------------------------------------------------------------
diff --git a/libcloud/storage/drivers/dummy.py b/libcloud/storage/drivers/dummy.py
index affd265..4065123 100644
--- a/libcloud/storage/drivers/dummy.py
+++ b/libcloud/storage/drivers/dummy.py
@@ -75,6 +75,11 @@ class DummyIterator(object):
     def __next__(self):
         return self.next()
 
+    def __enter__(self):
+        pass
+
+    def __exit__(self, type, value, traceback):
+        pass
 
 class DummyStorageDriver(StorageDriver):
     """

http://git-wip-us.apache.org/repos/asf/libcloud/blob/7ec75c19/libcloud/storage/drivers/s3.py
----------------------------------------------------------------------
diff --git a/libcloud/storage/drivers/s3.py b/libcloud/storage/drivers/s3.py
index 34fc455..28e205e 100644
--- a/libcloud/storage/drivers/s3.py
+++ b/libcloud/storage/drivers/s3.py
@@ -435,173 +435,6 @@ class BaseS3StorageDriver(StorageDriver):
                                 verify_hash=verify_hash,
                                 storage_class=ex_storage_class)
 
-    def _upload_multipart(self, response, data, iterator, container,
-                          object_name, calculate_hash=True):
-        """
-        Callback invoked for uploading data to S3 using Amazon's
-        multipart upload mechanism
-
-        :param response: Response object from the initial POST request
-        :type response: :class:`S3RawResponse`
-
-        :param data: Any data from the initial POST request
-        :type data: ``str``
-
-        :param iterator: The generator for fetching the upload data
-        :type iterator: ``generator``
-
-        :param container: The container owning the object to which data is
-            being uploaded
-        :type container: :class:`Container`
-
-        :param object_name: The name of the object to which we are uploading
-        :type object_name: ``str``
-
-        :keyword calculate_hash: Indicates if we must calculate the data hash
-        :type calculate_hash: ``bool``
-
-        :return: A tuple of (status, checksum, bytes transferred)
-        :rtype: ``tuple``
-        """
-
-        object_path = self._get_object_path(container, object_name)
-
-        # Get the upload id from the response xml
-        response.body = response.response.read()
-        body = response.parse_body()
-        upload_id = body.find(fixxpath(xpath='UploadId',
-                                       namespace=self.namespace)).text
-
-        try:
-            # Upload the data through the iterator
-            result = self._upload_from_iterator(iterator, object_path,
-                                                upload_id, calculate_hash)
-            (chunks, data_hash, bytes_transferred) = result
-
-            # Commit the chunk info and complete the upload
-            etag = self._commit_multipart(object_path, upload_id, chunks)
-        except Exception:
-            exc = sys.exc_info()[1]
-            # Amazon provides a mechanism for aborting an upload.
-            self._abort_multipart(object_path, upload_id)
-            raise exc
-
-        # Modify the response header of the first request. This is used
-        # by other functions once the callback is done
-        response.headers['etag'] = etag
-
-        return (True, data_hash, bytes_transferred)
-
-    def _upload_from_iterator(self, iterator, object_path, upload_id,
-                              calculate_hash=True):
-        """
-        Uploads data from an iterator in fixed sized chunks to S3
-
-        :param iterator: The generator for fetching the upload data
-        :type iterator: ``generator``
-
-        :param object_path: The path of the object to which we are uploading
-        :type object_name: ``str``
-
-        :param upload_id: The upload id allocated for this multipart upload
-        :type upload_id: ``str``
-
-        :keyword calculate_hash: Indicates if we must calculate the data hash
-        :type calculate_hash: ``bool``
-
-        :return: A tuple of (chunk info, checksum, bytes transferred)
-        :rtype: ``tuple``
-        """
-
-        data_hash = None
-        if calculate_hash:
-            data_hash = self._get_hash_function()
-
-        bytes_transferred = 0
-        count = 1
-        chunks = []
-        params = {'uploadId': upload_id}
-
-        # Read the input data in chunk sizes suitable for AWS
-        for data in read_in_chunks(iterator, chunk_size=CHUNK_SIZE,
-                                   fill_size=True, yield_empty=True):
-            bytes_transferred += len(data)
-
-            if calculate_hash:
-                data_hash.update(data)
-
-            chunk_hash = self._get_hash_function()
-            chunk_hash.update(data)
-            chunk_hash = base64.b64encode(chunk_hash.digest()).decode('utf-8')
-
-            # This provides an extra level of data check and is recommended
-            # by amazon
-            headers = {'Content-MD5': chunk_hash}
-            params['partNumber'] = count
-
-            request_path = '?'.join((object_path, urlencode(params)))
-
-            resp = self.connection.request(request_path, method='PUT',
-                                           data=data, headers=headers)
-
-            if resp.status != httplib.OK:
-                raise LibcloudError('Error uploading chunk', driver=self)
-
-            server_hash = resp.headers['etag']
-
-            # Keep this data for a later commit
-            chunks.append((count, server_hash))
-            count += 1
-
-        if calculate_hash:
-            data_hash = data_hash.hexdigest()
-
-        return (chunks, data_hash, bytes_transferred)
-
-    def _commit_multipart(self, object_path, upload_id, chunks):
-        """
-        Makes a final commit of the data.
-
-        :param object_path: Server side object path.
-        :type object_path: ``str``
-
-        :param upload_id: ID of the multipart upload.
-        :type upload_id: ``str``
-
-        :param upload_id: A list of (chunk_number, chunk_hash) tuples.
-        :type upload_id: ``list``
-        """
-
-        root = Element('CompleteMultipartUpload')
-
-        for (count, etag) in chunks:
-            part = SubElement(root, 'Part')
-            part_no = SubElement(part, 'PartNumber')
-            part_no.text = str(count)
-
-            etag_id = SubElement(part, 'ETag')
-            etag_id.text = str(etag)
-
-        data = tostring(root)
-
-        params = {'uploadId': upload_id}
-        request_path = '?'.join((object_path, urlencode(params)))
-        response = self.connection.request(request_path, data=data,
-                                           method='POST')
-
-        if response.status != httplib.OK:
-            element = response.object
-            # pylint: disable=maybe-no-member
-            code, message = response._parse_error_details(element=element)
-            msg = 'Error in multipart commit: %s (%s)' % (message, code)
-            raise LibcloudError(msg, driver=self)
-
-        # Get the server's etag to be passed back to the caller
-        body = response.parse_body()
-        server_hash = body.find(fixxpath(xpath='ETag',
-                                         namespace=self.namespace)).text
-        return server_hash
-
     def _abort_multipart(self, object_path, upload_id):
         """
         Aborts an already initiated multipart upload
@@ -637,9 +470,8 @@ class BaseS3StorageDriver(StorageDriver):
         # Amazon provides a different (complex?) mechanism to do multipart
         # uploads
         if self.supports_s3_multipart_upload:
-            method = 'POST'
-            params = 'uploads'
-
+            # @TODO: This needs implementing again from scratch.
+            pass
         return self._put_object(container=container, object_name=object_name,
                                 extra=extra, method=method, query_args=params,
                                 stream=iterator, verify_hash=False,
@@ -808,7 +640,8 @@ class BaseS3StorageDriver(StorageDriver):
 
         if (verify_hash and result_dict['data_hash'] != server_hash):
             raise ObjectHashMismatchError(
-                value='MD5 hash checksum does not match',
+                value='MD5 hash {0} checksum does not match {1}'.format(
+                    server_hash, result_dict['data_hash']),
                 object_name=object_name, driver=self)
         elif response.status == httplib.OK:
             obj = Object(

http://git-wip-us.apache.org/repos/asf/libcloud/blob/7ec75c19/libcloud/test/__init__.py
----------------------------------------------------------------------
diff --git a/libcloud/test/__init__.py b/libcloud/test/__init__.py
index 2024f30..c4db57c 100644
--- a/libcloud/test/__init__.py
+++ b/libcloud/test/__init__.py
@@ -88,7 +88,13 @@ class MockResponse(object):
         self.body = body
         self.headers = headers or self.headers
         self.reason = reason or self.reason
-        self.body_iter = iter(self.body) if self.body is not None else None
+        if self.body:
+            if not hasattr(self.body, '__next__'):
+                self.body_iter = iter(self.body)
+            else:
+                self.body_iter = self.body
+        else:
+            self.body_iter = iter('')
 
     def read(self, *args, **kwargs):
         return self.body


Mime
View raw message