libcloud-notifications mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From to...@apache.org
Subject libcloud git commit: [LIBCLOUD-639] Fix upload_object_via_stream so it works correctly under Python 3.x if user manually passes an iterator to the method.
Date Tue, 09 Dec 2014 12:41:27 GMT
Repository: libcloud
Updated Branches:
  refs/heads/trunk f813ea17b -> 8fe3e4055


[LIBCLOUD-639] Fix upload_object_via_stream so it works correctly
under Python 3.x if user manually passes an iterator to the method.

Closes #408

Signed-off-by: Tomaz Muraus <tomaz@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo
Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/8fe3e405
Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/8fe3e405
Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/8fe3e405

Branch: refs/heads/trunk
Commit: 8fe3e4055d8479c2ccaebff45bf0071890639f8c
Parents: f813ea1
Author: Peter Schmidt <peter@peterjs.com>
Authored: Mon Dec 1 17:28:20 2014 +1100
Committer: Tomaz Muraus <tomaz@apache.org>
Committed: Tue Dec 9 13:34:13 2014 +0100

----------------------------------------------------------------------
 CHANGES.rst                              |  5 ++
 libcloud/storage/base.py                 |  3 +-
 libcloud/test/storage/test_cloudfiles.py | 69 +++++++++++++++++++++++++--
 libcloud/utils/files.py                  |  2 +-
 libcloud/utils/py3.py                    |  2 +
 5 files changed, 74 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/libcloud/blob/8fe3e405/CHANGES.rst
----------------------------------------------------------------------
diff --git a/CHANGES.rst b/CHANGES.rst
index e4858e7..3a2282d 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -79,6 +79,11 @@ Storage
   (GITHUB-403, GITHUB-404)
   [Peter Schmidt]
 
+- Fix upload_object_via_stream so it works correctly under Python 3.x if user
+  manually passes an iterator to the method.
+  (GITHUB-408, LIBCLOUD-639)
+  [Peter Schmidt]
+
 Changes with Apache Libcloud 0.16.0
 -----------------------------------
 

http://git-wip-us.apache.org/repos/asf/libcloud/blob/8fe3e405/libcloud/storage/base.py
----------------------------------------------------------------------
diff --git a/libcloud/storage/base.py b/libcloud/storage/base.py
index 66e5d44..ead1545 100644
--- a/libcloud/storage/base.py
+++ b/libcloud/storage/base.py
@@ -742,7 +742,8 @@ class StorageDriver(BaseDriver):
         if calculate_hash:
             data_hash = self._get_hash_function()
 
-        generator = libcloud.utils.files.read_in_chunks(iterator, chunk_size)
+        generator = libcloud.utils.files.read_in_chunks(iterator, chunk_size,
+                                                        fill_size=True)
 
         bytes_transferred = 0
         try:

http://git-wip-us.apache.org/repos/asf/libcloud/blob/8fe3e405/libcloud/test/storage/test_cloudfiles.py
----------------------------------------------------------------------
diff --git a/libcloud/test/storage/test_cloudfiles.py b/libcloud/test/storage/test_cloudfiles.py
index 6590270..3e911da 100644
--- a/libcloud/test/storage/test_cloudfiles.py
+++ b/libcloud/test/storage/test_cloudfiles.py
@@ -30,7 +30,7 @@ from libcloud.utils.py3 import httplib
 from libcloud.utils.py3 import urlquote
 
 from libcloud.common.types import LibcloudError, MalformedResponseError
-from libcloud.storage.base import Container, Object
+from libcloud.storage.base import CHUNK_SIZE, Container, Object
 from libcloud.storage.types import ContainerAlreadyExistsError
 from libcloud.storage.types import ContainerDoesNotExistError
 from libcloud.storage.types import ContainerIsNotEmptyError
@@ -665,10 +665,6 @@ class CloudFilesTests(unittest.TestCase):
 
         try:
             self.driver.upload_object_via_stream(
-                # We never reach the Python 3 only bytes vs int error
-                # currently at libcloud/utils/py3.py:89
-                #     raise TypeError("Invalid argument %r for b()" % (s,))
-                # because I raise a NotImplementedError.
                 iterator=iter(b'blob data like an image or video'),
                 container=container,
                 object_name="test_object",
@@ -682,6 +678,64 @@ class CloudFilesTests(unittest.TestCase):
             self.fail('Expected NotImplementedError to be thrown to '
                       'verify we actually checked the expected headers')
 
+    def test_upload_object_via_stream_python3_bytes_error(self):
+        container = Container(name='py3', extra={}, driver=self.driver)
+        bytes_blob = b'blob data like an image or video'
+
+        # This is mostly to check we didn't discover other errors along the way
+        mocked_response = container.upload_object_via_stream(
+            iterator=iter(bytes_blob),
+            object_name="img_or_vid",
+        )
+        self.assertEqual(len(bytes_blob), mocked_response.size)
+
+    def test_upload_object_via_stream_chunked_encoding(self):
+
+        # Create enough bytes it should get split into two chunks
+        bytes_blob = ''.join(['\0' for _ in range(CHUNK_SIZE + 1)])
+        hex_chunk_size = ('%X' % CHUNK_SIZE).encode('utf8')
+        expected = [
+            # Chunk 1
+            hex_chunk_size + b'\r\n',
+            bytes(bytes_blob[:CHUNK_SIZE].encode('utf8')),
+            b'\r\n',
+
+            # Chunk 2
+            b'1\r\n',
+            bytes(bytes_blob[CHUNK_SIZE:].encode('utf8')),
+            b'\r\n',
+
+            # If chunked, also send a final message
+            b'0\r\n\r\n',
+        ]
+        logged_data = []
+
+        class InterceptResponse(CloudFilesMockRawResponse):
+            def __init__(self, connection):
+                super(InterceptResponse, self).__init__(connection=connection)
+                old_send = self.connection.connection.send
+
+                def intercept_send(data):
+                    old_send(data)
+                    logged_data.append(data)
+                self.connection.connection.send = intercept_send
+
+            def _v1_MossoCloudFS_py3_img_or_vid2(self,
+                                                 method, url, body, headers):
+                headers = {'etag': 'd79fb00c27b50494a463e680d459c90c'}
+                headers.update(self.base_headers)
+                _201 = httplib.CREATED
+                return _201, '', headers, httplib.responses[_201]
+
+        self.driver_klass.connectionCls.rawResponseCls = InterceptResponse
+
+        container = Container(name='py3', extra={}, driver=self.driver)
+        container.upload_object_via_stream(
+            iterator=iter(bytes_blob),
+            object_name="img_or_vid2",
+        )
+        self.assertListEqual(expected, logged_data)
+
     def test__upload_object_manifest(self):
         hash_function = self.driver._get_hash_function()
         hash_function.update(b(''))
@@ -1087,6 +1141,11 @@ class CloudFilesMockRawResponse(MockRawResponse):
     fixtures = StorageFileFixtures('cloudfiles')
     base_headers = {'content-type': 'application/json; charset=UTF-8'}
 
+    def _v1_MossoCloudFS_py3_img_or_vid(self, method, url, body, headers):
+        headers = {'etag': 'e2378cace8712661ce7beec3d9362ef6'}
+        headers.update(self.base_headers)
+        return httplib.CREATED, '', headers, httplib.responses[httplib.CREATED]
+
     def _v1_MossoCloudFS_foo_bar_container_foo_test_upload(
             self, method, url, body, headers):
         # test_object_upload_success

http://git-wip-us.apache.org/repos/asf/libcloud/blob/8fe3e405/libcloud/utils/files.py
----------------------------------------------------------------------
diff --git a/libcloud/utils/files.py b/libcloud/utils/files.py
index a71e1c4..663677a 100644
--- a/libcloud/utils/files.py
+++ b/libcloud/utils/files.py
@@ -38,7 +38,7 @@ def read_in_chunks(iterator, chunk_size=None, fill_size=False,
     """
     Return a generator which yields data in chunks.
 
-    :param terator: An object which implements an iterator interface
+    :param iterator: An object which implements an iterator interface
                      or a File like object with read method.
     :type iterator: :class:`object` which implements iterator interface.
 

http://git-wip-us.apache.org/repos/asf/libcloud/blob/8fe3e405/libcloud/utils/py3.py
----------------------------------------------------------------------
diff --git a/libcloud/utils/py3.py b/libcloud/utils/py3.py
index 4e05419..2b695a4 100644
--- a/libcloud/utils/py3.py
+++ b/libcloud/utils/py3.py
@@ -85,6 +85,8 @@ if PY3:
             return s.encode('utf-8')
         elif isinstance(s, bytes):
             return s
+        elif isinstance(s, int):
+            return bytes([s])
         else:
             raise TypeError("Invalid argument %r for b()" % (s,))
 


Mime
View raw message