libcloud-notifications mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From to...@apache.org
Subject libcloud git commit: Allow user to pass "headers" argument to the upload_* methods in the storage API.
Date Mon, 01 Dec 2014 05:34:27 GMT
Repository: libcloud
Updated Branches:
  refs/heads/trunk a4323ca7d -> a19d1df1d


Allow user to pass "headers" argument to the upload_* methods in the storage
API.

This way user can specify CORS headers with the providers which supported that.

Closes #404

Signed-off-by: Tomaz Muraus <tomaz@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo
Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/a19d1df1
Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/a19d1df1
Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/a19d1df1

Branch: refs/heads/trunk
Commit: a19d1df1d093975eb0ef2d3c829dd107bada872f
Parents: a4323ca
Author: Peter Schmidt <peter@peterjs.com>
Authored: Thu Nov 27 11:20:38 2014 +1100
Committer: Tomaz Muraus <tomaz@apache.org>
Committed: Mon Dec 1 06:28:49 2014 +0100

----------------------------------------------------------------------
 CHANGES.rst                              | 10 ++++
 docs/development.rst                     |  4 +-
 libcloud/storage/base.py                 | 71 ++++++++++++++++-----------
 libcloud/storage/drivers/cloudfiles.py   | 14 +++---
 libcloud/test/storage/test_cloudfiles.py | 49 +++++++++++++++++-
 5 files changed, 109 insertions(+), 39 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/libcloud/blob/a19d1df1/CHANGES.rst
----------------------------------------------------------------------
diff --git a/CHANGES.rst b/CHANGES.rst
index f55ebea..012f5e9 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -61,6 +61,16 @@ Compute
   (GITHUB-401)
   [Eric Johnson]
 
+Storage
+~~~~~~~
+
+- Allow user to pass ``headers`` argument to the ``upload_object`` and
+  ``upload_object_via_stream`` method.
+
+  This way user can specify CORS headers with the drivers which support that.
+  (GITHUB-403, GITHUB-404)
+  [Peter Schmidt]
+
 Changes with Apache Libcloud 0.16.0
 -----------------------------------
 

http://git-wip-us.apache.org/repos/asf/libcloud/blob/a19d1df1/docs/development.rst
----------------------------------------------------------------------
diff --git a/docs/development.rst b/docs/development.rst
index 8899df9..09cfe2b 100644
--- a/docs/development.rst
+++ b/docs/development.rst
@@ -36,9 +36,9 @@ Code style guide
 * Use 79 characters in a line
 * Make sure edited file doesn't contain any trailing whitespace
 * You can verify that your modifications don't break any rules by running the
-  ``flake8`` script - e.g. ``flake8 libcloud/edited_file.py.`` or
+  ``flake8`` script - e.g. ``flake8 libcloud/edited_file.py`` or
   ``tox -e lint``.
-  Second command fill run flake8 on all the files in the repository.
+  Second command will run flake8 on all the files in the repository.
 
 And most importantly, follow the existing style in the file you are editing and
 **be consistent**.

http://git-wip-us.apache.org/repos/asf/libcloud/blob/a19d1df1/libcloud/storage/base.py
----------------------------------------------------------------------
diff --git a/libcloud/storage/base.py b/libcloud/storage/base.py
index 5e4f09a..66e5d44 100644
--- a/libcloud/storage/base.py
+++ b/libcloud/storage/base.py
@@ -361,7 +361,7 @@ class StorageDriver(BaseDriver):
             'download_object_as_stream not implemented for this driver')
 
     def upload_object(self, file_path, container, object_name, extra=None,
-                      verify_hash=True):
+                      verify_hash=True, headers=None):
         """
         Upload an object currently located on a disk.
 
@@ -380,6 +380,11 @@ class StorageDriver(BaseDriver):
         :param extra: Extra attributes (driver specific). (optional)
         :type extra: ``dict``
 
+        :param headers: (optional) Additional request headers,
+            such as CORS headers. For example:
+            headers = {'Access-Control-Allow-Origin': 'http://mozilla.com'}
+        :type headers: ``dict``
+
         :rtype: :class:`Object`
         """
         raise NotImplementedError(
@@ -387,7 +392,8 @@ class StorageDriver(BaseDriver):
 
     def upload_object_via_stream(self, iterator, container,
                                  object_name,
-                                 extra=None):
+                                 extra=None,
+                                 headers=None):
         """
         Upload an object using an iterator.
 
@@ -405,19 +411,24 @@ class StorageDriver(BaseDriver):
         function which uses fs.stat function to determine the file size and it
         doesn't need to buffer whole object in the memory.
 
-        :type iterator: :class:`object`
         :param iterator: An object which implements the iterator interface.
+        :type iterator: :class:`object`
 
-        :type container: :class:`Container`
         :param container: Destination container.
+        :type container: :class:`Container`
 
-        :type object_name: ``str``
         :param object_name: Object name.
+        :type object_name: ``str``
 
-        :type extra: ``dict``
         :param extra: (optional) Extra attributes (driver specific). Note:
             This dictionary must contain a 'content_type' key which represents
             a content type of the stored object.
+        :type extra: ``dict``
+
+        :param headers: (optional) Additional request headers,
+            such as CORS headers. For example:
+            headers = {'Access-Control-Allow-Origin': 'http://mozilla.com'}
+        :type headers: ``dict``
 
         :rtype: ``object``
         """
@@ -428,8 +439,8 @@ class StorageDriver(BaseDriver):
         """
         Delete an object.
 
-        :type obj: :class:`Object`
         :param obj: Object instance.
+        :type obj: :class:`Object`
 
         :return: ``bool`` True on success.
         :rtype: ``bool``
@@ -441,8 +452,8 @@ class StorageDriver(BaseDriver):
         """
         Create a new container.
 
-        :type container_name: ``str``
         :param container_name: Container name.
+        :type container_name: ``str``
 
         :return: Container instance on success.
         :rtype: :class:`Container`
@@ -454,8 +465,8 @@ class StorageDriver(BaseDriver):
         """
         Delete a container.
 
-        :type container: :class:`Container`
         :param container: Container instance
+        :type container: :class:`Container`
 
         :return: ``True`` on success, ``False`` otherwise.
         :rtype: ``bool``
@@ -468,23 +479,23 @@ class StorageDriver(BaseDriver):
         """
         Call passed callback and start transfer of the object'
 
-        :type obj: :class:`Object`
         :param obj: Object instance.
+        :type obj: :class:`Object`
 
-        :type callback: :class:`function`
         :param callback: Function which is called with the passed
             callback_kwargs
+        :type callback: :class:`function`
 
-        :type callback_kwargs: ``dict``
         :param callback_kwargs: Keyword arguments which are passed to the
              callback.
+        :type callback_kwargs: ``dict``
 
-        :typed response: :class:`Response`
         :param response: Response instance.
+        :type response: :class:`Response`
 
-        :type success_status_code: ``int``
         :param success_status_code: Status code which represents a successful
                                     transfer (defaults to httplib.OK)
+        :type success_status_code: ``int``
 
         :return: ``True`` on success, ``False`` otherwise.
         :rtype: ``bool``
@@ -507,26 +518,26 @@ class StorageDriver(BaseDriver):
         """
         Save object to the provided path.
 
-        :type response: :class:`RawResponse`
         :param response: RawResponse instance.
+        :type response: :class:`RawResponse`
 
-        :type obj: :class:`Object`
         :param obj: Object instance.
+        :type obj: :class:`Object`
 
-        :type destination_path: ``str``
         :param destination_path: Destination directory.
+        :type destination_path: ``str``
 
-        :type delete_on_failure: ``bool``
         :param delete_on_failure: True to delete partially downloaded object if
                                   the download fails.
+        :type delete_on_failure: ``bool``
 
-        :type overwrite_existing: ``bool``
         :param overwrite_existing: True to overwrite a local path if it already
                                    exists.
+        :type overwrite_existing: ``bool``
 
-        :type chunk_size: ``int``
         :param chunk_size: Optional chunk size
             (defaults to ``libcloud.storage.base.CHUNK_SIZE``, 8kb)
+        :type chunk_size: ``int``
 
         :return: ``True`` on success, ``False`` otherwise.
         :rtype: ``bool``
@@ -660,20 +671,20 @@ class StorageDriver(BaseDriver):
         """
         Upload data stored in a string.
 
-        :type response: :class:`RawResponse`
         :param response: RawResponse object.
+        :type response: :class:`RawResponse`
 
-        :type data: ``str``
         :param data: Data to upload.
+        :type data: ``str``
 
-        :type calculate_hash: ``bool``
         :param calculate_hash: True to calculate hash of the transferred data.
-                               (defauls to True).
+                               (defaults to True).
+        :type calculate_hash: ``bool``
 
-        :rtype: ``tuple``
         :return: First item is a boolean indicator of success, second
                  one is the uploaded data MD5 hash and the third one
                  is the number of transferred bytes.
+        :rtype: ``tuple``
         """
         bytes_transferred = 0
         data_hash = None
@@ -701,23 +712,23 @@ class StorageDriver(BaseDriver):
         """
         Stream a data over an http connection.
 
-        :type response: :class:`RawResponse`
         :param response: RawResponse object.
+        :type response: :class:`RawResponse`
 
-        :type iterator: :class:`object`
         :param response: An object which implements an iterator interface
                          or a File like object with read method.
+        :type iterator: :class:`object`
 
-        :type chunked: ``bool``
         :param chunked: True if the chunked transfer encoding should be used
                         (defauls to False).
+        :type chunked: ``bool``
 
-        :type calculate_hash: ``bool``
         :param calculate_hash: True to calculate hash of the transferred data.
                                (defauls to True).
+        :type calculate_hash: ``bool``
 
-        :type chunk_size: ``int``
         :param chunk_size: Optional chunk size (defaults to ``CHUNK_SIZE``)
+        :type chunk_size: ``int``
 
         :rtype: ``tuple``
         :return: First item is a boolean indicator of success, second

http://git-wip-us.apache.org/repos/asf/libcloud/blob/a19d1df1/libcloud/storage/drivers/cloudfiles.py
----------------------------------------------------------------------
diff --git a/libcloud/storage/drivers/cloudfiles.py b/libcloud/storage/drivers/cloudfiles.py
index a2189c1..314586b 100644
--- a/libcloud/storage/drivers/cloudfiles.py
+++ b/libcloud/storage/drivers/cloudfiles.py
@@ -414,7 +414,7 @@ class CloudFilesStorageDriver(StorageDriver, OpenStackDriverMixin):
                                 success_status_code=httplib.OK)
 
     def upload_object(self, file_path, container, object_name, extra=None,
-                      verify_hash=True):
+                      verify_hash=True, headers=None):
         """
         Upload an object.
 
@@ -427,10 +427,11 @@ class CloudFilesStorageDriver(StorageDriver, OpenStackDriverMixin):
                                 upload_func=upload_func,
                                 upload_func_kwargs=upload_func_kwargs,
                                 extra=extra, file_path=file_path,
-                                verify_hash=verify_hash)
+                                verify_hash=verify_hash, headers=headers)
 
     def upload_object_via_stream(self, iterator,
-                                 container, object_name, extra=None):
+                                 container, object_name, extra=None,
+                                 headers=None):
         if isinstance(iterator, file):
             iterator = iter(iterator)
 
@@ -440,7 +441,8 @@ class CloudFilesStorageDriver(StorageDriver, OpenStackDriverMixin):
         return self._put_object(container=container, object_name=object_name,
                                 upload_func=upload_func,
                                 upload_func_kwargs=upload_func_kwargs,
-                                extra=extra, iterator=iterator)
+                                extra=extra, iterator=iterator,
+                                headers=headers)
 
     def delete_object(self, obj):
         container_name = self._encode_container_name(obj.container.name)
@@ -752,7 +754,7 @@ class CloudFilesStorageDriver(StorageDriver, OpenStackDriverMixin):
 
     def _put_object(self, container, object_name, upload_func,
                     upload_func_kwargs, extra=None, file_path=None,
-                    iterator=None, verify_hash=True):
+                    iterator=None, verify_hash=True, headers=None):
         extra = extra or {}
         container_name_encoded = self._encode_container_name(container.name)
         object_name_encoded = self._encode_object_name(object_name)
@@ -760,7 +762,7 @@ class CloudFilesStorageDriver(StorageDriver, OpenStackDriverMixin):
         meta_data = extra.get('meta_data', None)
         content_disposition = extra.get('content_disposition', None)
 
-        headers = {}
+        headers = headers or {}
         if meta_data:
             for key, value in list(meta_data.items()):
                 key = 'X-Object-Meta-%s' % (key)

http://git-wip-us.apache.org/repos/asf/libcloud/blob/a19d1df1/libcloud/test/storage/test_cloudfiles.py
----------------------------------------------------------------------
diff --git a/libcloud/test/storage/test_cloudfiles.py b/libcloud/test/storage/test_cloudfiles.py
index 70e52c8..6590270 100644
--- a/libcloud/test/storage/test_cloudfiles.py
+++ b/libcloud/test/storage/test_cloudfiles.py
@@ -20,7 +20,6 @@ import os.path                          # pylint: disable-msg=W0404
 import math
 import sys
 import copy
-import unittest
 
 import mock
 
@@ -45,6 +44,7 @@ from libcloud.storage.drivers.dummy import DummyIterator
 
 from libcloud.test import StorageMockHttp, MockRawResponse  # pylint: disable-msg=E0611
 from libcloud.test import MockHttpTestCase  # pylint: disable-msg=E0611
+from libcloud.test import unittest
 from libcloud.test.file_fixtures import StorageFileFixtures  # pylint: disable-msg=E0611
 
 
@@ -635,6 +635,53 @@ class CloudFilesTests(unittest.TestCase):
         self.assertEqual(func_kwargs['object_name'], expected_name)
         self.assertEqual(func_kwargs['container'], container)
 
+    def test_upload_object_via_stream_with_cors_headers(self):
+        """
+        Test we can add some ``Cross-origin resource sharing`` headers
+        to the request about to be sent.
+        """
+        cors_headers = {
+            'Access-Control-Allow-Origin': 'http://mozilla.com',
+            'Origin': 'http://storage.clouddrive.com',
+        }
+        expected_headers = {
+            # Automatically added headers
+            'Content-Type': 'application/octet-stream',
+            'Transfer-Encoding': 'chunked',
+        }
+        expected_headers.update(cors_headers)
+
+        def intercept_request(request_path,
+                              method=None, data=None,
+                              headers=None, raw=True):
+
+            # What we're actually testing
+            self.assertDictEqual(expected_headers, headers)
+
+            raise NotImplementedError('oops')
+        self.driver.connection.request = intercept_request
+
+        container = Container(name='CORS', extra={}, driver=self.driver)
+
+        try:
+            self.driver.upload_object_via_stream(
+                # We never reach the Python 3 only bytes vs int error
+                # currently at libcloud/utils/py3.py:89
+                #     raise TypeError("Invalid argument %r for b()" % (s,))
+                # because I raise a NotImplementedError.
+                iterator=iter(b'blob data like an image or video'),
+                container=container,
+                object_name="test_object",
+                headers=cors_headers,
+            )
+        except NotImplementedError:
+            # Don't care about the response we'd have to mock anyway
+            # as long as we intercepted the request and checked its headers
+            pass
+        else:
+            self.fail('Expected NotImplementedError to be thrown to '
+                      'verify we actually checked the expected headers')
+
     def test__upload_object_manifest(self):
         hash_function = self.driver._get_hash_function()
         hash_function.update(b(''))


Mime
View raw message