libcloud-notifications mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From anthonys...@apache.org
Subject [37/51] [abbrv] libcloud git commit: download_object_as_stream now returns a proper iterator
Date Mon, 09 Jan 2017 04:52:57 GMT
download_object_as_stream now returns a proper iterator


Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo
Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/109ecad9
Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/109ecad9
Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/109ecad9

Branch: refs/heads/trunk
Commit: 109ecad94257615b57f196f3c36d13d7b3c8e8fd
Parents: 5528f5b
Author: Anthony Shaw <anthonyshaw@apache.org>
Authored: Fri Jan 6 19:52:51 2017 +1100
Committer: Anthony Shaw <anthonyshaw@apache.org>
Committed: Fri Jan 6 19:52:51 2017 +1100

----------------------------------------------------------------------
 libcloud/common/base.py        | 15 +++++++++++----
 libcloud/httplib_ssl.py        |  7 ++++---
 libcloud/storage/drivers/s3.py |  4 ++--
 3 files changed, 17 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/libcloud/blob/109ecad9/libcloud/common/base.py
----------------------------------------------------------------------
diff --git a/libcloud/common/base.py b/libcloud/common/base.py
index 15233a9..5266163 100644
--- a/libcloud/common/base.py
+++ b/libcloud/common/base.py
@@ -152,6 +152,7 @@ class Response(object):
         self.error = response.reason
         self.status = response.status_code
         self.request = response.request
+        self.iter_content = response.iter_content
 
         self.body = response.text.strip() \
             if response.text is not None and hasattr(response.text, 'strip') \
@@ -501,7 +502,7 @@ class Connection(object):
         self.ua.append(token)
 
     def request(self, action, params=None, data=None, headers=None,
-                method='GET', raw=False):
+                method='GET', raw=False, stream=False):
         """
         Request a given `action`.
 
@@ -531,6 +532,10 @@ class Connection(object):
                      and use the rawResponseCls class. This is used with
                      storage API when uploading a file.
 
+        :type stream: ``bool``
+        :param stream: True to return an iterator in Response.iter_content and allow
+                    streaming of the response data (for downloading large files)
+
         :return: An :class:`Response` instance.
         :rtype: :class:`Response` instance
 
@@ -601,7 +606,8 @@ class Connection(object):
                     method=method,
                     url=url,
                     body=data,
-                    headers=headers)
+                    headers=headers,
+                    stream=stream)
             else:
                 if retry_enabled:
                     retry_request = retry(timeout=self.timeout,
@@ -610,10 +616,11 @@ class Connection(object):
                     retry_request(self.connection.request)(method=method,
                                                            url=url,
                                                            body=data,
-                                                           headers=headers)
+                                                           headers=headers,
+                                                           stream=stream)
                 else:
                     self.connection.request(method=method, url=url, body=data,
-                                            headers=headers)
+                                            headers=headers, stream=stream)
         except socket.gaierror:
             e = sys.exc_info()[1]
             message = str(e)

http://git-wip-us.apache.org/repos/asf/libcloud/blob/109ecad9/libcloud/httplib_ssl.py
----------------------------------------------------------------------
diff --git a/libcloud/httplib_ssl.py b/libcloud/httplib_ssl.py
index 640d31a..ac20ee7 100644
--- a/libcloud/httplib_ssl.py
+++ b/libcloud/httplib_ssl.py
@@ -180,19 +180,20 @@ class LibcloudConnection(LibcloudBaseConnection):
             self.set_http_proxy(proxy_url=proxy_url)
         self.session.timeout = kwargs.get('timeout', 60)
 
-    def request(self, method, url, body=None, headers=None, raw=False):
+    def request(self, method, url, body=None, headers=None, raw=False,
+                stream=False):
         self.response = self.session.request(
             method=method.lower(),
             url=''.join([self.host, url]),
             data=body,
             headers=headers,
             allow_redirects=1,
-            stream=raw,
+            stream=stream,
             verify=self.ca_cert if self.ca_cert is not None else self.verify
         )
 
     def prepared_request(self, method, url, body=None,
-                         headers=None, raw=False):
+                         headers=None, raw=False, stream=False):
         req = requests.Request(method, ''.join([self.host, url]),
                                data=body, headers=headers)
 

http://git-wip-us.apache.org/repos/asf/libcloud/blob/109ecad9/libcloud/storage/drivers/s3.py
----------------------------------------------------------------------
diff --git a/libcloud/storage/drivers/s3.py b/libcloud/storage/drivers/s3.py
index 28e205e..b0cbd82 100644
--- a/libcloud/storage/drivers/s3.py
+++ b/libcloud/storage/drivers/s3.py
@@ -414,11 +414,11 @@ class BaseS3StorageDriver(StorageDriver):
 
     def download_object_as_stream(self, obj, chunk_size=None):
         obj_path = self._get_object_path(obj.container, obj.name)
-        response = self.connection.request(obj_path, method='GET', raw=True)
+        response = self.connection.request(obj_path, method='GET', stream=True)
 
         return self._get_object(obj=obj, callback=read_in_chunks,
                                 response=response,
-                                callback_kwargs={'iterator': response.response,
+                                callback_kwargs={'iterator': response.iter_content,
                                                  'chunk_size': chunk_size},
                                 success_status_code=httplib.OK)
 


Mime
View raw message