libcloud-notifications mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From to...@apache.org
Subject svn commit: r1103251 - in /incubator/libcloud/trunk: libcloud/storage/ libcloud/storage/drivers/ test/storage/
Date Sat, 14 May 2011 23:23:12 GMT
Author: tomaz
Date: Sat May 14 23:23:11 2011
New Revision: 1103251

URL: http://svn.apache.org/viewvc?rev=1103251&view=rev
Log:
Remove file_hash and add verify_hash argument and make it consistent across all
the drivers and update affects tests.

Modified:
    incubator/libcloud/trunk/libcloud/storage/base.py
    incubator/libcloud/trunk/libcloud/storage/drivers/cloudfiles.py
    incubator/libcloud/trunk/libcloud/storage/drivers/dummy.py
    incubator/libcloud/trunk/libcloud/storage/drivers/s3.py
    incubator/libcloud/trunk/libcloud/storage/types.py
    incubator/libcloud/trunk/test/storage/test_cloudfiles.py
    incubator/libcloud/trunk/test/storage/test_s3.py

Modified: incubator/libcloud/trunk/libcloud/storage/base.py
URL: http://svn.apache.org/viewvc/incubator/libcloud/trunk/libcloud/storage/base.py?rev=1103251&r1=1103250&r2=1103251&view=diff
==============================================================================
--- incubator/libcloud/trunk/libcloud/storage/base.py (original)
+++ incubator/libcloud/trunk/libcloud/storage/base.py Sat May 14 23:23:11 2011
@@ -122,9 +122,9 @@ class Container(object):
         return self.driver.get_object(container_name=self.name,
                                       object_name=object_name)
 
-    def upload_object(self, file_path, object_name, extra=None, file_hash=None):
+    def upload_object(self, file_path, object_name, extra=None, verify_hash=True):
         return self.driver.upload_object(
-            file_path, self, object_name, extra, file_hash)
+            file_path, self, object_name, extra, verify_hash)
 
     def upload_object_via_stream(self, iterator, object_name, extra=None):
         return self.driver.upload_object_via_stream(
@@ -301,7 +301,7 @@ class StorageDriver(object):
             'download_object_as_stream not implemented for this driver')
 
     def upload_object(self, file_path, container, object_name, extra=None,
-                      file_hash=None):
+                      verify_hash=True):
         """
         Upload an object.
 
@@ -317,10 +317,8 @@ class StorageDriver(object):
         @type extra: C{dict}
         @param extra: (optional) Extra attributes (driver specific).
 
-        @type file_hash: C{str}
-        @param file_hash: (optional) File hash. If provided object hash is
-                          on upload and if it doesn't match the one provided an
-                          exception is thrown.
+        @type verify_hash: C{boolean}
+        @param verify_hash: True to do a file integrity check.
         """
         raise NotImplementedError(
             'upload_object not implemented for this driver')

Modified: incubator/libcloud/trunk/libcloud/storage/drivers/cloudfiles.py
URL: http://svn.apache.org/viewvc/incubator/libcloud/trunk/libcloud/storage/drivers/cloudfiles.py?rev=1103251&r1=1103250&r2=1103251&view=diff
==============================================================================
--- incubator/libcloud/trunk/libcloud/storage/drivers/cloudfiles.py (original)
+++ incubator/libcloud/trunk/libcloud/storage/drivers/cloudfiles.py Sat May 14 23:23:11 2011
@@ -203,7 +203,6 @@ class CloudFilesStorageDriver(StorageDri
         response = self.connection.request('/%s/%s' % (container_name,
                                                        object_name),
                                                        method='HEAD')
-
         if response.status in [ httplib.OK, httplib.NO_CONTENT ]:
             obj = self._headers_to_object(
                 object_name, container, response.headers)
@@ -309,7 +308,7 @@ class CloudFilesStorageDriver(StorageDri
                                 success_status_code=httplib.OK)
 
     def upload_object(self, file_path, container, object_name, extra=None,
-                      file_hash=None):
+                      verify_hash=True):
         """
         Upload an object.
 
@@ -322,7 +321,7 @@ class CloudFilesStorageDriver(StorageDri
                                 upload_func=upload_func,
                                 upload_func_kwargs=upload_func_kwargs,
                                 extra=extra, file_path=file_path,
-                                file_hash=file_hash)
+                                verify_hash=verify_hash)
 
     def upload_object_via_stream(self, iterator,
                                  container, object_name, extra=None):
@@ -354,7 +353,7 @@ class CloudFilesStorageDriver(StorageDri
 
     def _put_object(self, container, object_name, upload_func,
                     upload_func_kwargs, extra=None, file_path=None,
-                    iterator=None, file_hash=None):
+                    iterator=None, verify_hash=True):
         extra = extra or {}
         container_name_cleaned = self._clean_container_name(container.name)
         object_name_cleaned = self._clean_object_name(object_name)
@@ -362,9 +361,6 @@ class CloudFilesStorageDriver(StorageDri
         meta_data = extra.get('meta_data', None)
 
         headers = {}
-        if not iterator and file_hash:
-            headers['ETag'] = file_hash
-
         if meta_data:
             for key, value in meta_data.iteritems():
                 key = 'X-Object-Meta-%s' % (key)
@@ -382,17 +378,23 @@ class CloudFilesStorageDriver(StorageDri
 
         response = result_dict['response'].response
         bytes_transferred = result_dict['bytes_transferred']
+        print result_dict['data_hash']
+        server_hash = result_dict['response'].headers.get('etag', None)
 
         if response.status == httplib.EXPECTATION_FAILED:
             raise LibcloudError(value='Missing content-type header',
                                 driver=self)
-        elif response.status == httplib.UNPROCESSABLE_ENTITY:
+        elif verify_hash and not server_hash:
+            raise LibcloudError(value='Server didn\'t return etag',
+                                driver=self)
+        elif (verify_hash and result_dict['data_hash'] != server_hash):
             raise ObjectHashMismatchError(
-                value='MD5 hash checksum does not match',
+                value=('MD5 hash checksum does not match (expected=%s, ' +
+                       'actual=%s)') % (result_dict['data_hash'], server_hash),
                 object_name=object_name, driver=self)
         elif response.status == httplib.CREATED:
             obj = Object(
-                name=object_name, size=bytes_transferred, hash=file_hash,
+                name=object_name, size=bytes_transferred, hash=server_hash,
                 extra=None, meta_data=meta_data, container=container,
                 driver=self)
 

Modified: incubator/libcloud/trunk/libcloud/storage/drivers/dummy.py
URL: http://svn.apache.org/viewvc/incubator/libcloud/trunk/libcloud/storage/drivers/dummy.py?rev=1103251&r1=1103250&r2=1103251&view=diff
==============================================================================
--- incubator/libcloud/trunk/libcloud/storage/drivers/dummy.py (original)
+++ incubator/libcloud/trunk/libcloud/storage/drivers/dummy.py Sat May 14 23:23:11 2011
@@ -15,6 +15,7 @@
 
 import os.path
 import random
+import hashlib
 
 from libcloud.common.types import LibcloudError
 
@@ -48,14 +49,19 @@ class DummyFileObject(file):
 
 class DummyIterator(object):
     def __init__(self, data=None):
+        self.hash = hashlib.md5()
         self._data = data or []
         self._current_item = 0
 
+    def get_md5_hash(self):
+        return self.hash.hexdigest()
+
     def next(self):
         if self._current_item == len(self._data):
             raise StopIteration
 
         value = self._data[self._current_item]
+        self.hash.update(value)
         self._current_item += 1
         return value
 

Modified: incubator/libcloud/trunk/libcloud/storage/drivers/s3.py
URL: http://svn.apache.org/viewvc/incubator/libcloud/trunk/libcloud/storage/drivers/s3.py?rev=1103251&r1=1103250&r2=1103251&view=diff
==============================================================================
--- incubator/libcloud/trunk/libcloud/storage/drivers/s3.py (original)
+++ incubator/libcloud/trunk/libcloud/storage/drivers/s3.py Sat May 14 23:23:11 2011
@@ -290,7 +290,7 @@ class S3StorageDriver(StorageDriver):
                                 success_status_code=httplib.OK)
 
     def upload_object(self, file_path, container, object_name, extra=None,
-                      file_hash=None, ex_storage_class=None):
+                      verify_hash=True, ex_storage_class=None):
         upload_func = self._upload_file
         upload_func_kwargs = { 'file_path': file_path }
 
@@ -298,7 +298,7 @@ class S3StorageDriver(StorageDriver):
                                 upload_func=upload_func,
                                 upload_func_kwargs=upload_func_kwargs,
                                 extra=extra, file_path=file_path,
-                                file_hash=file_hash,
+                                verify_hash=verify_hash,
                                 storage_class=ex_storage_class)
 
     def upload_object_via_stream(self, iterator, container, object_name,
@@ -328,7 +328,7 @@ class S3StorageDriver(StorageDriver):
 
     def _put_object(self, container, object_name, upload_func,
                     upload_func_kwargs, extra=None, file_path=None,
-                    iterator=None, file_hash=None, storage_class=None):
+                    iterator=None, verify_hash=True, storage_class=None):
         headers = {}
         extra = extra or {}
         storage_class = storage_class or 'standard'
@@ -342,9 +342,6 @@ class S3StorageDriver(StorageDriver):
         content_type = extra.get('content_type', None)
         meta_data = extra.get('meta_data', None)
 
-        if not iterator and file_hash:
-            headers['Content-MD5'] = base64.b64encode(file_hash.decode('hex'))
-
         if meta_data:
             for key, value in meta_data.iteritems():
                 key = 'x-amz-meta-%s' % (key)
@@ -368,19 +365,22 @@ class S3StorageDriver(StorageDriver):
         bytes_transferred = result_dict['bytes_transferred']
         headers = response.headers
         response = response.response
+        server_hash = headers['etag'].replace('"', '')
 
-        if (file_hash and response.status == httplib.BAD_REQUEST) or \
-           (file_hash and file_hash != headers['etag'].replace('"', '')):
+        if (verify_hash and result_dict['data_hash'] != server_hash):
             raise ObjectHashMismatchError(
                 value='MD5 hash checksum does not match',
                 object_name=object_name, driver=self)
         elif response.status == httplib.OK:
             obj = Object(
-                name=object_name, size=bytes_transferred, hash=file_hash,
+                name=object_name, size=bytes_transferred, hash=server_hash,
                 extra=None, meta_data=meta_data, container=container,
                 driver=self)
 
             return obj
+        else:
+            raise LibcloudError('Unexpected status code, status_code=%s' % (response.status),
+                                driver=self)
 
     def _to_containers(self, obj, xpath):
         return [ self._to_container(element) for element in \

Modified: incubator/libcloud/trunk/libcloud/storage/types.py
URL: http://svn.apache.org/viewvc/incubator/libcloud/trunk/libcloud/storage/types.py?rev=1103251&r1=1103250&r2=1103251&view=diff
==============================================================================
--- incubator/libcloud/trunk/libcloud/storage/types.py (original)
+++ incubator/libcloud/trunk/libcloud/storage/types.py Sat May 14 23:23:11 2011
@@ -67,8 +67,8 @@ class ObjectError(LibcloudError):
         super(ObjectError, self).__init__(value=value, driver=driver)
 
     def __str__(self):
-        return '<%s in %s, object = %s>' % (self.error_type, repr(self.driver),
-                                            self.object_name)
+        return '<%s in %s, value=%s, object = %s>' % (self.error_type, repr(self.driver),
+                                                      self.value, self.object_name)
 
 class ContainerAlreadyExistsError(ContainerError):
     error_type = 'ContainerAlreadyExistsError'

Modified: incubator/libcloud/trunk/test/storage/test_cloudfiles.py
URL: http://svn.apache.org/viewvc/incubator/libcloud/trunk/test/storage/test_cloudfiles.py?rev=1103251&r1=1103250&r2=1103251&view=diff
==============================================================================
--- incubator/libcloud/trunk/test/storage/test_cloudfiles.py (original)
+++ incubator/libcloud/trunk/test/storage/test_cloudfiles.py Sat May 14 23:23:11 2011
@@ -35,6 +35,8 @@ from libcloud.storage.drivers.dummy impo
 from test import StorageMockHttp, MockRawResponse # pylint: disable-msg=E0611
 from test.file_fixtures import StorageFileFixtures # pylint: disable-msg=E0611
 
+current_hash = None
+
 class CloudFilesTests(unittest.TestCase):
 
     def setUp(self):
@@ -279,7 +281,7 @@ class CloudFilesTests(unittest.TestCase)
         try:
             self.driver.upload_object(file_path=file_path, container=container,
                                       object_name=object_name,
-                                      file_hash='footest123')
+                                      verify_hash=True)
         except ObjectHashMismatchError:
             pass
         else:
@@ -598,16 +600,19 @@ class CloudFilesMockRawResponse(MockRawR
         # test_object_upload_success
 
         body = ''
-        headers = copy.deepcopy(self.base_headers)
-        headers.update(headers)
+        headers = {}
+        headers.update(self.base_headers)
+        headers['etag'] = 'hash343hhash89h932439jsaa89'
         return (httplib.CREATED, body, headers, httplib.responses[httplib.OK])
 
     def  _v1_MossoCloudFS_foo_bar_container_foo_test_upload_INVALID_HASH(
         self, method, url, body, headers):
         # test_object_upload_invalid_hash
         body = ''
-        headers = self.base_headers
-        return (httplib.UNPROCESSABLE_ENTITY, body, headers,
+        headers = {}
+        headers.update(self.base_headers)
+        headers['etag'] = 'foobar'
+        return (httplib.CREATED, body, headers,
                 httplib.responses[httplib.OK])
 
     def _v1_MossoCloudFS_foo_bar_container_foo_bar_object(
@@ -641,10 +646,13 @@ class CloudFilesMockRawResponse(MockRawR
         self, method, url, body, headers):
 
         # test_upload_object_via_stream_success
+        headers = {}
+        headers.update(self.base_headers)
+        headers['etag'] = '577ef1154f3240ad5b9b413aa7346a1e'
         body = 'test'
         return (httplib.CREATED,
                 body,
-                self.base_headers,
+                headers,
                 httplib.responses[httplib.OK])
 
 if __name__ == '__main__':

Modified: incubator/libcloud/trunk/test/storage/test_s3.py
URL: http://svn.apache.org/viewvc/incubator/libcloud/trunk/test/storage/test_s3.py?rev=1103251&r1=1103250&r2=1103251&view=diff
==============================================================================
--- incubator/libcloud/trunk/test/storage/test_s3.py (original)
+++ incubator/libcloud/trunk/test/storage/test_s3.py Sat May 14 23:23:11 2011
@@ -277,7 +277,7 @@ class S3Tests(unittest.TestCase):
         try:
             self.driver.upload_object(file_path=file_path, container=container,
                                       object_name=object_name,
-                                      file_hash='0cc175b9c0f1b6a831c399e269772661',
+                                      verify_hash=True,
                                       ex_storage_class='invalid-class')
         except ValueError, e:
             self.assertTrue(str(e).lower().find('invalid storage class') != -1)
@@ -302,7 +302,7 @@ class S3Tests(unittest.TestCase):
         try:
             self.driver.upload_object(file_path=file_path, container=container,
                                       object_name=object_name,
-                                      file_hash='0cc175b9c0f1b6a831c399e269772661')
+                                      verify_hash=True)
         except ObjectHashMismatchError:
             pass
         else:
@@ -328,7 +328,7 @@ class S3Tests(unittest.TestCase):
         try:
             self.driver.upload_object(file_path=file_path, container=container,
                                       object_name=object_name,
-                                      file_hash='0cc175b9c0f1b6a831c399e269772661')
+                                      verify_hash=True)
         except ObjectHashMismatchError:
             pass
         else:
@@ -351,7 +351,7 @@ class S3Tests(unittest.TestCase):
         obj = self.driver.upload_object(file_path=file_path, container=container,
                                       object_name=object_name,
                                       extra=extra,
-                                      file_hash='0cc175b9c0f1b6a831c399e269772661')
+                                      verify_hash=True)
         self.assertEqual(obj.name, 'foo_test_upload')
         self.assertEqual(obj.size, 1000)
         self.assertTrue('some-value' in obj.meta_data)
@@ -564,8 +564,10 @@ class S3MockRawResponse(MockRawResponse)
 
     def _foo_bar_container_foo_test_upload_INVALID_HASH1(self, method, url, body, headers):
         body = ''
+        headers = {}
+        headers['etag'] = '"foobar"'
         # test_upload_object_invalid_hash1
-        return (httplib.BAD_REQUEST,
+        return (httplib.OK,
                 body,
                 headers,
                 httplib.responses[httplib.OK])



Mime
View raw message