libcloud-notifications mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From clewo...@apache.org
Subject [libcloud] 01/01: Add integration tests for Azure Storage driver
Date Sat, 20 Mar 2021 15:51:31 GMT
This is an automated email from the ASF dual-hosted git repository.

clewolff pushed a commit to branch storage-azure-integration-tests
in repository https://gitbox.apache.org/repos/asf/libcloud.git

commit 4b8d856672b3ebae3cba94f2c5f2252338d7c021
Author: Clemens Wolff <clewolff@microsoft.com>
AuthorDate: Sat Mar 20 11:46:18 2021 -0400

    Add integration tests for Azure Storage driver
---
 .../test/storage/test_azure_blobs_integration.py   | 401 +++++++++++++++++++++
 requirements-tests.txt                             |   1 +
 2 files changed, 402 insertions(+)

diff --git a/libcloud/test/storage/test_azure_blobs_integration.py b/libcloud/test/storage/test_azure_blobs_integration.py
new file mode 100644
index 0000000..ee813f6
--- /dev/null
+++ b/libcloud/test/storage/test_azure_blobs_integration.py
@@ -0,0 +1,401 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import base64
+import gzip
+import io
+import os
+import random
+import sys
+import re
+import socket
+import string
+import tempfile
+import time
+import unittest
+
+import requests
+
+try:
+    import docker
+except ImportError:
+    docker = None
+
+from libcloud.storage import providers, types
+
+MB = 1024 * 1024
+
+
+class Base:
+    class TestBase(unittest.TestCase):
+        provider = "azure_blobs"
+        account = None
+        secret = None
+
+        def setUp(self):
+            for required in "provider", "account", "secret":
+                value = getattr(self, required, None)
+                if value is None:
+                    raise unittest.SkipTest("config {} not set".format(required))
+
+            kwargs = {"key": self.account, "secret": self.secret}
+
+            for optional in "host", "port", "secure":
+                value = getattr(self, optional, None)
+                if value is not None:
+                    kwargs[optional] = value
+
+            self.driver = providers.get_driver(self.provider)(**kwargs)
+
+        def tearDown(self):
+            for container in self.driver.list_containers():
+                for obj in container.list_objects():
+                    obj.delete()
+                container.delete()
+
+        def test_containers(self):
+            # make a new container
+            container_name = _random_container_name()
+            container = self.driver.create_container(container_name)
+            self.assertEqual(container.name, container_name)
+            container = self.driver.get_container(container_name)
+            self.assertEqual(container.name, container_name)
+
+            # check that an existing container can't be re-created
+            with self.assertRaises(types.ContainerAlreadyExistsError):
+                self.driver.create_container(container_name)
+
+            # check that the new container can be listed
+            containers = self.driver.list_containers()
+            self.assertEqual([c.name for c in containers], [container_name])
+
+            # delete the container
+            self.driver.delete_container(container)
+
+            # check that a deleted container can't be looked up
+            with self.assertRaises(types.ContainerDoesNotExistError):
+                self.driver.get_container(container_name)
+
+            # check that the container is deleted
+            containers = self.driver.list_containers()
+            self.assertEqual([c.name for c in containers], [])
+
+        def _test_objects(self, do_upload, do_download, size=1 * MB):
+            content = os.urandom(size)
+            blob_name = "testblob"
+            container = self.driver.create_container(_random_container_name())
+
+            # upload a file
+            obj = do_upload(container, blob_name, content)
+            self.assertEqual(obj.name, blob_name)
+            obj = self.driver.get_object(container.name, blob_name)
+
+            # check that the file can be listed
+            blobs = self.driver.list_container_objects(container)
+            self.assertEqual([blob.name for blob in blobs], [blob_name])
+
+            # upload another file and check it's excluded in prefix listing
+            do_upload(container, blob_name[::-1], content[::-1])
+            blobs = self.driver.list_container_objects(
+                container, ex_prefix=blob_name[0:3]
+            )
+            self.assertEqual([blob.name for blob in blobs], [blob_name])
+
+            # check that the file can be read back
+            self.assertEqual(do_download(obj), content)
+
+            # delete the file
+            self.driver.delete_object(obj)
+
+            # check that a missing file can't be deleted or looked up
+            with self.assertRaises(types.ObjectDoesNotExistError):
+                self.driver.delete_object(obj)
+            with self.assertRaises(types.ObjectDoesNotExistError):
+                self.driver.get_object(container.name, blob_name)
+
+            # check that the file is deleted
+            blobs = self.driver.list_container_objects(container)
+            self.assertEqual([blob.name for blob in blobs], [blob_name[::-1]])
+
+        def test_objects(self, size=1 * MB):
+            def do_upload(container, blob_name, content):
+                infile = self._create_tempfile(content=content)
+                return self.driver.upload_object(infile, container, blob_name)
+
+            def do_download(obj):
+                outfile = self._create_tempfile()
+                self.driver.download_object(obj, outfile, overwrite_existing=True)
+                with open(outfile, "rb") as fobj:
+                    return fobj.read()
+
+            self._test_objects(do_upload, do_download, size)
+
+        def test_objects_range_downloads(self):
+            blob_name = "testblob-range"
+            content = b"0123456789"
+            container = self.driver.create_container(_random_container_name())
+
+            infile = self._create_tempfile(content=content)
+            obj = self.driver.upload_object(infile, container, blob_name)
+            self.assertEqual(obj.name, blob_name)
+            self.assertEqual(obj.size, len(content))
+
+            obj = self.driver.get_object(container.name, blob_name)
+            self.assertEqual(obj.name, blob_name)
+            self.assertEqual(obj.size, len(content))
+
+            values = [
+                {"start_bytes": 0, "end_bytes": 1, "expected_content": b"0"},
+                {"start_bytes": 1, "end_bytes": 5, "expected_content": b"1234"},
+                {"start_bytes": 5, "end_bytes": None, "expected_content": b"56789"},
+                {
+                    "start_bytes": 5,
+                    "end_bytes": len(content),
+                    "expected_content": b"56789",
+                },
+                {
+                    "start_bytes": 0,
+                    "end_bytes": None,
+                    "expected_content": b"0123456789",
+                },
+                {
+                    "start_bytes": 0,
+                    "end_bytes": len(content),
+                    "expected_content": b"0123456789",
+                },
+            ]
+
+            for value in values:
+                # 1. download_object_range
+                start_bytes = value["start_bytes"]
+                end_bytes = value["end_bytes"]
+                outfile = self._create_tempfile()
+
+                result = self.driver.download_object_range(
+                    obj,
+                    outfile,
+                    start_bytes=start_bytes,
+                    end_bytes=end_bytes,
+                    overwrite_existing=True,
+                )
+                self.assertTrue(result)
+
+                with open(outfile, "rb") as fobj:
+                    downloaded_content = fobj.read()
+
+                if end_bytes is not None:
+                    expected_content = content[start_bytes:end_bytes]
+                else:
+                    expected_content = content[start_bytes:]
+
+                msg = 'Expected "%s", got "%s" for values: %s' % (
+                    expected_content,
+                    downloaded_content,
+                    str(value),
+                )
+                self.assertEqual(downloaded_content, expected_content, msg)
+                self.assertEqual(downloaded_content, value["expected_content"], msg)
+
+                # 2. download_object_range_as_stream
+                downloaded_content = _read_stream(
+                    self.driver.download_object_range_as_stream(
+                        obj, start_bytes=start_bytes, end_bytes=end_bytes
+                    )
+                )
+                self.assertEqual(downloaded_content, expected_content)
+
+        @unittest.skipUnless(os.getenv("LARGE_FILE_SIZE_MB"), "config not set")
+        def test_objects_large(self):
+            size = int(float(os.environ["LARGE_FILE_SIZE_MB"]) * MB)
+            self.test_objects(size)
+
+        def test_objects_stream_io(self):
+            def do_upload(container, blob_name, content):
+                content = io.BytesIO(content)
+                return self.driver.upload_object_via_stream(
+                    content, container, blob_name
+                )
+
+            def do_download(obj):
+                return _read_stream(self.driver.download_object_as_stream(obj))
+
+            self._test_objects(do_upload, do_download)
+
+        def test_objects_stream_iterable(self):
+            def do_upload(container, blob_name, content):
+                content = iter([content[i : i + 1] for i in range(len(content))])
+                return self.driver.upload_object_via_stream(
+                    content, container, blob_name
+                )
+
+            def do_download(obj):
+                return _read_stream(self.driver.download_object_as_stream(obj))
+
+            self._test_objects(do_upload, do_download)
+
+        def test_upload_via_stream_with_content_encoding(self):
+            object_name = "content_encoding.gz"
+            content = gzip.compress(os.urandom(MB // 100))
+            container = self.driver.create_container(_random_container_name())
+            self.driver.upload_object_via_stream(
+                iter(content),
+                container,
+                object_name,
+                headers={"Content-Encoding": "gzip"},
+            )
+
+            obj = self.driver.get_object(container.name, object_name)
+
+            self.assertEqual(obj.extra.get("content_encoding"), "gzip")
+
+        def test_cdn_url(self):
+            content = os.urandom(MB // 100)
+            container = self.driver.create_container(_random_container_name())
+            obj = self.driver.upload_object_via_stream(iter(content), container, "cdn")
+
+            response = requests.get(self.driver.get_object_cdn_url(obj))
+            response.raise_for_status()
+
+            self.assertEqual(response.content, content)
+
+        def _create_tempfile(self, prefix="", content=b""):
+            fobj, path = tempfile.mkstemp(prefix=prefix, text=False)
+            os.write(fobj, content)
+            os.close(fobj)
+            self.addCleanup(os.remove, path)
+            return path
+
+    class ContainerTestBase(TestBase):
+        image = None
+        version = "latest"
+        environment = {}
+
+        host = "localhost"
+        port = None
+        secure = False
+
+        client = None
+        container = None
+
+        @classmethod
+        def setUpClass(cls):
+            if docker is None:
+                raise unittest.SkipTest("missing docker library")
+
+            try:
+                cls.client = docker.from_env()
+            except docker.errors.DockerException:
+                raise unittest.SkipTest("unable to create docker client")
+
+            for required in "image", "port":
+                value = getattr(cls, required, None)
+                if value is None:
+                    raise unittest.SkipTest("config {} not set".format(required))
+
+            cls.container = cls.client.containers.run(
+                "{}:{}".format(cls.image, cls.version),
+                detach=True,
+                auto_remove=True,
+                ports={cls.port: cls.port},
+                environment=cls.environment,
+            )
+
+            timeout = 10
+            start_time = time.perf_counter()
+
+            while True:
+                try:
+                    with socket.create_connection(
+                        (cls.host, cls.port), timeout=timeout
+                    ):
+                        break
+                except OSError as ex:
+                    time.sleep(1)
+                    if time.perf_counter() - start_time >= timeout:
+                        raise TimeoutError(
+                            "Waited too long for the port {} on host {} to start accepting
"
+                            "connections.".format(cls.port, cls.host)
+                        ) from ex
+
+        @classmethod
+        def tearDownClass(cls):
+            for line in cls.container.logs().splitlines():
+                print(line)
+            cls.container.kill()
+
+
+class AzuriteStorageTest(Base.ContainerTestBase):
+    account = "devstoreaccount1"
+    secret = (
+        "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6I"
+        "FsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
+    )
+
+    image = "arafato/azurite"
+    port = 10000
+    environment = {"executable": "blob"}
+
+    has_sas_support = False
+
+    def test_cdn_url(self):
+        if not self.has_sas_support:
+            self.skipTest("Storage backend has no account SAS support")
+
+
+class AzuriteV3StorageTest(AzuriteStorageTest):
+    image = "mcr.microsoft.com/azure-storage/azurite"
+    has_sas_support = True
+
+    def test_upload_via_stream_with_content_encoding(self):
+        self.skipTest(
+            "Possible bug in AzuriteV3, see https://github.com/Azure/Azurite/issues/629"
+        )
+
+
+class IotedgeStorageTest(Base.ContainerTestBase):
+    account = "".join(random.choice(string.ascii_lowercase) for _ in range(10))
+    secret = base64.b64encode(
+        "".join(random.choice(string.hexdigits) for _ in range(20)).encode("ascii")
+    ).decode("ascii")
+
+    image = "mcr.microsoft.com/azure-blob-storage"
+    port = 11002
+    environment = {
+        "LOCAL_STORAGE_ACCOUNT_NAME": account,
+        "LOCAL_STORAGE_ACCOUNT_KEY": secret,
+    }
+
+
+def _random_container_name(prefix="test"):
+    max_length = 63
+    suffix = "".join(random.choice(string.ascii_lowercase) for _ in range(max_length))
+    name = prefix + suffix
+    name = re.sub("[^a-z0-9-]", "-", name)
+    name = re.sub("-+", "-", name)
+    name = name[:max_length]
+    name = name.lower()
+    return name
+
+
+def _read_stream(stream):
+    buffer = io.BytesIO()
+    buffer.writelines(stream)
+    buffer.seek(0)
+    return buffer.read()
+
+
+if __name__ == "__main__":
+    sys.exit(unittest.main())
diff --git a/requirements-tests.txt b/requirements-tests.txt
index 7ebe7ca..b85a08b 100644
--- a/requirements-tests.txt
+++ b/requirements-tests.txt
@@ -5,6 +5,7 @@ pylint==2.4.4
 mock==3.0.5
 codecov==2.1.10
 coverage==4.5.4
+docker
 requests
 requests_mock
 pytest==5.3.2

Mime
View raw message