libcloud-notifications mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From clewo...@apache.org
Subject [libcloud] 02/02: Add support for blob storage in Azure Government
Date Sun, 07 Feb 2021 16:17:41 GMT
This is an automated email from the ASF dual-hosted git repository.

clewolff pushed a commit to branch storage-azureblobs-fix-1551-govcloud
in repository https://gitbox.apache.org/repos/asf/libcloud.git

commit 505894fa9e2b2776889b673789651ff46734c7d7
Author: Clemens Wolff <clewolff@microsoft.com>
AuthorDate: Sun Feb 7 11:02:54 2021 -0500

    Add support for blob storage in Azure Government
    
    Note that this change also enables using the driver with Azure China and
    Azure Private Link.
---
 CHANGES.rst                                    | 13 ++++++++
 docs/examples/storage/azure/instantiate_gov.py |  8 +++++
 docs/storage/drivers/azure_blobs.rst           | 16 +++++++++
 libcloud/storage/drivers/azure_blobs.py        | 45 +++++++++++++++++++-------
 4 files changed, 71 insertions(+), 11 deletions(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index a99a8bd..f4d6301 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,6 +1,19 @@
 ´╗┐Changelog
 =========
 
+Changes in Apache Libcloud 3.3.2
+--------------------------------
+
+Storage
+~~~~~~~
+
+- [Azure Blobs] Enable the Azure storage driver to be used with
+  Azure Government, Azure China, and Azure Private Link by setting
+  the driver host argument to the endpoint suffix for the environment.
+
+  Reported by Melissa Kersh - @mkcello96
+  (GITHUB-1551)
+
 Changes in Apache Libcloud 3.3.1
 --------------------------------
 
diff --git a/docs/examples/storage/azure/instantiate_gov.py b/docs/examples/storage/azure/instantiate_gov.py
new file mode 100644
index 0000000..46a6c01
--- /dev/null
+++ b/docs/examples/storage/azure/instantiate_gov.py
@@ -0,0 +1,8 @@
+from libcloud.storage.types import Provider
+from libcloud.storage.providers import get_driver
+
+cls = get_driver(Provider.AZURE_BLOBS)
+
+driver = cls(key='your storage account name',
+             secret='your access key',
+             host='blob.core.usgovcloudapi.net')
diff --git a/docs/storage/drivers/azure_blobs.rst b/docs/storage/drivers/azure_blobs.rst
index 824889c..956b3f1 100644
--- a/docs/storage/drivers/azure_blobs.rst
+++ b/docs/storage/drivers/azure_blobs.rst
@@ -33,6 +33,19 @@ below.
 .. literalinclude:: /examples/storage/azure/instantiate.py
    :language: python
 
+Connecting to Azure Government
+------------------------------
+
+To target an `Azure Government`_ storage account, you can instantiate the driver
+by setting a custom storage host argument as shown below.
+
+.. literalinclude:: /examples/storage/azure/instantiate_gov.py
+   :language: python
+
+Setting a custom host argument can also be leveraged to customize the blob
+endpoint and connect to a storage account in `Azure China`_ or
+`Azure Private Link`_.
+
 Connecting to self-hosted Azure Storage implementations
 -------------------------------------------------------
 
@@ -51,3 +64,6 @@ Azure Storage implementations such as `Azure Blob Storage on IoT Edge`_.
 .. _`BlockBlobStorage accounts`: https://docs.microsoft.com/en-us/azure/storage/common/storage-account-overview#blockblobstorage-accounts
 .. _`Azurite storage emulator`: https://github.com/Azure/Azurite
 .. _`Azure Blob Storage on IoT Edge`: https://docs.microsoft.com/en-us/azure/iot-edge/how-to-store-data-blob
+.. _`Azure Government`: https://docs.microsoft.com/en-us/azure/azure-government/documentation-government-developer-guide
+.. _`Azure China`: https://docs.microsoft.com/en-us/azure/china/resources-developer-guide
+.. _`Azure Private Link`: https://docs.microsoft.com/en-us/azure/private-link/private-link-overview
diff --git a/libcloud/storage/drivers/azure_blobs.py b/libcloud/storage/drivers/azure_blobs.py
index c332521..6c65c51 100644
--- a/libcloud/storage/drivers/azure_blobs.py
+++ b/libcloud/storage/drivers/azure_blobs.py
@@ -68,6 +68,9 @@ AZURE_LEASE_PERIOD = int(
 )
 
 AZURE_STORAGE_HOST_SUFFIX = 'blob.core.windows.net'
+AZURE_STORAGE_HOST_SUFFIX_CHINA = 'blob.core.chinacloudapi.cn'
+AZURE_STORAGE_HOST_SUFFIX_GOVERNMENT = 'blob.core.usgovcloudapi.net'
+AZURE_STORAGE_HOST_SUFFIX_PRIVATELINK = 'privatelink.blob.core.windows.net'
 
 AZURE_STORAGE_CDN_URL_DATE_FORMAT = '%Y-%m-%dT%H:%M:%SZ'
 
@@ -173,9 +176,10 @@ class AzureBlobsConnection(AzureConnection):
     these deployments, the parameter ``account_prefix`` must be set on the
     connection. This is done by instantiating the driver with arguments such
     as ``host='somewhere.tld'`` and ``key='theaccount'``. To specify a custom
-    host without an account prefix, e.g. for use-cases where the custom host
-    implements an auditing proxy or similar, the driver can be instantiated
-    with ``host='theaccount.somewhere.tld'`` and ``key=''``.
+    host without an account prefix, e.g. to connect to Azure Government or
+    Azure China, the driver can be instantiated with the appropriate storage
+    endpoint suffix, e.g. ``host='blob.core.usgovcloudapi.net'`` and
+    ``key='theaccount'``.
 
     :param account_prefix: Optional prefix identifying the storage account.
                            Used when connecting to a custom deployment of the
@@ -206,7 +210,7 @@ class AzureBlobsStorageDriver(StorageDriver):
 
     def __init__(self, key, secret=None, secure=True, host=None, port=None,
                  **kwargs):
-        self._host_argument_set = bool(host)
+        self._host = host
 
         # B64decode() this key and keep it, so that we don't have to do
         # so for every request. Minor performance improvement
@@ -217,15 +221,34 @@ class AzureBlobsStorageDriver(StorageDriver):
                                                       port=port, **kwargs)
 
     def _ex_connection_class_kwargs(self):
-        result = {}
-
-        # host argument has precedence
-        if not self._host_argument_set:
-            result['host'] = '%s.%s' % (self.key, AZURE_STORAGE_HOST_SUFFIX)
+        # if the user didn't provide a custom host value, assume we're
+        # targeting the default Azure Storage endpoints
+        if self._host is None:
+            return {'host': '%s.%s' % (self.key, AZURE_STORAGE_HOST_SUFFIX)}
+
+        # connecting to a special storage region like Azure Government or
+        # Azure China requires setting a custom storage endpoint but we
+        # still use the same scheme to identify a specific account as for
+        # the standard storage endpoint
+        try:
+            host_suffix = next(
+                host_suffix
+                for host_suffix in (
+                    AZURE_STORAGE_HOST_SUFFIX_CHINA,
+                    AZURE_STORAGE_HOST_SUFFIX_GOVERNMENT,
+                    AZURE_STORAGE_HOST_SUFFIX_PRIVATELINK,
+                )
+                if self._host.endswith(host_suffix)
+            )
+        except StopIteration:
+            pass
         else:
-            result['account_prefix'] = self.key
+            return {'host': '%s.%s' % (self.key, host_suffix)}
 
-        return result
+        # if the host isn't targeting one of the special storage regions, it
+        # must be pointing to Azurite or IoT Edge Storage so switch to prefix
+        # identification
+        return {'account_prefix': self.key}
 
     def _xml_to_container(self, node):
         """


Mime
View raw message