libcloud-notifications mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From quent...@apache.org
Subject [1/4] libcloud git commit: Limit number of retries in destroy_node.
Date Wed, 08 Nov 2017 03:36:42 GMT
Repository: libcloud
Updated Branches:
  refs/heads/trunk 42709622c -> 0c5bee8b1


Limit number of retries in destroy_node.

Signed-off-by: Quentin Pradet <quentinp@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo
Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/a71e9557
Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/a71e9557
Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/a71e9557

Branch: refs/heads/trunk
Commit: a71e95579dd461dd180c798091bce1216924165c
Parents: 4270962
Author: Peter Amstutz <peter.amstutz@curoverse.com>
Authored: Mon Oct 16 16:12:45 2017 -0400
Committer: Quentin Pradet <quentinp@apache.org>
Committed: Wed Nov 8 07:32:04 2017 +0400

----------------------------------------------------------------------
 libcloud/compute/drivers/azure_arm.py | 38 +++++++++++++++++++++---------
 1 file changed, 27 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/libcloud/blob/a71e9557/libcloud/compute/drivers/azure_arm.py
----------------------------------------------------------------------
diff --git a/libcloud/compute/drivers/azure_arm.py b/libcloud/compute/drivers/azure_arm.py
index fcc9ef3..db7ef65 100644
--- a/libcloud/compute/drivers/azure_arm.py
+++ b/libcloud/compute/drivers/azure_arm.py
@@ -691,7 +691,10 @@ class AzureNodeDriver(NodeDriver):
             else:
                 return False
 
-    def destroy_node(self, node, ex_destroy_nic=True, ex_destroy_vhd=True):
+    def destroy_node(self, node,
+                     ex_destroy_nic=True,
+                     ex_destroy_vhd=True,
+                     ex_retries=10):
         """
         Destroy a node.
 
@@ -706,6 +709,10 @@ class AzureNodeDriver(NodeDriver):
         this node (default True).
         :type node: ``bool``
 
+        :param ex_retries: Number of times to retry checking if the node is gone,
+        destroying the NIC or destroying the VHD.
+        :type node: ``int``
+
         :return: True if the destroy was successful, raises exception
         otherwise.
         :rtype: ``bool``
@@ -733,12 +740,14 @@ class AzureNodeDriver(NodeDriver):
 
         # Poll until the node actually goes away (otherwise attempt to delete
         # NIC and VHD will fail with "resource in use" errors).
-        while do_node_polling:
+        retries = ex_retries
+        while do_node_polling and retries > 0:
             try:
                 time.sleep(10)
                 self.connection.request(
                     node.id,
                     params={"api-version": RESOURCE_API_VERSION})
+                retries -= 1
             except BaseHTTPError as h:
                 if h.code in (204, 404):
                     # Node is gone
@@ -752,13 +761,16 @@ class AzureNodeDriver(NodeDriver):
             node.extra["properties"]["networkProfile"]["networkInterfaces"]
         if ex_destroy_nic:
             for nic in interfaces:
-                while True:
+                retries = ex_retries
+                while retries > 0:
                     try:
                         self.ex_destroy_nic(self._to_nic(nic))
                         break
                     except BaseHTTPError as h:
+                        retries -= 1
                         if (h.code == 400 and
-                                h.message.startswith("[NicInUse]")):
+                                h.message.startswith("[NicInUse]") and
+                                retries > 0):
                             time.sleep(10)
                         else:
                             raise
@@ -767,19 +779,23 @@ class AzureNodeDriver(NodeDriver):
         vhd = node.extra["properties"]["storageProfile"]["osDisk"].get("vhd")
         if ex_destroy_vhd and vhd is not None:
             resourceGroup = node.id.split("/")[4]
-            while True:
+            retries = ex_retries
+            while retries > 0:
                 try:
-                    if self._ex_delete_old_vhd(
-                            resourceGroup,
-                            vhd["uri"]):
+                    if self._ex_delete_old_vhd(resourceGroup, vhd["uri"]):
                         break
                     # Unfortunately lease errors usually result in it returning
                     # "False" with no more information.  Need to wait and try
                     # again.
                 except LibcloudError as e:
-                    if "LeaseIdMissing" in str(e):
-                        # If we get an lease error, need to wait and try again.
-                        pass
+                    retries -= 1
+                    if "LeaseIdMissing" in str(e) and retries > 0:
+                        # Unfortunately lease errors
+                        # (which occur if the vhd blob
+                        # hasn't yet been released by the VM being destroyed)
+                        # get raised as plain
+                        # LibcloudError.  Wait a bit and try again.
+                        time.sleep(10)
                     else:
                         raise
                 time.sleep(10)


Mime
View raw message