libcloud-notifications mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vd...@apache.org
Subject [1/2] libcloud git commit: list calls paginate in OpenStackNodeDriver
Date Tue, 04 Dec 2018 09:51:57 GMT
Repository: libcloud
Updated Branches:
  refs/heads/trunk a4d0081b2 -> 1a3ebe5d6


list calls paginate in OpenStackNodeDriver

the default max_limits for OpenStack is 1000. If you have more than 1000
resources (i.e. snapshots) then everything but the newest 1000 will not be
listed. If you set the max_limits lower even less will not be returned, etc.
This change implements pagination for ex_list_snapshots, ex_list_ports,
list_volumes and list_nodes in the OpenStack_2_NodeDriver.

Signed-off-by: Rick van de Loo <rickvandeloo@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo
Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/1aaeff45
Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/1aaeff45
Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/1aaeff45

Branch: refs/heads/trunk
Commit: 1aaeff4537dbe23886cff505ef3f5c708d823995
Parents: a4d0081
Author: Rick van de Loo <rickvandeloo@gmail.com>
Authored: Wed Nov 28 17:29:45 2018 +0100
Committer: Rick van de Loo <rickvandeloo@gmail.com>
Committed: Tue Dec 4 10:49:30 2018 +0100

----------------------------------------------------------------------
 libcloud/compute/drivers/openstack.py           | 84 ++++++++++++++++++--
 .../openstack_v1.1/_v2_0__snapshots.json        |  2 +-
 .../_v2_0__snapshots_paginate_start.json        | 52 ++++++++++++
 libcloud/test/compute/test_openstack.py         | 36 ++++++++-
 4 files changed, 163 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/libcloud/blob/1aaeff45/libcloud/compute/drivers/openstack.py
----------------------------------------------------------------------
diff --git a/libcloud/compute/drivers/openstack.py b/libcloud/compute/drivers/openstack.py
index 5affde6..1d2ba4b 100644
--- a/libcloud/compute/drivers/openstack.py
+++ b/libcloud/compute/drivers/openstack.py
@@ -31,6 +31,7 @@ from libcloud.utils.py3 import httplib
 from libcloud.utils.py3 import b
 from libcloud.utils.py3 import next
 from libcloud.utils.py3 import urlparse
+from libcloud.utils.py3 import parse_qs
 
 
 from libcloud.common.openstack import OpenStackBaseConnection
@@ -70,6 +71,8 @@ ATOM_NAMESPACE = "http://www.w3.org/2005/Atom"
 
 DEFAULT_API_VERSION = '1.1'
 
+PAGINATION_LIMIT = 1000
+
 
 class OpenStackComputeConnection(OpenStackBaseConnection):
     # default config for http://devstack.org/
@@ -169,6 +172,57 @@ class OpenStackNodeDriver(NodeDriver, OpenStackDriverMixin):
         OpenStackDriverMixin.__init__(self, **kwargs)
         super(OpenStackNodeDriver, self).__init__(*args, **kwargs)
 
+    @staticmethod
+    def _paginated_request(url, obj, connection, params=None):
+        """
+        Perform multiple calls in order to have a full list of elements when
+        the API responses are paginated.
+
+        :param url: API endpoint
+        :type url: ``str``
+
+        :param obj: Result object key
+        :type obj: ``str``
+
+        :param connection: The API connection to use to perform the request
+        :type connection: ``obj``
+
+        :param params: Any request parameters
+        :type params: ``dict``
+
+        :return: ``list`` of API response objects
+        :rtype: ``list``
+        """
+        params = params or {}
+        objects = list()
+        loop_count = 0
+        while True:
+            data = connection.request(url, params=params)
+            values = data.object.get(obj, list())
+            objects.extend(values)
+            links = data.object.get('%s_links' % obj, list())
+            next_links = [n for n in links if n['rel'] == 'next']
+            if next_links:
+                next_link = next_links[0]
+                query = urlparse.urlparse(next_link['href'])
+                # The query[4] references the query parameters from the url
+                params.update(parse_qs(query[4]))
+            else:
+                break
+
+            # Prevent the pagination from looping indefinitely in case
+            # the API returns a loop for some reason.
+            loop_count += 1
+            if loop_count > PAGINATION_LIMIT:
+                raise OpenStackException(
+                    'Pagination limit reached for %s, the limit is %d. '
+                    'This might indicate that your API is returning a '
+                    'looping next target for pagination!' % (
+                        url, PAGINATION_LIMIT
+                    ), None
+                )
+        return {obj: objects}
+
     def destroy_node(self, node):
         uri = '/servers/%s' % (node.id)
         resp = self.connection.request(uri, method='DELETE')
@@ -2699,6 +2753,21 @@ class OpenStack_2_NodeDriver(OpenStack_1_1_NodeDriver):
             )
         )
 
+    def list_nodes(self, ex_all_tenants=False):
+        """
+        List the nodes in a tenant
+
+        :param ex_all_tenants: List nodes for all the tenants. Note: Your user
+                               must have admin privileges for this
+                               functionality to work.
+        :type ex_all_tenants: ``bool``
+        """
+        params = {}
+        if ex_all_tenants:
+            params = {'all_tenants': 1}
+        return self._to_nodes(self._paginated_request(
+            '/servers/detail', 'servers', self.connection, params=params))
+
     def get_image(self, image_id):
         """
         Get a NodeImage using the V2 Glance API
@@ -2955,10 +3024,9 @@ class OpenStack_2_NodeDriver(OpenStack_1_1_NodeDriver):
 
         :rtype: ``list`` of :class:`OpenStack_2_PortInterface`
         """
-        response = self.network_connection.request(
-            '/v2.0/ports'
-        )
-        return [self._to_port(port) for port in response.object['ports']]
+        response = self._paginated_request(
+            '/v2.0/ports', 'ports', self.network_connection)
+        return [self._to_port(port) for port in response['ports']]
 
     def ex_delete_port(self, port):
         """
@@ -3069,8 +3137,8 @@ class OpenStack_2_NodeDriver(OpenStack_1_1_NodeDriver):
 
         :rtype: ``list`` of :class:`StorageVolume`
         """
-        return self._to_volumes(
-            self.volumev2_connection.request('/volumes/detail').object)
+        return self._to_volumes(self._paginated_request(
+            '/volumes/detail', 'volumes', self.volumev2_connection))
 
     def ex_get_volume(self, volumeId):
         """
@@ -3152,8 +3220,8 @@ class OpenStack_2_NodeDriver(OpenStack_1_1_NodeDriver):
 
         :rtype: ``list`` of :class:`VolumeSnapshot`
         """
-        return self._to_snapshots(
-            self.volumev2_connection.request('/snapshots/detail').object)
+        return self._to_snapshots(self._paginated_request(
+            '/snapshots/detail', 'snapshots', self.volumev2_connection))
 
     def create_volume_snapshot(self, volume, name=None, ex_description=None,
                                ex_force=True):

http://git-wip-us.apache.org/repos/asf/libcloud/blob/1aaeff45/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__snapshots.json
----------------------------------------------------------------------
diff --git a/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__snapshots.json b/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__snapshots.json
index 763831c..1fad9da 100644
--- a/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__snapshots.json
+++ b/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__snapshots.json
@@ -43,4 +43,4 @@
             "description": "volume snapshot"
         }
     ]
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/libcloud/blob/1aaeff45/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__snapshots_paginate_start.json
----------------------------------------------------------------------
diff --git a/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__snapshots_paginate_start.json
b/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__snapshots_paginate_start.json
new file mode 100644
index 0000000..8603f67
--- /dev/null
+++ b/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__snapshots_paginate_start.json
@@ -0,0 +1,52 @@
+{
+    "snapshots_links": [
+      {
+        "href": "https://api.example.com:8776/v2/abcdec85bee34bb0a44ab8255eb36fbd/snapshots?&marker=abcde279-4cc0-4b47-a4ee-ec9c9e474b1b",
+        "rel": "next"
+      }
+    ],
+    "snapshots": [
+        {
+            "status": "available",
+            "metadata": {
+                "name": "test"
+            },
+            "os-extended-snapshot-attributes:progress": "100%",
+            "name": "snap-101",
+            "volume_id": "473f7b48-c4c1-4e70-9acc-086b39073506",
+            "os-extended-snapshot-attributes:project_id": "bab7d5c60cd041a0a36f7c4b6e1dd978",
+            "created_at": "2012-02-29T03:50:07Z",
+            "size": 1,
+            "id": "3fbbcccf-d058-4502-8844-6feeffdf4cb5",
+            "description": "volume snapshot"
+        },
+        {
+            "status": "available",
+            "metadata": {
+                "name": "test2"
+            },
+            "os-extended-snapshot-attributes:progress": "100%",
+            "name": "test-volume-snapshot2",
+            "volume_id": "7edbc2f4-1507-44f8-ac0d-eed1d2608d38",
+            "os-extended-snapshot-attributes:project_id": "bab7d5c60cd041a0a36f7c4b6e1dd978",
+            "created_at": "2015-11-29T02:25:51.000000",
+            "size": 1,
+            "id": "5fbbdccf-e058-6502-8844-6feeffdf4cb5",
+            "description": "volume snapshot"
+        },
+        {
+            "status": "available",
+            "metadata": {
+                "name": "test2"
+            },
+            "os-extended-snapshot-attributes:progress": "100%",
+            "name": "test-volume-snapshot",
+            "volume_id": "473f7b48-c4c1-4e70-9acc-086b39073506",
+            "os-extended-snapshot-attributes:project_id": "bab7d5c60cd041a0a36f7c4b6e1dd978",
+            "created_at": "2013-02-29T03:50:07Z",
+            "size": 1,
+            "id": "2fbbcccf-d058-4502-8844-6feeffdf4cb5",
+            "description": "volume snapshot"
+        }
+    ]
+}

http://git-wip-us.apache.org/repos/asf/libcloud/blob/1aaeff45/libcloud/test/compute/test_openstack.py
----------------------------------------------------------------------
diff --git a/libcloud/test/compute/test_openstack.py b/libcloud/test/compute/test_openstack.py
index e567f17..d626474 100644
--- a/libcloud/test/compute/test_openstack.py
+++ b/libcloud/test/compute/test_openstack.py
@@ -20,6 +20,7 @@ import sys
 import unittest
 import datetime
 import pytest
+
 from libcloud.utils.iso8601 import UTC
 
 try:
@@ -47,7 +48,8 @@ from libcloud.compute.drivers.openstack import (
     OpenStack_1_1_FloatingIpAddress, OpenStackKeyPair,
     OpenStack_1_0_Connection, OpenStack_2_FloatingIpPool,
     OpenStackNodeDriver,
-    OpenStack_2_NodeDriver, OpenStack_2_PortInterfaceState, OpenStackNetwork)
+    OpenStack_2_NodeDriver, OpenStack_2_PortInterfaceState, OpenStackNetwork,
+    OpenStackException)
 from libcloud.compute.base import Node, NodeImage, NodeSize
 from libcloud.pricing import set_pricing, clear_pricing_data
 
@@ -1628,6 +1630,32 @@ class OpenStack_2_Tests(OpenStack_1_1_Tests):
         # normally authentication happens lazily, but we force it here
         self.driver.volumev2_connection._populate_hosts_and_request_paths()
 
+    def test__paginated_request_single_page(self):
+        snapshots = self.driver._paginated_request(
+            '/snapshots/detail', 'snapshots',
+            self.driver.volumev2_connection
+        )['snapshots']
+
+        self.assertEqual(len(snapshots), 3)
+        self.assertEqual(snapshots[0]['name'], 'snap-001')
+
+    def test__paginated_request_two_pages(self):
+        snapshots = self.driver._paginated_request(
+            '/snapshots/detail?unit_test=paginate', 'snapshots',
+            self.driver.volumev2_connection
+        )['snapshots']
+
+        self.assertEqual(len(snapshots), 6)
+        self.assertEqual(snapshots[0]['name'], 'snap-101')
+        self.assertEqual(snapshots[3]['name'], 'snap-001')
+
+    def test__paginated_request_raises_if_stuck_in_a_loop(self):
+        with pytest.raises(OpenStackException):
+            self.driver._paginated_request(
+                '/snapshots/detail?unit_test=pagination_loop', 'snapshots',
+                self.driver.volumev2_connection
+            )
+
     def test_ex_force_auth_token_passed_to_connection(self):
         base_url = 'https://servers.api.rackspacecloud.com/v1.1/slug'
         kwargs = {
@@ -2485,7 +2513,11 @@ class OpenStack_1_1_MockHttp(MockHttp, unittest.TestCase):
             return (httplib.NO_CONTENT, body, self.json_content_headers, httplib.responses[httplib.OK])
     
     def _v2_1337_snapshots_detail(self, method, url, body, headers):
-        body = self.fixtures.load('_v2_0__snapshots.json')
+        if ('unit_test=paginate' in url and 'marker' not in url) or \
+                'unit_test=pagination_loop' in url:
+            body = self.fixtures.load('_v2_0__snapshots_paginate_start.json')
+        else:
+            body = self.fixtures.load('_v2_0__snapshots.json')
         return (httplib.OK, body, self.json_content_headers, httplib.responses[httplib.OK])
 
     def _v2_1337_snapshots(self, method, url, body, headers):


Mime
View raw message