libcloud-notifications mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From erjoh...@apache.org
Subject libcloud git commit: [google|compute] Add support for GCE list paging and filtering
Date Wed, 13 May 2015 13:53:51 GMT
Repository: libcloud
Updated Branches:
  refs/heads/trunk 582633929 -> 1ed502045


[google|compute] Add support for GCE list paging and filtering

GCE will return a maximum of 500 resources in a single list. This change
adds an iterator that allows filtering and/or paging of list results.

Closes #491

Signed-off-by: Eric Johnson <erjohnso@google.com>


Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo
Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/1ed50204
Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/1ed50204
Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/1ed50204

Branch: refs/heads/trunk
Commit: 1ed5020450e6e8deb0361239bfb91ba795447e87
Parents: 5826339
Author: Lee Verberne <verb@google.com>
Authored: Tue Jan 6 09:54:05 2015 -0800
Committer: Eric Johnson <erjohnso@google.com>
Committed: Wed May 13 13:52:46 2015 +0000

----------------------------------------------------------------------
 CHANGES.rst                                     |   4 +
 libcloud/compute/drivers/gce.py                 | 174 ++++++++++++++++++-
 .../compute/fixtures/gce/regions-paged-1.json   |  97 +++++++++++
 .../compute/fixtures/gce/regions-paged-2.json   |  52 ++++++
 libcloud/test/compute/fixtures/gce/zones.json   |  12 +-
 libcloud/test/compute/test_gce.py               |  63 ++++++-
 6 files changed, 393 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/libcloud/blob/1ed50204/CHANGES.rst
----------------------------------------------------------------------
diff --git a/CHANGES.rst b/CHANGES.rst
index 98ba7a4..5b7ef6f 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -25,6 +25,10 @@ General
 Compute
 ~~~~~~~
 
+- Google Compute now supports paginated lists including filtering.
+  (GITHUB-491)
+  [Lee Verberne]
+
 - OpenStackNodeSize objects now support optional, additional fields that are
   supported in OpenStack 2.1: `ephemeral_disk`, `swap`, `extra`.
   (GITHUB-488, LIBCLOUD-682)

http://git-wip-us.apache.org/repos/asf/libcloud/blob/1ed50204/libcloud/compute/drivers/gce.py
----------------------------------------------------------------------
diff --git a/libcloud/compute/drivers/gce.py b/libcloud/compute/drivers/gce.py
index 666f1c1..e1e7549 100644
--- a/libcloud/compute/drivers/gce.py
+++ b/libcloud/compute/drivers/gce.py
@@ -64,7 +64,28 @@ class GCEResponse(GoogleResponse):
 
 
 class GCEConnection(GoogleBaseConnection):
-    """Connection class for the GCE driver."""
+    """
+    Connection class for the GCE driver.
+
+    GCEConnection extends :class:`google.GoogleBaseConnection` for 2 reasons:
+      1. modify request_path for GCE URI.
+      2. Implement gce_params functionality described below.
+
+    If the parameter gce_params is set to a dict prior to calling request(),
+    the URL parameters will be updated to include those key/values FOR A
+    SINGLE REQUEST. If the response contains a nextPageToken,
+    gce_params['pageToken'] will be set to its value. This can be used to
+    implement paging in list:
+
+    >>> params, more_results = {'maxResults': 2}, True
+    >>> while more_results:
+    ...     driver.connection.gce_params=params
+    ...     driver.ex_list_urlmaps()
+    ...     more_results = 'pageToken' in params
+    ...
+    [<GCEUrlMap id="..." name="cli-map">, <GCEUrlMap id="..." name="lc-map">]
+    [<GCEUrlMap id="..." name="web-map">]
+    """
     host = 'www.googleapis.com'
     responseCls = GCEResponse
 
@@ -76,6 +97,138 @@ class GCEConnection(GoogleBaseConnection):
                                             **kwargs)
         self.request_path = '/compute/%s/projects/%s' % (API_VERSION,
                                                          project)
+        self.gce_params = None
+
+    def pre_connect_hook(self, params, headers):
+        """
+        Update URL parameters with values from self.gce_params.
+
+        @inherits: :class:`GoogleBaseConnection.pre_connect_hook`
+        """
+        params, headers = super(GCEConnection, self).pre_connect_hook(params,
+                                                                      headers)
+        if self.gce_params:
+            params.update(self.gce_params)
+        return params, headers
+
+    def request(self, *args, **kwargs):
+        """
+        Perform request then do GCE-specific processing of URL params.
+
+        @inherits: :class:`GoogleBaseConnection.request`
+        """
+        response = super(GCEConnection, self).request(*args, **kwargs)
+
+        # If gce_params has been set, then update the pageToken with the
+        # nextPageToken so it can be used in the next request.
+        if self.gce_params:
+            if 'nextPageToken' in response.object:
+                self.gce_params['pageToken'] = response.object['nextPageToken']
+            elif 'pageToken' in self.gce_params:
+                del self.gce_params['pageToken']
+            self.gce_params = None
+
+        return response
+
+
+class GCEList(object):
+    """
+    An Iterator that wraps list functions to provide additional features.
+
+    GCE enforces a limit on the number of objects returned by a list operation,
+    so users with more than 500 objects of a particular type will need to use
+    filter(), page() or both.
+
+    >>> l=GCEList(driver, driver.ex_list_urlmaps)
+    >>> for sublist in l.filter('name eq ...-map').page(1):
+    ...   sublist
+    ...
+    [<GCEUrlMap id="..." name="cli-map">]
+    [<GCEUrlMap id="..." name="web-map">]
+
+    One can create a GCEList manually, but it's slightly easier to use the
+    ex_list() method of :class:`GCENodeDriver`.
+    """
+
+    def __init__(self, driver, list_fn, **kwargs):
+        """
+        :param  driver: An initialized :class:``GCENodeDriver``
+        :type   driver: :class:``GCENodeDriver``
+
+        :param  list_fn: A bound list method from :class:`GCENodeDriver`.
+        :type   list_fn: ``instancemethod``
+        """
+        self.driver = driver
+        self.list_fn = list_fn
+        self.kwargs = kwargs
+        self.params = {}
+
+    def __iter__(self):
+        list_fn = self.list_fn
+        more_results = True
+        while more_results:
+            self.driver.connection.gce_params = self.params
+            yield list_fn(**self.kwargs)
+            more_results = 'pageToken' in self.params
+
+    def __repr__(self):
+        return '<GCEList list="%s" params="%s">' % (
+            self.list_fn.__name__, repr(self.params))
+
+    def filter(self, expression):
+        """
+        Filter results of a list operation.
+
+        GCE supports server-side filtering of resources returned by a list
+        operation. Syntax of the filter expression is fully descripted in the
+        GCE API reference doc, but in brief it is::
+
+            FIELD_NAME COMPARISON_STRING LITERAL_STRING
+
+        where FIELD_NAME is the resource's property name, COMPARISON_STRING is
+        'eq' or 'ne', and LITERAL_STRING is a regular expression in RE2 syntax.
+
+        >>> for sublist in l.filter('name eq ...-map'):
+        ...   sublist
+        ...
+        [<GCEUrlMap id="..." name="cli-map">, \
+                <GCEUrlMap id="..." name="web-map">]
+
+        API reference: https://cloud.google.com/compute/docs/reference/latest/
+        RE2 syntax: https://github.com/google/re2/blob/master/doc/syntax.txt
+
+        :param  expression: Filter expression described above.
+        :type   expression: ``str``
+
+        :return: This :class:`GCEList` instance
+        :rtype:  :class:`GCEList`
+        """
+        self.params['filter'] = expression
+        return self
+
+    def page(self, max_results=500):
+        """
+        Limit the number of results by each iteration.
+
+        This implements the paging functionality of the GCE list methods and
+        returns this GCEList instance so that results can be chained:
+
+        >>> for sublist in GCEList(driver, driver.ex_list_urlmaps).page(2):
+        ...   sublist
+        ...
+        [<GCEUrlMap id="..." name="cli-map">, \
+                <GCEUrlMap id="..." name="lc-map">]
+        [<GCEUrlMap id="..." name="web-map">]
+
+        :keyword  max_results: Maximum number of results to return per
+                               iteration. Defaults to the GCE default of 500.
+        :type     max_results: ``int``
+
+        :return: This :class:`GCEList` instance
+        :rtype:  :class:`GCEList`
+        """
+        self.params['maxResults'] = max_results
+        return self
 
 
 class GCELicense(UuidMixin):
@@ -1032,6 +1185,25 @@ class GCENodeDriver(NodeDriver):
         response = self.connection.request(request, method='GET').object
         return response['contents']
 
+    def ex_list(self, list_fn, **kwargs):
+        """
+        Wrap a list method in a :class:`GCEList` iterator.
+
+        >>> for sublist in driver.ex_list(driver.ex_list_urlmaps).page(1):
+        ...   sublist
+        ...
+        [<GCEUrlMap id="..." name="cli-map">]
+        [<GCEUrlMap id="..." name="lc-map">]
+        [<GCEUrlMap id="..." name="web-map">]
+
+        :param  list_fn: A bound list method from :class:`GCENodeDriver`.
+        :type   list_fn: ``instancemethod``
+
+        :return: An iterator that returns sublists from list_fn.
+        :rtype: :class:`GCEList`
+        """
+        return GCEList(driver=self, list_fn=list_fn, **kwargs)
+
     def ex_list_disktypes(self, zone=None):
         """
         Return a list of DiskTypes for a zone or all.

http://git-wip-us.apache.org/repos/asf/libcloud/blob/1ed50204/libcloud/test/compute/fixtures/gce/regions-paged-1.json
----------------------------------------------------------------------
diff --git a/libcloud/test/compute/fixtures/gce/regions-paged-1.json b/libcloud/test/compute/fixtures/gce/regions-paged-1.json
new file mode 100644
index 0000000..790f31b
--- /dev/null
+++ b/libcloud/test/compute/fixtures/gce/regions-paged-1.json
@@ -0,0 +1,97 @@
+{
+ "kind": "compute#regionList",
+ "selfLink": "https://www.googleapis.com/compute/v1/projects/project_name/regions",
+ "id": "projects/project_name/regions",
+ "items": [
+  {
+   "kind": "compute#region",
+   "selfLink": "https://www.googleapis.com/compute/v1/projects/project_name/regions/asia-east1",
+   "id": "1220",
+   "creationTimestamp": "2014-04-11T13:47:12.495-07:00",
+   "name": "asia-east1",
+   "description": "asia-east1",
+   "status": "UP",
+   "zones": [
+    "https://www.googleapis.com/compute/v1/projects/project_name/zones/asia-east1-a"
+   ],
+   "quotas": [
+    {
+     "metric": "CPUS",
+     "limit": 24.0,
+     "usage": 0.0
+    },
+    {
+     "metric": "DISKS_TOTAL_GB",
+     "limit": 5120.0,
+     "usage": 0.0
+    },
+    {
+     "metric": "STATIC_ADDRESSES",
+     "limit": 7.0,
+     "usage": 0.0
+    },
+    {
+     "metric": "IN_USE_ADDRESSES",
+     "limit": 23.0,
+     "usage": 0.0
+    },
+    {
+     "metric": "SSD_TOTAL_GB",
+     "limit": 1024.0,
+     "usage": 0.0
+    },
+    {
+     "metric": "LOCAL_SSD_TOTAL_GB",
+     "limit": 1500.0,
+     "usage": 0.0
+    }
+   ]
+  },
+  {
+   "kind": "compute#region",
+   "selfLink": "https://www.googleapis.com/compute/v1/projects/project_name/regions/europe-west1",
+   "id": "1100",
+   "creationTimestamp": "2014-04-11T13:47:12.495-07:00",
+   "name": "europe-west1",
+   "description": "europe-west1",
+   "status": "UP",
+   "zones": [
+    "https://www.googleapis.com/compute/v1/projects/project_name/zones/europe-west1-a",
+    "https://www.googleapis.com/compute/v1/projects/project_name/zones/europe-west1-b"
+   ],
+   "quotas": [
+    {
+     "metric": "CPUS",
+     "limit": 24.0,
+     "usage": 0.0
+    },
+    {
+     "metric": "DISKS_TOTAL_GB",
+     "limit": 5120.0,
+     "usage": 0.0
+    },
+    {
+     "metric": "STATIC_ADDRESSES",
+     "limit": 7.0,
+     "usage": 0.0
+    },
+    {
+     "metric": "IN_USE_ADDRESSES",
+     "limit": 23.0,
+     "usage": 0.0
+    },
+    {
+     "metric": "SSD_TOTAL_GB",
+     "limit": 1024.0,
+     "usage": 0.0
+    },
+    {
+     "metric": "LOCAL_SSD_TOTAL_GB",
+     "limit": 1500.0,
+     "usage": 0.0
+    }
+   ]
+  }
+ ],
+ "nextPageToken": "CjQIz5W-w6HRxAI6KQoCGAEKAiAACgIYAQoCIAAKAhgTCg4qDGV1cm9wZS13ZXN0MQoDIMwI"
+}

http://git-wip-us.apache.org/repos/asf/libcloud/blob/1ed50204/libcloud/test/compute/fixtures/gce/regions-paged-2.json
----------------------------------------------------------------------
diff --git a/libcloud/test/compute/fixtures/gce/regions-paged-2.json b/libcloud/test/compute/fixtures/gce/regions-paged-2.json
new file mode 100644
index 0000000..d99ea6d
--- /dev/null
+++ b/libcloud/test/compute/fixtures/gce/regions-paged-2.json
@@ -0,0 +1,52 @@
+{
+ "kind": "compute#regionList",
+ "selfLink": "https://www.googleapis.com/compute/v1/projects/project_name/regions",
+ "id": "projects/project_name/regions",
+ "items": [
+  {
+   "kind": "compute#region",
+   "selfLink": "https://www.googleapis.com/compute/v1/projects/project_name/regions/us-central1",
+   "id": "1000",
+   "creationTimestamp": "2014-04-11T13:47:12.495-07:00",
+   "name": "us-central1",
+   "description": "us-central1",
+   "status": "UP",
+   "zones": [
+    "https://www.googleapis.com/compute/v1/projects/project_name/zones/us-central1-a",
+    "https://www.googleapis.com/compute/v1/projects/project_name/zones/us-central1-b"
+   ],
+   "quotas": [
+    {
+     "metric": "CPUS",
+     "limit": 24.0,
+     "usage": 1.0
+    },
+    {
+     "metric": "DISKS_TOTAL_GB",
+     "limit": 5120.0,
+     "usage": 60.0
+    },
+    {
+     "metric": "STATIC_ADDRESSES",
+     "limit": 7.0,
+     "usage": 1.0
+    },
+    {
+     "metric": "IN_USE_ADDRESSES",
+     "limit": 23.0,
+     "usage": 1.0
+    },
+    {
+     "metric": "SSD_TOTAL_GB",
+     "limit": 1024.0,
+     "usage": 0.0
+    },
+    {
+     "metric": "LOCAL_SSD_TOTAL_GB",
+     "limit": 1500.0,
+     "usage": 0.0
+    }
+   ]
+  }
+ ]
+}

http://git-wip-us.apache.org/repos/asf/libcloud/blob/1ed50204/libcloud/test/compute/fixtures/gce/zones.json
----------------------------------------------------------------------
diff --git a/libcloud/test/compute/fixtures/gce/zones.json b/libcloud/test/compute/fixtures/gce/zones.json
index 0d564e7..1ecf5ac 100644
--- a/libcloud/test/compute/fixtures/gce/zones.json
+++ b/libcloud/test/compute/fixtures/gce/zones.json
@@ -2,6 +2,16 @@
   "id": "projects/project_name/zones",
   "items": [
     {
+     "kind": "compute#zone",
+     "selfLink": "https://www.googleapis.com/compute/v1/projects/verb-test/zones/asia-east1-a",
+     "id": "2220",
+     "creationTimestamp": "2014-05-30T18:35:16.575-07:00",
+     "name": "asia-east1-a",
+     "description": "asia-east1-a",
+     "status": "UP",
+     "region": "https://www.googleapis.com/compute/v1/projects/verb-test/regions/asia-east1"
+    },
+    {
       "creationTimestamp": "2013-02-05T16:19:23.254-08:00",
       "description": "europe-west1-a",
       "id": "13416642339679437530",
@@ -82,4 +92,4 @@
   ],
   "kind": "compute#zoneList",
   "selfLink": "https://www.googleapis.com/compute/v1/projects/project_name/zones"
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/libcloud/blob/1ed50204/libcloud/test/compute/test_gce.py
----------------------------------------------------------------------
diff --git a/libcloud/test/compute/test_gce.py b/libcloud/test/compute/test_gce.py
index 6759656..09466c9 100644
--- a/libcloud/test/compute/test_gce.py
+++ b/libcloud/test/compute/test_gce.py
@@ -121,6 +121,45 @@ class GCENodeDriverTest(LibcloudTestCase, TestCaseMixin):
         self.assertTrue(self.driver.ex_get_serial_output(node),
                         'This is some serial\r\noutput for you.')
 
+    def test_ex_list(self):
+        d = self.driver
+        # Test the default case for all list methods
+        # (except list_volume_snapshots, which requires an arg)
+        for list_fn in (d.ex_list_addresses,
+                        d.ex_list_backendservices,
+                        d.ex_list_disktypes,
+                        d.ex_list_firewalls,
+                        d.ex_list_forwarding_rules,
+                        d.ex_list_healthchecks,
+                        d.ex_list_networks,
+                        d.ex_list_project_images,
+                        d.ex_list_regions,
+                        d.ex_list_routes,
+                        d.ex_list_snapshots,
+                        d.ex_list_targethttpproxies,
+                        d.ex_list_targetinstances,
+                        d.ex_list_targetpools,
+                        d.ex_list_urlmaps,
+                        d.ex_list_zones,
+                        d.list_images,
+                        d.list_locations,
+                        d.list_nodes,
+                        d.list_sizes,
+                        d.list_volumes):
+            full_list = [item.name for item in list_fn()]
+            li = d.ex_list(list_fn)
+            iter_list = [item.name for sublist in li for item in sublist]
+            self.assertEqual(full_list, iter_list)
+
+        # Test paging & filtering with a single list function as they require
+        # additional test fixtures
+        list_fn = d.ex_list_regions
+        for count, sublist in zip((2, 1), d.ex_list(list_fn).page(2)):
+            self.assertTrue(len(sublist) == count)
+        for sublist in d.ex_list(list_fn).filter('name eq us-central1'):
+            self.assertTrue(len(sublist) == 1)
+            self.assertEqual(sublist[0].name, 'us-central1')
+
     def test_ex_list_addresses(self):
         address_list = self.driver.ex_list_addresses()
         address_list_all = self.driver.ex_list_addresses('all')
@@ -190,8 +229,8 @@ class GCENodeDriverTest(LibcloudTestCase, TestCaseMixin):
 
     def test_list_locations(self):
         locations = self.driver.list_locations()
-        self.assertEqual(len(locations), 5)
-        self.assertEqual(locations[0].name, 'europe-west1-a')
+        self.assertEqual(len(locations), 6)
+        self.assertEqual(locations[0].name, 'asia-east1-a')
 
     def test_ex_list_routes(self):
         routes = self.driver.ex_list_routes()
@@ -306,8 +345,8 @@ class GCENodeDriverTest(LibcloudTestCase, TestCaseMixin):
 
     def test_ex_list_zones(self):
         zones = self.driver.ex_list_zones()
-        self.assertEqual(len(zones), 5)
-        self.assertEqual(zones[0].name, 'europe-west1-a')
+        self.assertEqual(len(zones), 6)
+        self.assertEqual(zones[0].name, 'asia-east1-a')
 
     def test_ex_create_address_global(self):
         address_name = 'lcaddressglobal'
@@ -1398,8 +1437,10 @@ class GCEMockHttp(MockHttpTestCase):
         if method == 'POST':
             body = self.fixtures.load('global_backendServices_post.json')
         else:
+            backend_name = getattr(self.test, 'backendservices_mock',
+                                   'web-service')
             body = self.fixtures.load('global_backendServices-%s.json' %
-                                      self.test.backendservices_mock)
+                                      backend_name)
         return (httplib.OK, body, self.json_hdr, httplib.responses[httplib.OK])
 
     def _global_backendServices_no_backends(self, method, url, body, headers):
@@ -1987,8 +2028,12 @@ class GCEMockHttp(MockHttpTestCase):
         return (httplib.OK, body, self.json_hdr, httplib.responses[httplib.OK])
 
     def _regions(self, method, url, body, headers):
-        body = self.fixtures.load(
-            'regions.json')
+        if 'pageToken' in url or 'filter' in url:
+            body = self.fixtures.load('regions-paged-2.json')
+        elif 'maxResults' in url:
+            body = self.fixtures.load('regions-paged-1.json')
+        else:
+            body = self.fixtures.load('regions.json')
         return (httplib.OK, body, self.json_hdr, httplib.responses[httplib.OK])
 
     def _global_addresses(self, method, url, body, headers):
@@ -2147,6 +2192,10 @@ class GCEMockHttp(MockHttpTestCase):
         body = self.fixtures.load('zones.json')
         return (httplib.OK, body, self.json_hdr, httplib.responses[httplib.OK])
 
+    def _zones_asia_east_1a(self, method, url, body, headers):
+        body = self.fixtures.load('zones_asia-east1-a.json')
+        return (httplib.OK, body, self.json_hdr, httplib.responses[httplib.OK])
+
     def _zones_us_central1_a_diskTypes(self, method, url, body, headers):
         body = self.fixtures.load('zones_us-central1-a_diskTypes.json')
         return (httplib.OK, body, self.json_hdr, httplib.responses[httplib.OK])


Mime
View raw message