Add cinder support LIBCLOUD-874 Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/e9dcb93e Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/e9dcb93e Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/e9dcb93e Branch: refs/heads/trunk Commit: e9dcb93e4f8591e0980399563e53424215c6fd86 Parents: cd2faa7 Author: micafer Authored: Mon Sep 24 12:54:47 2018 +0200 Committer: Rick van de Loo Committed: Tue Dec 4 09:45:48 2018 +0100 ---------------------------------------------------------------------- libcloud/compute/drivers/openstack.py | 167 +++++++++++++++++-- .../openstack_v1.1/_v2_0__snapshot.json | 14 ++ .../openstack_v1.1/_v2_0__snapshots.json | 46 +++++ .../fixtures/openstack_v1.1/_v2_0__volume.json | 18 ++ .../fixtures/openstack_v1.1/_v2_0__volumes.json | 44 +++++ libcloud/test/compute/test_openstack.py | 78 +++++++++ 6 files changed, 357 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/libcloud/blob/e9dcb93e/libcloud/compute/drivers/openstack.py ---------------------------------------------------------------------- diff --git a/libcloud/compute/drivers/openstack.py b/libcloud/compute/drivers/openstack.py index 75106f1..bfdf082 100644 --- a/libcloud/compute/drivers/openstack.py +++ b/libcloud/compute/drivers/openstack.py @@ -89,6 +89,12 @@ class OpenStackNetworkConnection(OpenStackBaseConnection): service_region = 'RegionOne' +class OpenStackVolumeConnection(OpenStackBaseConnection): + service_type = 'volume' + service_name = 'cinder' + service_region = 'RegionOne' + + class OpenStackNodeDriver(NodeDriver, OpenStackDriverMixin): """ Base OpenStack node driver. Should not be used directly. @@ -2200,20 +2206,27 @@ class OpenStack_1_1_NodeDriver(OpenStackNodeDriver): return StorageVolume( id=api_node['id'], - name=api_node['displayName'], + name=api_node.get('name', api_node.get('displayName', None)), size=api_node['size'], state=state, driver=self, extra={ - 'description': api_node['displayDescription'], + 'description': api_node.get('description', + api_node.get('displayDescription', + None)), 'attachments': [att for att in api_node['attachments'] if att], # TODO: remove in 1.18.0 'state': api_node.get('status', None), - 'snapshot_id': api_node.get('snapshotId', None), - 'location': api_node.get('availabilityZone', None), - 'volume_type': api_node.get('volumeType', None), + 'snapshot_id': api_node.get('snapshot_id', + api_node.get('snapshotId', None)), + 'location': api_node.get('availability_zone', + api_node.get('availabilityZone', + None)), + 'volume_type': api_node.get('volume_type', + api_node.get('volumeType', None)), 'metadata': api_node.get('metadata', None), - 'created_at': api_node.get('createdAt', None) + 'created_at': api_node.get('created_at', + api_node.get('createdAt', None)) } ) @@ -2222,10 +2235,13 @@ class OpenStack_1_1_NodeDriver(OpenStackNodeDriver): data = data['snapshot'] volume_id = data.get('volume_id', data.get('volumeId', None)) - display_name = data.get('display_name', data.get('displayName', None)) + display_name = data.get('name', + data.get('display_name', + data.get('displayName', None))) created_at = data.get('created_at', data.get('createdAt', None)) - description = data.get('display_description', - data.get('displayDescription', None)) + description = data.get('description', + data.get('display_description', + data.get('displayDescription', None))) status = data.get('status', None) extra = {'volume_id': volume_id, @@ -2514,6 +2530,15 @@ class OpenStack_2_NetworkConnection(OpenStackNetworkConnection): return json.dumps(data) +class OpenStack_2_VolumeConnection(OpenStackVolumeConnection): + responseCls = OpenStack_1_1_Response + accept_format = 'application/json' + default_content_type = 'application/json; charset=UTF-8' + + def encode_data(self, data): + return json.dumps(data) + + class OpenStack_2_PortInterfaceState(Type): """ Standard states of OpenStack_2_PortInterfaceState @@ -2562,6 +2587,14 @@ class OpenStack_2_NodeDriver(OpenStack_1_1_NodeDriver): # accessed from there. network_connectionCls = OpenStack_2_NetworkConnection network_connection = None + + # Similarly not all node-related operations are exposed through the + # compute API + # See https://developer.openstack.org/api-ref/compute/ + # For example, volume management are made in the cinder service + volume_connectionCls = OpenStack_2_VolumeConnection + volume_connection = None + type = Provider.OPENSTACK features = {"create_node": ["generates_password"]} @@ -2594,8 +2627,18 @@ class OpenStack_2_NodeDriver(OpenStack_1_1_NodeDriver): super(OpenStack_2_NodeDriver, self).__init__(*args, **kwargs) self.image_connection = self.connection + # We run the init once to get the Cinder V2 API connection + # and put that on the object under self.volume_connection. + if original_ex_force_base_url or kwargs.get('ex_force_volume_url'): + kwargs['ex_force_base_url'] = \ + str(kwargs.pop('ex_force_volume_url', + original_ex_force_base_url)) + self.connectionCls = self.volume_connectionCls + super(OpenStack_2_NodeDriver, self).__init__(*args, **kwargs) + self.volume_connection = self.connection + # We run the init once to get the Neutron V2 API connection - # and put that on the object under self.image_connection. + # and put that on the object under self.network_connection. if original_ex_force_base_url or kwargs.get('ex_force_network_url'): kwargs['ex_force_base_url'] = \ str(kwargs.pop('ex_force_network_url', @@ -2970,6 +3013,110 @@ class OpenStack_2_NodeDriver(OpenStack_1_1_NodeDriver): ) return self._to_port(response.object['port']) + def list_volumes(self): + return self._to_volumes( + self.connection.request('/volumes/detail').object) + + def ex_get_volume(self, volumeId): + return self._to_volume( + self.connection.request('/volumes/%s' % volumeId).object) + + def create_volume(self, size, name, location=None, snapshot=None, + ex_volume_type=None): + """ + Create a new volume. + + :param size: Size of volume in gigabytes (required) + :type size: ``int`` + + :param name: Name of the volume to be created + :type name: ``str`` + + :param location: Which data center to create a volume in. If + empty, undefined behavior will be selected. + (optional) + :type location: :class:`.NodeLocation` + + :param snapshot: Snapshot from which to create the new + volume. (optional) + :type snapshot: :class:`.VolumeSnapshot` + + :param ex_volume_type: What kind of volume to create. + (optional) + :type ex_volume_type: ``str`` + + :return: The newly created volume. + :rtype: :class:`StorageVolume` + """ + volume = { + 'name': name, + 'description': name, + 'size': size, + 'metadata': { + 'contents': name, + }, + } + + if ex_volume_type: + volume['volume_type'] = ex_volume_type + + if location: + volume['availability_zone'] = location + + if snapshot: + volume['snapshot_id'] = snapshot.id + + resp = self.connection.request('/volumes', + method='POST', + data={'volume': volume}) + return self._to_volume(resp.object) + + def destroy_volume(self, volume): + return self.connection.request('/volumes/%s' % volume.id, + method='DELETE').success() + + def ex_list_snapshots(self): + return self._to_snapshots( + self.connection.request('/snapshots/detail').object) + + def create_volume_snapshot(self, volume, name=None, ex_description=None, + ex_force=True): + """ + Create snapshot from volume + + :param volume: Instance of `StorageVolume` + :type volume: `StorageVolume` + + :param name: Name of snapshot (optional) + :type name: `str` | `NoneType` + + :param ex_description: Description of the snapshot (optional) + :type ex_description: `str` | `NoneType` + + :param ex_force: Specifies if we create a snapshot that is not in + state `available`. For example `in-use`. Defaults + to True. (optional) + :type ex_force: `bool` + + :rtype: :class:`VolumeSnapshot` + """ + data = {'snapshot': {'volume_id': volume.id, 'force': ex_force}} + + if name is not None: + data['snapshot']['name'] = name + + if ex_description is not None: + data['snapshot']['description'] = ex_description + + return self._to_snapshot(self.connection.request('/snapshots', + method='POST', + data=data).object) + + def destroy_volume_snapshot(self, snapshot): + resp = self.connection.request('/snapshots/%s' % snapshot.id, + method='DELETE') + return resp.status == httplib.ACCEPTED + class OpenStack_1_1_FloatingIpPool(object): """ http://git-wip-us.apache.org/repos/asf/libcloud/blob/e9dcb93e/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__snapshot.json ---------------------------------------------------------------------- diff --git a/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__snapshot.json b/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__snapshot.json new file mode 100644 index 0000000..3b4e846 --- /dev/null +++ b/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__snapshot.json @@ -0,0 +1,14 @@ +{ + "snapshot": { + "status": "available", + "os-extended-snapshot-attributes:progress": "100%", + "description": "Daily backup", + "created_at": "2013-02-25T04:13:17.07Z", + "metadata": {}, + "volume_id": "5aa119a8-d25b-45a7-8d1b-88e127885635", + "os-extended-snapshot-attributes:project_id": "0c2eba2c5af04d3f9e9d0d410b371fde", + "size": 1, + "id": "3fbbcccf-d058-4502-8844-6feeffdf4cb5", + "name": "test" + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/libcloud/blob/e9dcb93e/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__snapshots.json ---------------------------------------------------------------------- diff --git a/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__snapshots.json b/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__snapshots.json new file mode 100644 index 0000000..763831c --- /dev/null +++ b/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__snapshots.json @@ -0,0 +1,46 @@ +{ + "snapshots": [ + { + "status": "available", + "metadata": { + "name": "test" + }, + "os-extended-snapshot-attributes:progress": "100%", + "name": "snap-001", + "volume_id": "373f7b48-c4c1-4e70-9acc-086b39073506", + "os-extended-snapshot-attributes:project_id": "bab7d5c60cd041a0a36f7c4b6e1dd978", + "created_at": "2012-02-29T03:50:07Z", + "size": 1, + "id": "3fbbcccf-d058-4502-8844-6feeffdf4cb5", + "description": "volume snapshot" + }, + { + "status": "available", + "metadata": { + "name": "test" + }, + "os-extended-snapshot-attributes:progress": "100%", + "name": "test-volume-snapshot", + "volume_id": "6edbc2f4-1507-44f8-ac0d-eed1d2608d38", + "os-extended-snapshot-attributes:project_id": "bab7d5c60cd041a0a36f7c4b6e1dd978", + "created_at": "2015-11-29T02:25:51.000000", + "size": 1, + "id": "4fbbdccf-e058-6502-8844-6feeffdf4cb5", + "description": "volume snapshot" + }, + { + "status": "available", + "metadata": { + "name": "test" + }, + "os-extended-snapshot-attributes:progress": "100%", + "name": "test-volume-snapshot", + "volume_id": "373f7b48-c4c1-4e70-9acc-086b39073506", + "os-extended-snapshot-attributes:project_id": "bab7d5c60cd041a0a36f7c4b6e1dd978", + "created_at": "2013-02-29T03:50:07Z", + "size": 1, + "id": "1fbbcccf-d058-4502-8844-6feeffdf4cb5", + "description": "volume snapshot" + } + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/libcloud/blob/e9dcb93e/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__volume.json ---------------------------------------------------------------------- diff --git a/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__volume.json b/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__volume.json new file mode 100644 index 0000000..46587e6 --- /dev/null +++ b/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__volume.json @@ -0,0 +1,18 @@ +{ + "volume": { + "status": "available", + "attachments": [], + "availability_zone": "nova", + "bootable": "false", + "os-vol-host-attr:host": "ip-10-168-107-25", + "source_volid": null, + "snapshot_id": null, + "id": "cd76a3a1-c4ce-40f6-9b9f-07a61508938d", + "description": "Super volume.", + "name": "test", + "created_at": "2013-02-25T02:40:21.000000", + "volume_type": "None", + "size": 1, + "metadata": {} + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/libcloud/blob/e9dcb93e/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__volumes.json ---------------------------------------------------------------------- diff --git a/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__volumes.json b/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__volumes.json new file mode 100644 index 0000000..8415a18 --- /dev/null +++ b/libcloud/test/compute/fixtures/openstack_v1.1/_v2_0__volumes.json @@ -0,0 +1,44 @@ +{ + "volumes": [ + { + "status": "in-use", + "attachments": [ + { + "server_id": "f4fda93b-06e0-4743-8117-bc8bcecd651b", + "attachment_id": "3b4db356-253d-4fab-bfa0-e3626c0b8405", + "volume_id": "6edbc2f4-1507-44f8-ac0d-eed1d2608d38", + "device": "/dev/vdb", + "id": "6edbc2f4-1507-44f8-ac0d-eed1d2608d38" + } + ], + "availability_zone": "nova", + "replication_status": "disabled", + "snapshot_id": null, + "id": "6edbc2f4-1507-44f8-ac0d-eed1d2608d38", + "size": 2, + "user_id": "32779452fcd34ae1a53a797ac8a1e064", + "metadata": {}, + "description": "", + "name": "test-volume-attachments", + "created_at": "2013-06-24T11:20:13.000000", + "volume_type": "lvmdriver-1" + }, + { + "status": "some-unknown-state", + "migration_status": null, + "attachments": [], + "availability_zone": "nova", + "replication_status": "disabled", + "snapshot_id": "01f48111-7866-4cd2-986a-e92683c4a363", + "id": "cfcec3bc-b736-4db5-9535-4c24112691b5", + "size": 50, + "user_id": "32779452fcd34ae1a53a797ac8a1e064", + "metadata": {}, + "description": "some description", + "name": "test_volume", + "bootable": "false", + "created_at": "2013-06-21T12:39:02.000000", + "volume_type": null + } + ] +} http://git-wip-us.apache.org/repos/asf/libcloud/blob/e9dcb93e/libcloud/test/compute/test_openstack.py ---------------------------------------------------------------------- diff --git a/libcloud/test/compute/test_openstack.py b/libcloud/test/compute/test_openstack.py index 78c13e0..faf8fe4 100644 --- a/libcloud/test/compute/test_openstack.py +++ b/libcloud/test/compute/test_openstack.py @@ -1818,6 +1818,49 @@ class OpenStack_2_Tests(OpenStack_1_1_Tests): self.assertTrue(ret) + def test_list_volumes(self): + volumes = self.driver.list_volumes() + self.assertEqual(len(volumes), 2) + volume = volumes[0] + + self.assertEqual('6edbc2f4-1507-44f8-ac0d-eed1d2608d38', volume.id) + self.assertEqual('test-volume-attachments', volume.name) + self.assertEqual(StorageVolumeState.INUSE, volume.state) + self.assertEqual(2, volume.size) + self.assertEqual(volume.extra, { + 'description': '', + 'attachments': [{ + "attachment_id": "3b4db356-253d-4fab-bfa0-e3626c0b8405", + "id": '6edbc2f4-1507-44f8-ac0d-eed1d2608d38', + "device": "/dev/vdb", + "server_id": "f4fda93b-06e0-4743-8117-bc8bcecd651b", + "volume_id": "6edbc2f4-1507-44f8-ac0d-eed1d2608d38", + }], + 'snapshot_id': None, + 'state': 'in-use', + 'location': 'nova', + 'volume_type': 'lvmdriver-1', + 'metadata': {}, + 'created_at': '2013-06-24T11:20:13.000000' + }) + + # also test that unknown state resolves to StorageVolumeState.UNKNOWN + volume = volumes[1] + self.assertEqual('cfcec3bc-b736-4db5-9535-4c24112691b5', volume.id) + self.assertEqual('test_volume', volume.name) + self.assertEqual(50, volume.size) + self.assertEqual(StorageVolumeState.UNKNOWN, volume.state) + self.assertEqual(volume.extra, { + 'description': 'some description', + 'attachments': [], + 'snapshot_id': '01f48111-7866-4cd2-986a-e92683c4a363', + 'state': 'some-unknown-state', + 'location': 'nova', + 'volume_type': None, + 'metadata': {}, + 'created_at': '2013-06-21T12:39:02.000000', + }) + class OpenStack_1_1_FactoryMethodTests(OpenStack_1_1_Tests): should_list_locations = False @@ -2308,6 +2351,41 @@ class OpenStack_1_1_MockHttp(MockHttp, unittest.TestCase): body = self.fixtures.load('_v2_0__subnets.json') return (httplib.OK, body, self.json_content_headers, httplib.responses[httplib.OK]) + def _v2_1337_volumes_detail(self, method, url, body, headers): + body = self.fixtures.load('_v2_0__volumes.json') + return (httplib.OK, body, self.json_content_headers, httplib.responses[httplib.OK]) + + def _v2_1337_volumes(self, method, url, body, headers): + if method == 'POST': + body = self.fixtures.load('_v2_0__volume.json') + return (httplib.OK, body, self.json_content_headers, httplib.responses[httplib.OK]) + + def _v2_1337_volumes_cd76a3a1_c4ce_40f6_9b9f_07a61508938d(self, method, url, body, headers): + if method == 'GET': + body = self.fixtures.load('_v2_0__volume.json') + return (httplib.OK, body, self.json_content_headers, httplib.responses[httplib.OK]) + if method == 'DELETE': + body = '' + return (httplib.ACCEPTED, body, self.json_content_headers, httplib.responses[httplib.OK]) + + def _v2_1337_snapshots_detail(self, method, url, body, headers): + body = self.fixtures.load('_v2_0__snapshots.json') + return (httplib.OK, body, self.json_content_headers, httplib.responses[httplib.OK]) + + def _v2_1337_snapshots(self, method, url, body, headers): + if method == 'POST': + body = self.fixtures.load('_v2_0__snapshot.json') + return (httplib.OK, body, self.json_content_headers, httplib.responses[httplib.OK]) + + def _v2_1337_snapshots_3fbbcccf_d058_4502_8844_6feeffdf4cb5(self, method, url, body, headers): + if method == 'GET': + body = self.fixtures.load('_v2_0__snapshot.json') + return (httplib.OK, body, self.json_content_headers, httplib.responses[httplib.OK]) + if method == 'DELETE': + body = '' + return (httplib.ACCEPTED, body, self.json_content_headers, httplib.responses[httplib.OK]) + + # This exists because the nova compute url in devstack has v2 in there but the v1.1 fixtures # work fine.