libcloud-notifications mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From GitBox <...@apache.org>
Subject [GitHub] [libcloud] Kami commented on a change in pull request #1395: Add LXD driver & tests
Date Tue, 24 Dec 2019 19:16:25 GMT
Kami commented on a change in pull request #1395: Add LXD driver & tests
URL: https://github.com/apache/libcloud/pull/1395#discussion_r361215492
 
 

 ##########
 File path: libcloud/container/drivers/lxd.py
 ##########
 @@ -0,0 +1,1269 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import base64
+import re
+import os
+
+
+try:
+    import simplejson as json
+except Exception:
+    import json
+
+from libcloud.utils.py3 import httplib
+from libcloud.utils.py3 import b
+
+from libcloud.common.base import JsonResponse, ConnectionUserAndKey
+from libcloud.common.base import KeyCertificateConnection
+from libcloud.common.types import InvalidCredsError
+
+from libcloud.container.base import (Container, ContainerDriver,
+                                     ContainerImage)
+from libcloud.common.exceptions import BaseHTTPError
+
+from libcloud.compute.base import StorageVolume
+
+from libcloud.container.providers import Provider
+from libcloud.container.types import ContainerState
+
+# Acceptable success strings comping from LXD API
+LXD_API_SUCCESS_STATUS = ['Success']
+LXD_API_STATE_ACTIONS = ['stop', 'start', 'restart', 'freeze', 'unfreeze']
+LXD_API_IMAGE_SOURCE_TYPE = ["image", "migration", "copy", "none"]
+
+# the wording used by LXD to indicate that an error
+# occurred for a request
+LXD_ERROR_STATUS_RESP = 'error'
+
+
+# helpers
+def strip_http_prefix(host):
+    # strip the prefix
+    prefixes = ['http://', 'https://']
+    for prefix in prefixes:
+        if host.startswith(prefix):
+            host = host.strip(prefix)
+    return host
+
+
+def check_certificates(key_file, cert_file, **kwargs):
+    """
+    Basic checks for the provided certificates in LXDtlsConnection
+    """
+
+    # there is no point attempting to connect if either is missing
+    if key_file is None or cert_file is None:
+        raise InvalidCredsError("TLS Connection requires specification "
+                                "of a key file and a certificate file")
+
+    # if they are not none they may be empty strings
+    # or certificates that are not appropriate
+    if key_file == '' or cert_file == '':
+        raise InvalidCredsError("TLS Connection requires specification "
+                                "of a key file and a certificate file")
+
+    # if none of the above check the types
+    if 'key_files_allowed' in kwargs.keys():
+        key_file_suffix = key_file.split('.')
+
+        if key_file_suffix[-1] not in kwargs['key_files_allowed']:
+            raise InvalidCredsError("Valid key files are: " +
+                                    str(kwargs['key_files_allowed']) +
+                                    "you provided: " + key_file_suffix[-1])
+
+            # if none of the above check the types
+    if 'cert_files_allowed' in kwargs.keys():
+        cert_file_suffix = cert_file.split('.')
+
+        if cert_file_suffix[-1] not in kwargs['cert_files_allowed']:
+            raise InvalidCredsError("Valid certification files are: " +
+                                    str(kwargs['cert_files_allowed']) +
+                                    "you provided: " + cert_file_suffix[-1])
+
+    # if all these are good check the paths
+    keypath = os.path.expanduser(key_file)
+    is_file_path = os.path.exists(keypath) and os.path.isfile(keypath)
+    if not is_file_path:
+        raise InvalidCredsError('You need a key file to authenticate with '
+                                'LXD tls. This can be found in the server.')
+
+    certpath = os.path.expanduser(cert_file)
+    is_file_path = os.path.exists(certpath) and os.path.isfile(certpath)
+    if not is_file_path:
+        raise InvalidCredsError('You need a certificate file to '
+                                'authenticate with LXD tls. '
+                                'This can be found in the server.')
+
+
+def assert_response(response_dict, status_code=200):
+
+    # if the type of the response is an error
+    if response_dict['type'] == LXD_ERROR_STATUS_RESP:
+        # an error returned
+        raise LXDAPIException(message="response type is error")
+
+    # anything else apart from the status_code given should be treated as error
+    if response_dict['status_code'] != status_code:
+        # we have an unknown error
+        msg = "Status code should be {0}\
+         but is {1}".format(status_code, response_dict['status_code'])
+        raise LXDAPIException(message=msg)
+
+
+class LXDAPIException(Exception):
+    """
+    Basic exception to be thrown when LXD API
+    returns with some kind of error
+    """
+
+    def __init__(self, message="Unknown Error Occurred"):
+        self.message = message
+
+        super(LXDAPIException, self).__init__(message)
+
+    def __str__(self):
+        return self.message
+
+
+class LXDStoragePool(object):
+    """
+    Utility class representing an LXD storage pool
+    https://lxd.readthedocs.io/en/latest/storage/
+    """
+    def __init__(self, name, driver, used_by, config, managed):
+
+        # the name of the storage pool
+        self.name = name
+
+        # the driver (or type of storage pool). e.g. ‘zfs’ or ‘btrfs’, etc.
+        self.driver = driver
+
+        # which containers (by API endpoint /1.0/containers/<name>)
+        # are using this storage-pool.
+        self.used_by = used_by
+
+        # a dictionary with some information about the storage-pool.
+        # e.g. size, source (path), volume.size, etc.
+        self.config = config
+
+        # Boolean that indicates whether LXD manages the pool or not.
+        self.managed = managed
+
+
+class LXDServerInfo(object):
+    """
+    Wraps the response form /1.0
+    """
+
+    @classmethod
+    def build_from_response(cls, metadata):
+
+        server_info = LXDServerInfo()
+        server_info.api_extensions = metadata.get("api_extensions", None)
+        server_info.api_status = metadata.get("api_status", None)
+        server_info.api_version = metadata.get("api_version", None)
+        server_info.auth = metadata.get("auth", None)
+        server_info.config = metadata.get("config", None)
+        server_info.environment = metadata.get("environment", None)
+        server_info.public = metadata.get("public", None)
+        return server_info
+
+    def __init__(self):
+
+        # List of API extensions added after
+        # the API was marked stable
+        self.api_extensions = None
+
+        # API implementation status
+        # (one of, development, stable or deprecated)
+        self.api_status = None
+
+        # The API version as a string
+        self.api_version = None
+
+        # Authentication state,
+        # one of "guest", "untrusted" or "trusted"
+        self.auth = None
+
+        self.config = None
+
+        # Various information about the host (OS, kernel, ...)
+        self.environment = None
+
+        self.public = None
+
+    def __str__(self):
+        return str(self.api_extensions) + str(self.api_status) + \
+            str(self.api_version) + str(self.auth) + str(self.config) + \
+            str(self.environment) + \
+            str(self.public)
+
+
+class LXDResponse(JsonResponse):
+    valid_response_codes = [httplib.OK, httplib.ACCEPTED, httplib.CREATED,
+                            httplib.NO_CONTENT]
+
+    def parse_body(self):
+
+        if len(self.body) == 0 and not self.parse_zero_length_body:
+            return self.body
+
+        try:
+            # error responses are tricky in Docker. Eg response could be
+            # an error, but response status could still be 200
+            content_type = self.headers.get('content-type', 'application/json')
+            if content_type == 'application/json' or content_type == '':
+                if self.headers.get('transfer-encoding') == 'chunked' and \
+                        'fromImage' in self.request.url:
+                    body = [json.loads(chunk) for chunk in
+                            self.body.strip().replace('\r', '').split('\n')]
+                else:
+                    body = json.loads(self.body)
+            else:
+                body = self.body
+        except ValueError:
+            m = re.search('Error: (.+?)"', self.body)
+            if m:
+                error_msg = m.group(1)
+                raise Exception(error_msg)
+            else:
+                raise Exception(
+                    'ConnectionError: Failed to parse JSON response')
+        return body
+
+    def parse_error(self):
+        if self.status == 401:
+            raise InvalidCredsError('Invalid credentials')
+        return self.body
+
+    def success(self):
+        return self.status in self.valid_response_codes
+
+
+class LXDConnection(ConnectionUserAndKey):
+    responseCls = LXDResponse
+    timeout = 60
+
+    def add_default_headers(self, headers):
+        """
+        Add parameters that are necessary for every request
+        If user and password are specified, include a base http auth
+        header
+        """
+        headers['Content-Type'] = 'application/json'
+        if self.user_id and self.key:
+            user_b64 = base64.b64encode(b('%s:%s' % (self.user_id, self.key)))
+            headers['Authorization'] = 'Basic %s' % (user_b64.decode('utf-8'))
+        return headers
+
+
+class LXDtlsConnection(KeyCertificateConnection):
+
+    responseCls = LXDResponse
+
+    def __init__(self, key, secret, secure=True,
+                 host='localhost', port=8443, ca_cert='',
+                 key_file=None, cert_file=None, **kwargs):
+
+        if 'certificate_validator' in kwargs.keys():
+            certificate_validator = kwargs.pop('certificate_validator')
+            certificate_validator(key_file=key_file, cert_file=cert_file)
+        else:
+            check_certificates(key_file=key_file,
+                               cert_file=cert_file, **kwargs)
+
+        super(LXDtlsConnection, self).__init__(key_file=key_file,
+                                               cert_file=cert_file,
+                                               secure=secure, host=host,
+                                               port=port, url=None,
+                                               proxy_url=None,
+                                               timeout=None, backoff=None,
+                                               retry_delay=None)
+
+        self.key_file = key_file
+        self.cert_file = cert_file
+
+    def add_default_headers(self, headers):
+        headers['Content-Type'] = 'application/json'
+        return headers
+
+
+class LXDContainerDriver(ContainerDriver):
+    """
+    Driver for LXD REST API of LXC containers
+    https://lxd.readthedocs.io/en/stable-2.0/rest-api/
+    https://github.com/lxc/lxd/blob/master/doc/rest-api.md
+    """
+    type = Provider.LXD
+    name = 'LXD'
+    website = 'https://linuxcontainers.org/'
+    connectionCls = LXDConnection
+
+    # LXD supports clustering but still the functionality
+    # is not implemented yet on our side
+    supports_clusters = False
+    version = '1.0'
+    default_time_out = 30
+
+    # default configuration when creating a container
+    default_architecture = 'x86_64'
+    default_profiles = 'default'
+    default_ephemeral = False
+
+    def __init__(self, key='', secret='', secure=False,
+                 host='localhost', port=8443, key_file=None,
+                 cert_file=None, ca_cert=None,
+                 certificate_validator=check_certificates):
+
+        if key_file:
+
+            if not cert_file:
+                # LXD tls authentication-
+                # We pass two files, a key_file with the
+                # private key and cert_file with the certificate
+                # libcloud will handle them through LibcloudHTTPSConnection
+
+                raise LXDAPIException(message='Need both private key and'
+                                              ' certificate files for '
+                                              'tls authentication')
+
+            self.connectionCls = LXDtlsConnection
+            self.key_file = key_file
+            self.cert_file = cert_file
+            self.certificate_validator = certificate_validator
+            secure = True
+
+        if host.startswith('https://'):
+            secure = True
+
+        host = strip_http_prefix(host=host)
+
+        super(LXDContainerDriver, self).__init__(key=key, secret=secret,
+                                                 secure=secure, host=host,
+                                                 port=port,
+                                                 key_file=key_file,
+                                                 cert_file=cert_file)
+
+        if ca_cert:
+            self.connection.connection.ca_cert = ca_cert
+        else:
+            # do not verify SSL certificate
+            self.connection.connection.ca_cert = False
+
+        self.connection.secure = secure
+        self.connection.host = host
+        self.connection.port = port
+        self.version = self._get_api_version()
+
+    def ex_get_api_endpoints(self):
+        """
+        Description: List of supported APIs
+        Authentication: guest
+        Operation: sync
+        Return: list of supported API endpoint URLs
+
+        """
+        response = self.connection.request("/")
+        response_dict = response.parse_body()
+        assert_response(response_dict=response_dict, status_code=200)
+        return response_dict["metadata"]
+
+    def ex_get_server_configuration(self):
+        """
+
+        Description: Server configuration and environment information
+        Authentication: guest, untrusted or trusted
+        Operation: sync
+        Return: Dict representing server state
+
+        The returned configuration depends on whether the connection
+        is trusted or not
+        :rtype: :class: .LXDServerInfo
+
+        """
+        response = self.connection.request("/%s" % (self.version))
+        response_dict = response.parse_body()
+        assert_response(response_dict=response_dict, status_code=200)
+        meta = response_dict["metadata"]
+        return LXDServerInfo.build_from_response(metadata=meta)
+
+    def deploy_container(self, name, image, cluster=None,
+                         parameters=None, start=True,
+                         architecture=default_architecture,
+                         profiles=[default_profiles],
+                         ephemeral=default_ephemeral,
+                         config=None, devices=None,
+                         instance_type=None):
+
+        """
+        Create a new container
+        Authentication: trusted
+        Operation: async
+        Return: background operation or standard error
+
+        :param name: The name of the new container.
+        64 chars max, ASCII, no slash, no colon and no comma
+        :type  name: ``str``
+
+        :param image: The container image to deploy. Currently not used
+        :type  image: :class:`.ContainerImage`
+
+        :param cluster: The cluster to deploy to, None is default
+        :type  cluster: :class:`.ContainerCluster`
+
+        :param parameters: Container Image parameters.
+        This parameter should represent the
+        the ``source`` dictioanry expected by the  LXD API call. For more
+        information how this parameter should be structured see
+        https://github.com/lxc/lxd/blob/master/doc/rest-api.md
+        :type  parameters: ``str``
+
+        :param start: Start the container on deployment. True is the default
+        :type  start: ``bool``
+
+        :param architecture: string e.g. x86_64
+        :type  architecture: ``str``
+
+        :param profiles: List of profiles
+        :type  profiles: ``list``
+
+        :param ephemeral: Whether to destroy the container on shutdown
+        :type  ephemeral: ``bool``
+
+        :param config: Config override e.g.  {"limits.cpu": "2"}
+        :type  config: ``dict``
+
+        :param devices: optional list of devices the container should have
+        :type  devices: ``dict``
+
+        :param instance_type: An optional instance type
+        to use as basis for limits e.g. "c2.micro"
+        :type  instance_type: ``str``
+
+        :rtype: :class:`.Container`
+        """
+
+        c_params = {"architecture": architecture, "profiles": profiles,
+                    "ephemeral": ephemeral, "config": config,
+                    "devices": devices, "instance_type": instance_type}
+
+        if parameters:
+            parameters = json.loads(parameters)
+
+        container = self._deploy_container_from_image(name=name, image=image,
+                                                      parameters=parameters,
+                                                      cont_params=c_params)
+
+        if start:
+            container.start()
+
+        return container
+
+    def get_container(self, id, get_ip_addr=True):
+
+        """
+        Get a container by ID
+
+        :param id: The ID of the container to get
+        :type  id: ``str``
+
+        :param get_ip_addr: Indicates whether ip addresses
+        should also be included. This requires an extra GET request
+        :type  get_ip_addr: ``boolean```
+
+        :rtype: :class:`libcloud.container.base.Container`
+        """
+        req = "/%s/containers/%s" % (self.version, id)
+        response = self.connection.request(req)
+        result_dict = response.parse_body()
+        assert_response(response_dict=result_dict, status_code=200)
+
+        metadata = result_dict["metadata"]
+
+        ips = []
+        if get_ip_addr:
+            req = "/%s/containers/%s/state" % (self.version, id)
+            ip_response = self.connection.request(req)
+
+            ip_result_dict = ip_response.parse_body()
+            assert_response(response_dict=ip_result_dict, status_code=200)
+
+            networks = None
+
+            if ip_result_dict["metadata"]["network"] is not None:
+                networks = ip_result_dict["metadata"]["network"]["eth0"]
+
+                # the list of addresses
+                addresses = networks["addresses"]
+
+                for item in addresses:
+                    ips.append(item["address"])
+
+        metadata.update({"ips": ips})
+        return self._to_container(metadata=metadata)
+
+    def start_container(self, container, timeout=default_time_out):
 
 Review comment:
   Please prefix ``timeout`` with ``ex_`` since it's not part of the standard API.

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services

Mime
View raw message