tempest/lib/services/clients.py

# Copyright 2012 OpenStack Foundation
# Copyright (c) 2016 Hewlett-Packard Enterprise Development Company, L.P.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import copy
import importlib
import inspect
import sys

from debtcollector import removals
from oslo_log import log as logging
import testtools

from tempest.lib import auth
from tempest.lib.common.utils import misc
from tempest.lib import exceptions
from tempest.lib.services import compute
from tempest.lib.services import identity
from tempest.lib.services import image
from tempest.lib.services import network
from tempest.lib.services import object_storage
from tempest.lib.services import placement
from tempest.lib.services import volume

LOG = logging.getLogger(__name__)


def tempest_modules():
    """Dict of service client modules available in Tempest.

    Provides a dict of stable service modules available in Tempest, with
    ``service_version`` as key, and the module object as value.
    """
    return {
        'compute': compute,
        'placement': placement,
        'identity.v2': identity.v2,
        'identity.v3': identity.v3,
        'image.v1': image.v1,
        'image.v2': image.v2,
        'network': network,
        'object-storage': object_storage,
        'volume.v2': volume.v2,
        'volume.v3': volume.v3
    }


def available_modules():
    """Set of service client modules available in Tempest and plugins

    Set of stable service clients from Tempest and service clients exposed
    by plugins. This set of available modules can be used for automatic
    configuration.

    :raise PluginRegistrationException: if a plugin exposes a service_version
        already defined by Tempest or another plugin.

    Examples::

        from tempest import config
        params = {}
        for service_version in available_modules():
            service = service_version.split('.')[0]
            params[service] = config.service_client_config(service)
        service_clients = ServiceClients(creds, identity_uri,
                                         client_parameters=params)
    """
    extra_service_versions = set([])
    _tempest_modules = set(tempest_modules())
    plugin_services = ClientsRegistry().get_service_clients()
    name_conflicts = []
    for plugin_name in plugin_services:
        plug_service_versions = set([x['service_version'] for x in
                                     plugin_services[plugin_name]])
        # If a plugin exposes a duplicate service_version raise an exception
        if plug_service_versions:
            if not plug_service_versions.isdisjoint(extra_service_versions):
                detailed_error = (
                    'Plugin %s is trying to register a service %s already '
                    'claimed by another one' % (plugin_name,
                                                extra_service_versions &
                                                plug_service_versions))
                name_conflicts.append(exceptions.PluginRegistrationException(
                    name=plugin_name, detailed_error=detailed_error))
        extra_service_versions |= plug_service_versions
    if name_conflicts:
        LOG.error(
            'Failed to list available modules due to name conflicts: %s',
            name_conflicts)
        raise testtools.MultipleExceptions(*name_conflicts)
    return _tempest_modules | extra_service_versions


@misc.singleton
class ClientsRegistry(object):
    """Registry of all service clients available from plugins"""

    def __init__(self):
        self._service_clients = {}

    def register_service_client(self, plugin_name, service_client_data):
        if plugin_name in self._service_clients:
            detailed_error = 'Clients for plugin %s already registered'
            raise exceptions.PluginRegistrationException(
                name=plugin_name,
                detailed_error=detailed_error % plugin_name)
        self._service_clients[plugin_name] = service_client_data
        LOG.debug("Successfully registered plugin %s in the service client "
                  "registry with configuration: %s", plugin_name,
                  service_client_data)

    def get_service_clients(self):
        return self._service_clients


class ClientsFactory(object):
    """Builds service clients for a service client module

    This class implements the logic of feeding service client parameters
    to service clients from a specific module. It allows setting the
    parameters once and obtaining new instances of the clients without the
    need of passing any parameter.

    ClientsFactory can be used directly, or consumed via the `ServiceClients`
    class, which manages the authorization part.
    """

    def __init__(self, module_path, client_names, auth_provider, **kwargs):
        """Initialises the client factory

        :param module_path: Path to module that includes all service clients.
            All service client classes must be exposed by a single module.
            If they are separated in different modules, defining __all__
            in the root module can help, similar to what is done by service
            clients in tempest.
        :param client_names: List or set of names of the service client
            classes.
        :param auth_provider: The auth provider used to initialise client.
        :param kwargs: Parameters to be passed to all clients. Parameters
            values can be overwritten when clients are initialised, but
            parameters cannot be deleted.
        :raise ImportError: if the specified module_path cannot be imported

        Example::

            # Get credentials and an auth_provider
            clients = ClientsFactory(
                module_path='my_service.my_service_clients',
                client_names=['ServiceClient1', 'ServiceClient2'],
                auth_provider=auth_provider,
                service='my_service',
                region='region1')
            my_api_client = clients.MyApiClient()
            my_api_client_region2 = clients.MyApiClient(region='region2')

        """
        # Import the module. If it's not importable, the raised exception
        # provides good enough information about what happened
        _module = importlib.import_module(module_path)
        # If any of the classes is not in the module we fail
        for class_name in client_names:
            # TODO(andreaf) This always passes all parameters to all clients.
            # In future to allow clients to specify the list of parameters
            # that they accept based out of a list of standard ones.

            # Obtain the class
            klass = self._get_class(_module, class_name)
            final_kwargs = copy.copy(kwargs)

            # Set the function as an attribute of the factory
            setattr(self, class_name, self._get_partial_class(
                klass, auth_provider, final_kwargs))

    def _get_partial_class(self, klass, auth_provider, kwargs):

        # Define a function that returns a new class instance by
        # combining default kwargs with extra ones
        def partial_class(alias=None, **later_kwargs):
            """Returns a callable the initialises a service client

            Builds a callable that accepts kwargs, which are passed through
            to the __init__ of the service client, along with a set of defaults
            set in factory at factory __init__ time.
            Original args in the service client can only be passed as kwargs.

            It accepts one extra parameter 'alias' compared to the original
            service client. When alias is provided, the returned callable will
            also set an attribute called with a name defined in 'alias', which
            contains the instance of the service client.

            :param alias: str Name of the attribute set on the factory once
                the callable is invoked which contains the initialised
                service client. If None, no attribute is set.
            :param later_kwargs: kwargs passed through to the service client
                __init__ on top of defaults set at factory level.
            """
            kwargs.update(later_kwargs)
            _client = klass(auth_provider=auth_provider, **kwargs)
            if alias:
                setattr(self, alias, _client)
            return _client

        return partial_class

    @classmethod
    def _get_class(cls, module, class_name):
        klass = getattr(module, class_name, None)
        if not klass:
            msg = 'Invalid class name, %s is not found in %s'
            raise AttributeError(msg % (class_name, module))
        if not inspect.isclass(klass):
            msg = 'Expected a class, got %s of type %s instead'
            raise TypeError(msg % (klass, type(klass)))
        return klass


class ServiceClients(object):
    """Service client provider class

    The ServiceClients object provides a useful means for tests to access
    service clients configured for a specified set of credentials.
    It hides some of the complexity from the authorization and configuration
    layers.

    Examples::

        # johndoe is a tempest.lib.auth.Credentials type instance
        johndoe_clients = clients.ServiceClients(johndoe, identity_uri)

        # List servers in default region
        johndoe_servers_client = johndoe_clients.compute.ServersClient()
        johndoe_servers = johndoe_servers_client.list_servers()

        # List servers in Region B
        johndoe_servers_client_B = johndoe_clients.compute.ServersClient(
            region='B')
        johndoe_servers = johndoe_servers_client_B.list_servers()

    """
    # NOTE(andreaf) This class does not depend on tempest configuration
    # and its meant for direct consumption by external clients such as tempest
    # plugins. Tempest provides a wrapper class, `clients.Manager`, that
    # initialises this class using values from tempest CONF object. The wrapper
    # class should only be used by tests hosted in Tempest.

    @removals.removed_kwarg('client_parameters')
    def __init__(self, credentials, identity_uri, region=None, scope=None,
                 disable_ssl_certificate_validation=True, ca_certs=None,
                 trace_requests='', client_parameters=None, proxy_url=None):
        """Service Clients provider

        Instantiate a `ServiceClients` object, from a set of credentials and an
        identity URI. The identity version is inferred from the credentials
        object. Optionally auth scope can be provided.

        A few parameters can be given a value which is applied as default
        for all service clients: region, dscv, ca_certs, trace_requests.

        Parameters dscv, ca_certs and trace_requests all apply to the auth
        provider as well as any service clients provided by this manager.

        Any other client parameter should be set via ClientsRegistry.

        Client parameter used to be set via client_parameters, but this is
        deprecated, and it is actually already not honoured
        anymore: https://launchpad.net/bugs/1680915.

        The list of available parameters is defined in the service clients
        interfaces. For reference, most clients will accept 'region',
        'service', 'endpoint_type', 'build_timeout' and 'build_interval', which
        are all inherited from RestClient.

        The `config` module in Tempest exposes an helper function
        `service_client_config` that can be used to extract from configuration
        a dictionary ready to be injected in kwargs.

        Exceptions are:
        - Token clients for 'identity' must be given an 'auth_url' parameter
        - Volume client for 'volume' accepts 'default_volume_size'
        - Servers client from 'compute' accepts 'enable_instance_password'

        If Tempest configuration is used, parameters will be loaded in the
        Registry automatically for all service client (Tempest stable ones
        and plugins).

        Examples::

            identity_params = config.service_client_config('identity')
            params = {
                'identity': identity_params,
                'compute': {'region': 'region2'}}
            manager = lib_manager.Manager(
                my_creds, identity_uri, client_parameters=params)

        :param credentials: An instance of `auth.Credentials`
        :param identity_uri: URI of the identity API. This should be a
                             mandatory parameter, and it will so soon.
        :param region: Default value of region for service clients.
        :param scope: default scope for tokens produced by the auth provider
        :param disable_ssl_certificate_validation: Applies to auth and to all
                                                  service clients.
        :param ca_certs: Applies to auth and to all service clients.
        :param trace_requests: Applies to auth and to all service clients.
        :param client_parameters: Dictionary with parameters for service
            clients. Keys of the dictionary are the service client service
            name, as declared in `service_clients.available_modules()` except
            for the version. Values are dictionaries of parameters that are
            going to be passed to all clients in the service client module.
        :param proxy_url: Applies to auth and to all service clients, set a
            proxy url for the clients to use.
        """
        self._registered_services = set([])
        self.credentials = credentials
        self.identity_uri = identity_uri
        if not identity_uri:
            raise exceptions.InvalidCredentials(
                'ServiceClients requires a non-empty identity_uri.')
        self.region = region
        # Check if passed or default credentials are valid
        if not self.credentials.is_valid():
            raise exceptions.InvalidCredentials(credentials)
        # Get the identity classes matching the provided credentials
        # TODO(andreaf) Define a new interface in Credentials to get
        # the API version from an instance
        identity = [(k, auth.IDENTITY_VERSION[k][1]) for k in
                    auth.IDENTITY_VERSION.keys() if
                    isinstance(self.credentials, auth.IDENTITY_VERSION[k][0])]
        # Zero matches or more than one are both not valid.
        if len(identity) != 1:
            msg = "Zero or %d ambiguous auth provider found. identity: %s, " \
                "credentials: %s" % (len(identity), identity, credentials)
            raise exceptions.InvalidCredentials(msg)
        self.auth_version, auth_provider_class = identity[0]
        self.dscv = disable_ssl_certificate_validation
        self.ca_certs = ca_certs
        self.trace_requests = trace_requests
        self.proxy_url = proxy_url
        if self.credentials.project_id or self.credentials.project_name:
            scope = 'project'
        elif self.credentials.system:
            scope = 'system'
        elif self.credentials.domain_id or self.credentials.domain_name:
            scope = 'domain'
        else:
            scope = 'project'
        # Creates an auth provider for the credentials
        self.auth_provider = auth_provider_class(
            self.credentials, self.identity_uri, scope=scope,
            disable_ssl_certificate_validation=self.dscv,
            ca_certs=self.ca_certs, trace_requests=self.trace_requests,
            proxy_url=proxy_url)

        # Setup some defaults for client parameters of registered services
        client_parameters = client_parameters or {}
        self.parameters = {}

        # Parameters are provided for unversioned services
        all_modules = available_modules()
        unversioned_services = set(
            [x.split('.')[0] for x in all_modules])
        for service in unversioned_services:
            self.parameters[service] = self._setup_parameters(
                client_parameters.pop(service, {}))
        # Check that no client parameters was supplied for unregistered clients
        if client_parameters:
            raise exceptions.UnknownServiceClient(
                services=list(client_parameters.keys()))

        # Register service clients from the registry (__tempest__ and plugins)
        clients_registry = ClientsRegistry()
        plugin_service_clients = clients_registry.get_service_clients()
        registration_errors = []
        for plugin in plugin_service_clients:
            service_clients = plugin_service_clients[plugin]
            # Each plugin returns a list of service client parameters
            for service_client in service_clients:
                # NOTE(andreaf) If a plugin cannot register, stop the
                # registration process, log some details to help
                # troubleshooting, and re-raise
                try:
                    self.register_service_client_module(**service_client)
                except Exception:
                    registration_errors.append(sys.exc_info())
                    LOG.exception(
                        'Failed to register service client from plugin %s '
                        'with parameters %s', plugin, service_client)
        if registration_errors:
            raise testtools.MultipleExceptions(*registration_errors)

    def register_service_client_module(self, name, service_version,
                                       module_path, client_names, **kwargs):
        """Register a service client module

        Initiates a client factory for the specified module, using this
        class auth_provider, and accessible via a `name` attribute in the
        service client.

        :param name: Name used to access the client
        :param service_version: Name of the service complete with version.
            Used to track registered services. When a plugin implements it,
            it can be used by other plugins to obtain their configuration.
        :param module_path: Path to module that includes all service clients.
            All service client classes must be exposed by a single module.
            If they are separated in different modules, defining __all__
            in the root module can help, similar to what is done by service
            clients in tempest.
        :param client_names: List or set of names of service client classes.
        :param kwargs: Extra optional parameters to be passed to all clients.
            ServiceClient provides defaults for region, dscv, ca_certs, http
            proxies and trace_requests.
        :raise ServiceClientRegistrationException: if the provided name is
            already in use or if service_version is already registered.
        :raise ImportError: if module_path cannot be imported.
        """
        if hasattr(self, name):
            using_name = getattr(self, name)
            detailed_error = 'Module name already in use: %s' % using_name
            raise exceptions.ServiceClientRegistrationException(
                name=name, service_version=service_version,
                module_path=module_path, client_names=client_names,
                detailed_error=detailed_error)
        if service_version in self.registered_services:
            detailed_error = 'Service %s already registered.' % service_version
            raise exceptions.ServiceClientRegistrationException(
                name=name, service_version=service_version,
                module_path=module_path, client_names=client_names,
                detailed_error=detailed_error)
        params = dict(region=self.region,
                      disable_ssl_certificate_validation=self.dscv,
                      ca_certs=self.ca_certs,
                      trace_requests=self.trace_requests,
                      proxy_url=self.proxy_url)
        params.update(kwargs)
        # Instantiate the client factory
        _factory = ClientsFactory(module_path=module_path,
                                  client_names=client_names,
                                  auth_provider=self.auth_provider,
                                  **params)
        # Adds the client factory to the service_client
        setattr(self, name, _factory)
        # Add the name of the new service in self.SERVICES for discovery
        self._registered_services.add(service_version)

    @property
    def registered_services(self):
        return self._registered_services

    def _setup_parameters(self, parameters):
        """Setup default values for client parameters

        Region by default is the region passed as an __init__ parameter.
        Checks that no parameter for an unknown service is provided.
        """
        _parameters = {}
        # Use region from __init__
        if self.region:
            _parameters['region'] = self.region
        # Update defaults with specified parameters
        _parameters.update(parameters)
        # If any parameter is left, parameters for an unknown service were
        # provided as input. Fail rather than ignore silently.
        return _parameters