tempest/common/custom_matchers.py

# Copyright 2013 NTT Corporation
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import re

from testtools import helpers


class ExistsAllResponseHeaders(object):
    """Specific matcher to check the existence of Swift's response headers

    This matcher checks the existence of common headers for each HTTP method
    or the target, which means account, container or object.
    When checking the existence of 'specific' headers such as
    X-Account-Meta-* or X-Object-Manifest for example, those headers must be
    checked in each test code.
    """

    def __init__(self, target, method, policies=None):
        """Initialization of ExistsAllResponseHeaders

        param: target Account/Container/Object
        param: method PUT/GET/HEAD/DELETE/COPY/POST
        """
        self.target = target
        self.method = method
        self.policies = policies or []

    def _content_length_required(self, resp):
        # Verify whether given HTTP response must contain content-length.
        # Take into account the exceptions defined in RFC 7230.
        if resp.status in range(100, 200) or resp.status == 204:
            return False

        return True

    def match(self, actual):
        """Check headers

        param: actual HTTP response object containing headers and status
        """
        # Check common headers for all HTTP methods.
        #
        # Please note that for 1xx and 204 responses Content-Length presence
        # is not checked intensionally. According to RFC 7230 a server MUST
        # NOT send the header in such responses. Thus, clients should not
        # depend on this header. However, the standard does not require them
        # to validate the server's behavior. We leverage that to not refuse
        # any implementation violating it like Swift [1] or some versions of
        # Ceph RadosGW [2].
        # [1] https://bugs.launchpad.net/swift/+bug/1537811
        # [2] http://tracker.ceph.com/issues/13582
        if ('content-length' not in actual and
                'transfer-encoding' not in actual and
                self._content_length_required(actual)):
            return NonExistentHeaders(['content-length', 'transfer-encoding'])
        if 'content-type' not in actual:
            return NonExistentHeader('content-type')
        if 'x-trans-id' not in actual:
            return NonExistentHeader('x-trans-id')
        if 'date' not in actual:
            return NonExistentHeader('date')

        # Check headers for a specific method or target
        if self.method == 'GET' or self.method == 'HEAD':
            if 'x-timestamp' not in actual:
                return NonExistentHeader('x-timestamp')
            if self.target == 'Account':
                if 'x-account-bytes-used' not in actual:
                    return NonExistentHeader('x-account-bytes-used')
                if 'x-account-container-count' not in actual:
                    return NonExistentHeader('x-account-container-count')
                if 'x-account-object-count' not in actual:
                    return NonExistentHeader('x-account-object-count')
                if int(actual['x-account-container-count']) > 0:
                    acct_header = "x-account-storage-policy-"
                    matched_policy_count = 0

                    # Loop through the policies and look for account
                    # usage data.  There should be at least 1 set
                    for policy in self.policies:
                        front_header = acct_header + policy['name'].lower()

                        usage_policies = [
                            front_header + '-bytes-used',
                            front_header + '-object-count',
                            front_header + '-container-count'
                        ]

                        # There should be 3 usage values for a give storage
                        # policy in an account bytes, object count, and
                        # container count
                        policy_hdrs = sum(1 for use_hdr in usage_policies
                                          if use_hdr in actual)

                        # If there are less than 3 headers here then 1 is
                        # missing, let's figure out which one and report
                        if policy_hdrs == 3:
                            matched_policy_count = matched_policy_count + 1
                        else:
                            if policy_hdrs > 0 and policy_hdrs < 3:
                                for use_hdr in usage_policies:
                                    if use_hdr not in actual:
                                        return NonExistentHeader(use_hdr)

                    # Only flag an error if actual policies have been read and
                    # no usage has been found
                    if self.policies and matched_policy_count == 0:
                        return GenericError("No storage policy usage headers")

            elif self.target == 'Container':
                if 'x-container-bytes-used' not in actual:
                    return NonExistentHeader('x-container-bytes-used')
                if 'x-container-object-count' not in actual:
                    return NonExistentHeader('x-container-object-count')
                if 'x-storage-policy' not in actual:
                    return NonExistentHeader('x-storage-policy')
                else:
                    policy_name = actual['x-storage-policy']

                    # loop through the policies and ensure that
                    # the value in the container header matches
                    # one of the storage policies
                    for policy in self.policies:
                        if policy['name'] == policy_name:
                            break
                    else:
                        # Ensure that there are actual policies stored
                        if self.policies:
                            return InvalidHeaderValue('x-storage-policy',
                                                      policy_name)
            elif self.target == 'Object':
                if 'etag' not in actual:
                    return NonExistentHeader('etag')
                if 'last-modified' not in actual:
                    return NonExistentHeader('last-modified')
        elif self.method == 'PUT':
            if self.target == 'Object':
                if 'etag' not in actual:
                    return NonExistentHeader('etag')
                if 'last-modified' not in actual:
                    return NonExistentHeader('last-modified')
        elif self.method == 'COPY':
            if self.target == 'Object':
                if 'etag' not in actual:
                    return NonExistentHeader('etag')
                if 'last-modified' not in actual:
                    return NonExistentHeader('last-modified')
                if 'x-copied-from' not in actual:
                    return NonExistentHeader('x-copied-from')
                if 'x-copied-from-last-modified' not in actual:
                    return NonExistentHeader('x-copied-from-last-modified')

        return None


class GenericError(object):
    """Informs an error message of a generic error during header evaluation"""

    def __init__(self, body):
        self.body = body

    def describe(self):
        return "%s" % self.body

    def get_details(self):
        return {}


class NonExistentHeader(object):
    """Informs an error message in the case of missing a certain header"""

    def __init__(self, header):
        self.header = header

    def describe(self):
        return "%s header does not exist" % self.header

    def get_details(self):
        return {}


class NonExistentHeaders(object):
    """Informs an error message in the case of missing certain headers"""

    def __init__(self, headers):
        self.headers = headers

    def describe(self):
        return "none of these headers exist: %s" % self.headers

    def get_details(self):
        return {}


class InvalidHeaderValue(object):
    """Informs an error message when a header contains a bad value"""

    def __init__(self, header, value):
        self.header = header
        self.value = value

    def describe(self):
        return "InvalidValue (%s, %s)" % (self.header, self.value)

    def get_details(self):
        return {}


class AreAllWellFormatted(object):
    """Specific matcher to check the correctness of formats of values

    This matcher checks the format of values of response headers.
    When checking the format of values of 'specific' headers such as
    X-Account-Meta-* or X-Object-Manifest for example, those values must be
    checked in each test code.
    """

    def match(self, actual):
        for key, value in actual.items():
            if key in ('content-length', 'x-account-bytes-used',
                       'x-account-container-count', 'x-account-object-count',
                       'x-container-bytes-used', 'x-container-object-count')\
                and not value.isdigit():
                return InvalidFormat(key, value)
            elif key in ('content-type', 'date', 'last-modified',
                         'x-copied-from-last-modified') and not value:
                return InvalidFormat(key, value)
            elif key == 'x-timestamp' and not re.match(r"^\d+\.?\d*\Z", value):
                return InvalidFormat(key, value)
            elif key == 'x-copied-from' and not re.match(r"\S+/\S+", value):
                return InvalidFormat(key, value)
            elif key == 'x-trans-id' and \
                not re.match("^tx[0-9a-f]{21}-[0-9a-f]{10}.*", value):
                return InvalidFormat(key, value)
            elif key == 'accept-ranges' and not value == 'bytes':
                return InvalidFormat(key, value)
            elif key == 'etag' and not value.isalnum():
                return InvalidFormat(key, value)
            elif key == 'transfer-encoding' and not value == 'chunked':
                return InvalidFormat(key, value)

        return None


class InvalidFormat(object):
    """Informs an error message if a format of a certain header is invalid"""

    def __init__(self, key, value):
        self.key = key
        self.value = value

    def describe(self):
        return "InvalidFormat (%s, %s)" % (self.key, self.value)

    def get_details(self):
        return {}


class MatchesDictExceptForKeys(object):
    """Matches two dictionaries.

    Verifies all items are equals except for those identified by a list of keys
    """

    def __init__(self, expected, excluded_keys=None):
        self.expected = expected
        self.excluded_keys = excluded_keys if excluded_keys is not None else []

    def match(self, actual):
        filtered_expected = helpers.dict_subtract(self.expected,
                                                  self.excluded_keys)
        filtered_actual = helpers.dict_subtract(actual,
                                                self.excluded_keys)
        if filtered_actual != filtered_expected:
            return DictMismatch(filtered_expected, filtered_actual)


class DictMismatch(object):
    """Mismatch between two dicts describes deltas"""

    def __init__(self, expected, actual):
        self.expected = expected
        self.actual = actual
        self.intersect = set(self.expected) & set(self.actual)
        self.symmetric_diff = set(self.expected) ^ set(self.actual)

    def _format_dict(self, dict_to_format):
        # Ensure the error string dict is printed in a set order
        # NOTE(mtreinish): needed to ensure a deterministic error msg for
        # testing. Otherwise the error message will be dependent on the
        # dict ordering.
        dict_string = "{"
        for key in sorted(dict_to_format):
            dict_string += "'%s': %s, " % (key, dict_to_format[key])
        dict_string = dict_string[:-2] + '}'
        return dict_string

    def describe(self):
        msg = ""
        if self.symmetric_diff:
            only_expected = helpers.dict_subtract(self.expected, self.actual)
            only_actual = helpers.dict_subtract(self.actual, self.expected)
            if only_expected:
                msg += "Only in expected:\n  %s\n" % self._format_dict(
                    only_expected)
            if only_actual:
                msg += "Only in actual:\n  %s\n" % self._format_dict(
                    only_actual)
        diff_set = set(o for o in self.intersect if
                       self.expected[o] != self.actual[o])
        if diff_set:
            msg += "Differences:\n"
            for o in diff_set:
                msg += "  %s: expected %s, actual %s\n" % (
                    o, self.expected[o], self.actual[o])
        return msg

    def get_details(self):
        return {}