diff --git a/requirements.txt b/requirements.txt
index 6e66046..b0df18b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -23,3 +23,4 @@
 debtcollector>=1.2.0 # Apache-2.0
 defusedxml>=0.7.1 # PSFL
 fasteners>=0.16.0 # Apache-2.0
+testscenarios>=0.5.0
diff --git a/tempest/api/image/v2/test_images_formats.py b/tempest/api/image/v2/test_images_formats.py
new file mode 100644
index 0000000..a234fa2
--- /dev/null
+++ b/tempest/api/image/v2/test_images_formats.py
@@ -0,0 +1,188 @@
+# Copyright 2024 Red Hat, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os
+
+import testscenarios
+import yaml
+
+from tempest.api.compute import base as compute_base
+from tempest.api.image import base
+from tempest.common import waiters
+from tempest import config
+from tempest import exceptions
+from tempest.lib.common.utils import data_utils
+from tempest.lib import decorators
+from tempest.lib import exceptions as lib_exc
+
+CONF = config.CONF
+
+
+def load_tests(loader, suite, pattern):
+    """Generate scenarios from the image manifest."""
+    if CONF.image.images_manifest_file is None:
+        return suite
+    ImagesFormatTest.scenarios = []
+    with open(CONF.image.images_manifest_file) as f:
+        ImagesFormatTest._manifest = yaml.load(f, Loader=yaml.SafeLoader)
+        for imgdef in ImagesFormatTest._manifest['images']:
+            ImagesFormatTest.scenarios.append((imgdef['name'],
+                                               {'imgdef': imgdef}))
+    result = loader.suiteClass()
+    result.addTests(testscenarios.generate_scenarios(suite))
+    return result
+
+
+class ImagesFormatTest(base.BaseV2ImageTest,
+                       compute_base.BaseV2ComputeTest):
+    def setUp(self):
+        super().setUp()
+        if CONF.image.images_manifest_file is None:
+            self.skipTest('Image format testing is not configured')
+        self._image_base = os.path.dirname(os.path.abspath(
+            CONF.image.images_manifest_file))
+
+        self.images = []
+
+    def tearDown(self):
+        for img in self.images:
+            try:
+                self.client.delete_image(img['id'])
+            except lib_exc.NotFound:
+                pass
+        return super().tearDown()
+
+    @classmethod
+    def resource_setup(cls):
+        super().resource_setup()
+        cls.available_import_methods = cls.client.info_import()[
+            'import-methods']['value']
+
+    def _test_image(self, image_def, override_format=None, asimport=False):
+        image_name = data_utils.rand_name(
+            prefix=CONF.resource_name_prefix,
+            name=image_def['name'])
+        image = self.client.create_image(
+            name=image_name,
+            container_format='bare',
+            disk_format=override_format or image_def['format'])
+        self.images.append(image)
+        image_fn = os.path.join(self._image_base, image_def['filename'])
+        with open(image_fn, 'rb') as f:
+            if asimport:
+                self.client.stage_image_file(image['id'], f)
+                self.client.image_import(image['id'], method='glance-direct')
+            else:
+                self.client.store_image_file(image['id'], f)
+        return image
+
+    @decorators.idempotent_id('a245fcbe-63ce-4dc1-a1d0-c16d76d9e6df')
+    def test_accept_usable_formats(self):
+        if self.imgdef['usable']:
+            if self.imgdef['format'] in CONF.image.disk_formats:
+                # These are expected to work
+                self._test_image(self.imgdef)
+            else:
+                # If this is not configured to be supported, we should get
+                # a BadRequest from glance
+                self.assertRaises(lib_exc.BadRequest,
+                                  self._test_image, self.imgdef)
+        else:
+            self.skipTest(
+                'Glance does not currently reject unusable images on upload')
+
+    @decorators.idempotent_id('7c7c2f16-2e97-4dce-8cb4-bc10be031c85')
+    def test_accept_reject_formats_import(self):
+        """Make sure glance rejects invalid images during conversion."""
+        if 'glance-direct' not in self.available_import_methods:
+            self.skipTest('Import via glance-direct is not available')
+        if not CONF.image_feature_enabled.image_conversion:
+            self.skipTest('Import image_conversion not enabled')
+
+        if self.imgdef['format'] == 'iso':
+            # TODO(danms): Glance does not properly handle ISO conversions
+            # today and this is being fixed currently. Remove when this
+            # is stable and able to be tested.
+            self.skipTest('Glance ISO conversion is not testable')
+
+        glance_noconvert = [
+            # Glance does not support vmdk-sparse-with-footer with the
+            # in-tree format_inspector
+            'vmdk-sparse-with-footer',
+            ]
+        # Any images glance does not support in *conversion* for some
+        # reason will fail, even though the manifest marks them as usable.
+        expect_fail = any(x in self.imgdef['name']
+                          for x in glance_noconvert)
+
+        if (self.imgdef['format'] in CONF.image.disk_formats and
+                self.imgdef['usable'] and not expect_fail):
+            # Usable images should end up in active state
+            image = self._test_image(self.imgdef, asimport=True)
+            waiters.wait_for_image_status(self.client, image['id'],
+                                          'active')
+        else:
+            # FIXME(danms): Make this better, but gpt will fail before
+            # the import even starts until glance has it in its API
+            # schema as a valid value. Other formats expected to fail
+            # do so during import and return to queued state.
+            if self.imgdef['format'] not in CONF.image.disk_formats:
+                self.assertRaises(lib_exc.BadRequest,
+                                  self._test_image,
+                                  self.imgdef, asimport=True)
+            else:
+                image = self._test_image(self.imgdef, asimport=True)
+                waiters.wait_for_image_status(self.client, image['id'],
+                                              'queued')
+                self.client.delete_image(image['id'])
+
+    def _create_server_with_image_def(self, image_def, **overrides):
+        image_def = dict(image_def, **overrides)
+        image = self._test_image(image_def)
+        server = self.create_test_server(name='server-%s' % image['name'],
+                                         image_id=image['id'],
+                                         wait_until='ACTIVE')
+        return server
+
+    @decorators.idempotent_id('f77394bc-81f4-4d54-9f5b-e48f3d6b5376')
+    def test_compute_rejects_invalid(self):
+        """Make sure compute rejects invalid/insecure images."""
+        if self.imgdef['format'] not in CONF.image.disk_formats:
+            # if this format is not allowed by glance, we can not create
+            # a properly-formatted image for it, so skip it.
+            self.skipTest(
+                'Format %s not allowed by config' % self.imgdef['format'])
+
+        # VMDK with footer is not supported by anyone yet until fixed:
+        # https://bugs.launchpad.net/glance/+bug/2073262
+        is_broken = 'footer' in self.imgdef['name']
+
+        if self.imgdef['usable'] and not is_broken:
+            server = self._create_server_with_image_def(self.imgdef)
+            self.delete_server(server['id'])
+        else:
+            self.assertRaises(exceptions.BuildErrorException,
+                              self._create_server_with_image_def,
+                              self.imgdef)
+
+    @decorators.idempotent_id('ffe21610-e801-4992-9b81-e2d646e2e2e9')
+    def test_compute_rejects_format_mismatch(self):
+        """Make sure compute rejects any image with a format mismatch."""
+        # Lying about the disk_format should always fail
+        override_fmt = (
+            self.imgdef['format'] in ('raw', 'gpt') and 'qcow2' or 'raw')
+        self.assertRaises(exceptions.BuildErrorException,
+                          self._create_server_with_image_def,
+                          self.imgdef,
+                          format=override_fmt)
diff --git a/tempest/cmd/cleanup_service.py b/tempest/cmd/cleanup_service.py
index b202940..db4407d 100644
--- a/tempest/cmd/cleanup_service.py
+++ b/tempest/cmd/cleanup_service.py
@@ -115,21 +115,32 @@
         return [item for item in item_list
                 if item['tenant_id'] == self.tenant_id]
 
-    def _filter_by_prefix(self, item_list):
-        items = [item for item in item_list
-                 if item['name'].startswith(self.prefix)]
+    def _filter_by_prefix(self, item_list, top_key=None):
+        items = []
+        for item in item_list:
+            name = item[top_key]['name'] if top_key else item['name']
+            if name.startswith(self.prefix):
+                items.append(item)
         return items
 
     def _filter_by_resource_list(self, item_list, attr):
         if attr not in self.resource_list_json:
             return []
-        items = [item for item in item_list if item['id']
-                 in self.resource_list_json[attr].keys()]
+        items = []
+        for item in item_list:
+            item_id = (item['keypair']['name'] if attr == 'keypairs'
+                       else item['id'])
+            if item_id in self.resource_list_json[attr].keys():
+                items.append(item)
         return items
 
     def _filter_out_ids_from_saved(self, item_list, attr):
-        items = [item for item in item_list if item['id']
-                 not in self.saved_state_json[attr].keys()]
+        items = []
+        for item in item_list:
+            item_id = (item['keypair']['name'] if attr == 'keypairs'
+                       else item['id'])
+            if item_id not in self.saved_state_json[attr].keys():
+                items.append(item)
         return items
 
     def list(self):
@@ -294,16 +305,11 @@
         keypairs = client.list_keypairs()['keypairs']
 
         if self.prefix:
-            keypairs = self._filter_by_prefix(keypairs)
+            keypairs = self._filter_by_prefix(keypairs, 'keypair')
         elif self.is_resource_list:
-            keypairs = [keypair for keypair in keypairs
-                        if keypair['keypair']['name']
-                        in self.resource_list_json['keypairs'].keys()]
+            keypairs = self._filter_by_resource_list(keypairs, 'keypairs')
         elif not self.is_save_state:
-            # recreate list removing saved keypairs
-            keypairs = [keypair for keypair in keypairs
-                        if keypair['keypair']['name']
-                        not in self.saved_state_json['keypairs'].keys()]
+            keypairs = self._filter_out_ids_from_saved(keypairs, 'keypairs')
         LOG.debug("List count, %s Keypairs", len(keypairs))
         return keypairs
 
diff --git a/tempest/config.py b/tempest/config.py
index 9d7526a..b1f736c 100644
--- a/tempest/config.py
+++ b/tempest/config.py
@@ -688,7 +688,11 @@
                 default=['qcow2', 'raw', 'ami', 'ari', 'aki', 'vhd', 'vmdk',
                          'vdi', 'iso', 'vhdx'],
                 help="A list of image's disk formats "
-                     "users can specify.")
+                     "users can specify."),
+    cfg.StrOpt('images_manifest_file',
+               default=None,
+               help="A path to a manifest.yml generated using the "
+                    "os-test-images project"),
 ]
 
 image_feature_group = cfg.OptGroup(name='image-feature-enabled',
@@ -721,6 +725,9 @@
                 help=('Is show_multiple_locations enabled in glance. '
                       'Note that at least one http store must be enabled as '
                       'well, because we use that location scheme to test.')),
+    cfg.BoolOpt('image_conversion',
+                default=False,
+                help=('Is image_conversion enabled in glance.')),
 ]
 
 network_group = cfg.OptGroup(name='network',
diff --git a/tempest/tests/cmd/test_cleanup_services.py b/tempest/tests/cmd/test_cleanup_services.py
index 2557145..7f8db9f 100644
--- a/tempest/tests/cmd/test_cleanup_services.py
+++ b/tempest/tests/cmd/test_cleanup_services.py
@@ -610,21 +610,14 @@
         self._test_prefix_opt_precedence(delete_mock)
 
     def test_resource_list_opt_precedence(self):
-        delete_mock = [(self.filter_prefix, [], None),
+        delete_mock = [(self.filter_saved_state, [], None),
+                       (self.filter_resource_list, [], None),
+                       (self.filter_prefix, [], None),
                        (self.get_method, self.response, 200),
                        (self.validate_response, 'validate', None),
                        (self.delete_method, 'error', None),
                        (self.log_method, 'exception', None)]
-        serv = self._create_cmd_service(
-            self.service_class, is_resource_list=True)
-
-        _, fixtures = self.run_function_with_mocks(
-            serv.delete,
-            delete_mock
-        )
-
-        # Check that prefix was not used for filtering
-        fixtures[0].mock.assert_not_called()
+        self._test_resource_list_opt_precedence(delete_mock)
 
 
 class TestVolumeService(BaseCmdServiceTests):
diff --git a/zuul.d/base.yaml b/zuul.d/base.yaml
index 633f501..4de4111 100644
--- a/zuul.d/base.yaml
+++ b/zuul.d/base.yaml
@@ -12,6 +12,10 @@
     timeout: 7200
     roles: &base_roles
       - zuul: opendev.org/openstack/devstack
+    failure-output:
+      # This matches stestr/tempest output when a test fails
+      # {1} tempest.api.test_blah [5.743446s] ... FAILED
+      - '\{\d+\} (.*?) \[[\d\.]+s\] \.\.\. FAILED'
     vars: &base_vars
       devstack_localrc:
         IMAGE_URLS: http://download.cirros-cloud.net/0.6.2/cirros-0.6.2-x86_64-disk.img, http://download.cirros-cloud.net/0.6.1/cirros-0.6.1-x86_64-disk.img
@@ -60,6 +64,10 @@
     required-projects: *base_required-projects
     timeout: 7200
     roles: *base_roles
+    failure-output:
+      # This matches stestr/tempest output when a test fails
+      # {1} tempest.api.test_blah [5.743446s] ... FAILED
+      - '\{\d+\} (.*?) \[[\d\.]+s\] \.\.\. FAILED'
     vars: *base_vars
     run: playbooks/devstack-tempest-ipv6.yaml
     post-run: playbooks/post-tempest.yaml