Separate the extra tests to run in a separate job
Recently we are seeing a lot of job timeout(bug#2004780)
and we see many tests taking time and also number of tests
increasing over time. This is to prepare the list of extra tests
(here extra tests means the tests which are covered by the other
API or scenario tests) which we do not need to run in every
integrated jobs. Instead, we can run them in a separete job(s).
Currently I am adding admin (except keystone) and negative tests
in the 'extra tests' list but we can add more tests here which
we think are covered in some other tests.
As negative tests are important for interop, adding those extra
tests coverage for stable branch job also but running them in
periodic run only.
Related-Bug: #2004780
Change-Id: Id02221df0d6180519751c63e890851bd59fdafa0
diff --git a/zuul.d/integrated-gate.yaml b/zuul.d/integrated-gate.yaml
index a67f6b4..d88c988 100644
--- a/zuul.d/integrated-gate.yaml
+++ b/zuul.d/integrated-gate.yaml
@@ -60,6 +60,15 @@
c-bak: false
- job:
+ name: tempest-extra-tests
+ parent: devstack-tempest
+ description: |
+ This job runs the extra tests mentioned in
+ tools/tempest-extra-tests-list.txt.
+ vars:
+ tox_envlist: extra-tests
+
+- job:
name: tempest-full-py3
parent: devstack-tempest
# This job version is with swift enabled on py3
@@ -74,7 +83,7 @@
required-projects:
- openstack/horizon
vars:
- tox_envlist: full
+ tox_envlist: integrated-full
devstack_localrc:
USE_PYTHON3: true
FORCE_CONFIG_DRIVE: true
@@ -107,6 +116,7 @@
# Required until bug/1949606 is resolved when using libvirt and QEMU
# >=5.0.0 with a [libvirt]virt_type of qemu (TCG).
configure_swap_size: 4096
+ tox_envlist: full
- job:
name: tempest-integrated-networking
diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml
index 126119b..4281d6b 100644
--- a/zuul.d/project.yaml
+++ b/zuul.d/project.yaml
@@ -26,6 +26,8 @@
- ^.gitignore$
- ^.gitreview$
- ^.mailmap$
+ - tempest-extra-tests:
+ irrelevant-files: *tempest-irrelevant-files
- tempest-full-ubuntu-focal:
irrelevant-files: *tempest-irrelevant-files
- glance-multistore-cinder-import:
@@ -63,6 +65,7 @@
- ^tools/tempest-integrated-gate-placement-exclude-list.txt
- ^tools/tempest-integrated-gate-storage-blacklist.txt
- ^tools/tempest-integrated-gate-storage-exclude-list.txt
+ - ^tools/tempest-extra-tests-list.txt
- ^tools/verify-ipv6-only-deployments.sh
- ^tools/with_venv.sh
# tools/ is not here since this relies on a script in tools/.
@@ -86,6 +89,7 @@
- ^tools/tempest-integrated-gate-placement-exclude-list.txt
- ^tools/tempest-integrated-gate-storage-blacklist.txt
- ^tools/tempest-integrated-gate-storage-exclude-list.txt
+ - ^tools/tempest-extra-tests-list.txt
- ^tools/tempest-plugin-sanity.sh
- ^tools/with_venv.sh
- ^.coveragerc$
@@ -131,6 +135,8 @@
irrelevant-files: *tempest-irrelevant-files
- tempest-full-py3:
irrelevant-files: *tempest-irrelevant-files
+ - tempest-extra-tests:
+ irrelevant-files: *tempest-irrelevant-files
- grenade:
irrelevant-files: *tempest-irrelevant-files
- tempest-ipv6-only:
@@ -177,6 +183,9 @@
- tempest-slow-zed
- tempest-slow-yoga
- tempest-slow-xena
+ - tempest-full-zed-extra-tests
+ - tempest-full-yoga-extra-tests
+ - tempest-full-xena-extra-tests
periodic:
jobs:
- tempest-all
diff --git a/zuul.d/stable-jobs.yaml b/zuul.d/stable-jobs.yaml
index fb2300b..f70e79c 100644
--- a/zuul.d/stable-jobs.yaml
+++ b/zuul.d/stable-jobs.yaml
@@ -18,6 +18,24 @@
override-checkout: stable/xena
- job:
+ name: tempest-full-zed-extra-tests
+ parent: tempest-extra-tests
+ nodeset: openstack-single-node-focal
+ override-checkout: stable/zed
+
+- job:
+ name: tempest-full-yoga-extra-tests
+ parent: tempest-extra-tests
+ nodeset: openstack-single-node-focal
+ override-checkout: stable/yoga
+
+- job:
+ name: tempest-full-xena-extra-tests
+ parent: tempest-extra-tests
+ nodeset: openstack-single-node-focal
+ override-checkout: stable/xena
+
+- job:
name: tempest-slow-zed
parent: tempest-slow-py3
nodeset: openstack-two-node-focal
diff --git a/zuul.d/tempest-specific.yaml b/zuul.d/tempest-specific.yaml
index 684270a..972123e 100644
--- a/zuul.d/tempest-specific.yaml
+++ b/zuul.d/tempest-specific.yaml
@@ -30,6 +30,8 @@
- opendev.org/openstack/oslo.utils
- opendev.org/openstack/oslo.versionedobjects
- opendev.org/openstack/oslo.vmware
+ vars:
+ tox_envlist: full
- job:
name: tempest-full-parallel