Merge "Remove deprecated 'deactivate_image' feature flag setting"
diff --git a/.zuul.yaml b/.zuul.yaml
index 710b229..037e9a9 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -23,6 +23,9 @@
nodes:
- controller
- compute1
+ - name: subnode
+ nodes:
+ - compute1
- job:
name: devstack
@@ -39,9 +42,11 @@
- openstack/requirements
- openstack/swift
roles:
+ - zuul: openstack-infra/devstack-gate
- zuul: openstack-infra/openstack-zuul-jobs
timeout: 7200
vars:
+ test_matrix_configs: ['neutron', 'tlsproxy']
devstack_localrc:
DATABASE_PASSWORD: secretdatabase
RABBIT_PASSWORD: secretrabbit
@@ -57,6 +62,7 @@
FLOATING_HOST_MASK: 23
SWIFT_REPLICAS: 1
SWIFT_START_ALL_SERVICES: false
+ SWIFT_HASH: 1234123412341234
LOGFILE: /opt/stack/logs/devstacklog.txt
LOG_COLOR: false
VERBOSE: true
@@ -74,10 +80,85 @@
pre-run: playbooks/pre.yaml
run: playbooks/devstack.yaml
post-run: playbooks/post.yaml
+ irrelevant-files:
+ # Documentation related
+ - ^.*\.rst$
+ - ^api-ref/.*$
+ - ^doc/.*$
+ - ^releasenotes/.*$
+ # Translations
+ - ^.*/locale/.*po$
+- job:
+ name: devstack-multinode
+ parent: devstack
+ description: Base devstack multinode job
+ nodeset: openstack-two-node
+ # NOTE(andreaf) The multinode job is useful to see the setup of different
+ # services on different nodes, however the subnode configuration is not
+ # ready yet. Until then this job should stay non-voting.
+ voting: false
+
+- job:
+ name: devstack-tox-base
+ parent: devstack
+ description: |
+ Base job for devstack-based functional tests that use tox.
+
+ This job is not intended to be run directly. It's just here
+ for organizational purposes for devstack-tox-functional and
+ devstack-tox-functional-consumer.
+ post-run: playbooks/tox/post.yaml
+ vars:
+ tox_envlist: functional
+ tox_install_siblings: false
+
+- job:
+ name: devstack-tox-functional
+ parent: devstack-tox-base
+ description: |
+ Base job for devstack-based functional tests that use tox.
+
+ Runs devstack, then runs the tox ``functional`` environment,
+ then collects tox/testr build output like normal tox jobs.
+
+ Turns off tox sibling installation. Projects may be involved
+ in the devstack deployment and so may be in the required-projects
+ list, but may not want to test against master of the other
+ projects in their tox env. Child jobs can set tox_install_siblings
+ to True to re-enable sibling processing.
+ run: playbooks/tox/run-both.yaml
+
+- job:
+ name: devstack-tox-functional-consumer
+ parent: devstack
+ description: |
+ Base job for devstack-based functional tests for projects that
+ consume the devstack cloud.
+
+ This base job should only be used by projects that are not involved
+ in the devstack deployment step, but are instead projects that are using
+ devstack to get a cloud against which they can test things.
+
+ Runs devstack in pre-run, then runs the tox ``functional`` environment,
+ then collects tox/testr build output like normal tox jobs.
+
+ Turns off tox sibling installation. Projects may be involved
+ in the devstack deployment and so may be in the required-projects
+ list, but may not want to test against master of the other
+ projects in their tox env. Child jobs can set tox_install_siblings
+ to True to re-enable sibling processing.
+ pre-run:
+ - playbooks/devstack.yaml
+ - playbooks/tox/pre.yaml
+ run: playbooks/tox/run.yaml
- project:
name: openstack-dev/devstack
check:
jobs:
- devstack
+ - devstack-multinode
+ gate:
+ jobs:
+ - devstack
diff --git a/doc/requirements.txt b/doc/requirements.txt
new file mode 100644
index 0000000..e140bc0
--- /dev/null
+++ b/doc/requirements.txt
@@ -0,0 +1,10 @@
+pbr>=2.0.0,!=2.1.0
+
+Pygments
+docutils
+sphinx>=1.6.2
+openstackdocstheme>=1.11.0
+nwdiag
+blockdiag
+sphinxcontrib-blockdiag
+sphinxcontrib-nwdiag
diff --git a/doc/source/overview.rst b/doc/source/overview.rst
index c07a8e6..814a2b1 100644
--- a/doc/source/overview.rst
+++ b/doc/source/overview.rst
@@ -24,7 +24,7 @@
- Ubuntu: current LTS release plus current development release
- Fedora: current release plus previous release
-- RHEL/Centos: current major release
+- RHEL/CentOS: current major release
- Other OS platforms may continue to be included but the maintenance of
those platforms shall not be assumed simply due to their presence.
Having a listed point-of-contact for each additional OS will greatly
diff --git a/doc/source/plugin-registry.rst b/doc/source/plugin-registry.rst
index 43dd3c2..31812ae 100644
--- a/doc/source/plugin-registry.rst
+++ b/doc/source/plugin-registry.rst
@@ -26,7 +26,6 @@
====================================== ===
almanach `git://git.openstack.org/openstack/almanach <https://git.openstack.org/cgit/openstack/almanach>`__
aodh `git://git.openstack.org/openstack/aodh <https://git.openstack.org/cgit/openstack/aodh>`__
-app-catalog-ui `git://git.openstack.org/openstack/app-catalog-ui <https://git.openstack.org/cgit/openstack/app-catalog-ui>`__
astara `git://git.openstack.org/openstack/astara <https://git.openstack.org/cgit/openstack/astara>`__
barbican `git://git.openstack.org/openstack/barbican <https://git.openstack.org/cgit/openstack/barbican>`__
bilean `git://git.openstack.org/openstack/bilean <https://git.openstack.org/cgit/openstack/bilean>`__
@@ -127,6 +126,7 @@
networking-plumgrid `git://git.openstack.org/openstack/networking-plumgrid <https://git.openstack.org/cgit/openstack/networking-plumgrid>`__
networking-powervm `git://git.openstack.org/openstack/networking-powervm <https://git.openstack.org/cgit/openstack/networking-powervm>`__
networking-sfc `git://git.openstack.org/openstack/networking-sfc <https://git.openstack.org/cgit/openstack/networking-sfc>`__
+networking-spp `git://git.openstack.org/openstack/networking-spp <https://git.openstack.org/cgit/openstack/networking-spp>`__
networking-vpp `git://git.openstack.org/openstack/networking-vpp <https://git.openstack.org/cgit/openstack/networking-vpp>`__
networking-vsphere `git://git.openstack.org/openstack/networking-vsphere <https://git.openstack.org/cgit/openstack/networking-vsphere>`__
neutron `git://git.openstack.org/openstack/neutron <https://git.openstack.org/cgit/openstack/neutron>`__
@@ -135,6 +135,7 @@
neutron-fwaas-dashboard `git://git.openstack.org/openstack/neutron-fwaas-dashboard <https://git.openstack.org/cgit/openstack/neutron-fwaas-dashboard>`__
neutron-lbaas `git://git.openstack.org/openstack/neutron-lbaas <https://git.openstack.org/cgit/openstack/neutron-lbaas>`__
neutron-lbaas-dashboard `git://git.openstack.org/openstack/neutron-lbaas-dashboard <https://git.openstack.org/cgit/openstack/neutron-lbaas-dashboard>`__
+neutron-tempest-plugin `git://git.openstack.org/openstack/neutron-tempest-plugin <https://git.openstack.org/cgit/openstack/neutron-tempest-plugin>`__
neutron-vpnaas `git://git.openstack.org/openstack/neutron-vpnaas <https://git.openstack.org/cgit/openstack/neutron-vpnaas>`__
neutron-vpnaas-dashboard `git://git.openstack.org/openstack/neutron-vpnaas-dashboard <https://git.openstack.org/cgit/openstack/neutron-vpnaas-dashboard>`__
nova-dpm `git://git.openstack.org/openstack/nova-dpm <https://git.openstack.org/cgit/openstack/nova-dpm>`__
@@ -166,6 +167,7 @@
tacker `git://git.openstack.org/openstack/tacker <https://git.openstack.org/cgit/openstack/tacker>`__
tap-as-a-service `git://git.openstack.org/openstack/tap-as-a-service <https://git.openstack.org/cgit/openstack/tap-as-a-service>`__
tap-as-a-service-dashboard `git://git.openstack.org/openstack/tap-as-a-service-dashboard <https://git.openstack.org/cgit/openstack/tap-as-a-service-dashboard>`__
+telemetry-tempest-plugin `git://git.openstack.org/openstack/telemetry-tempest-plugin <https://git.openstack.org/cgit/openstack/telemetry-tempest-plugin>`__
tricircle `git://git.openstack.org/openstack/tricircle <https://git.openstack.org/cgit/openstack/tricircle>`__
trio2o `git://git.openstack.org/openstack/trio2o <https://git.openstack.org/cgit/openstack/trio2o>`__
trove `git://git.openstack.org/openstack/trove <https://git.openstack.org/cgit/openstack/trove>`__
@@ -173,6 +175,7 @@
valet `git://git.openstack.org/openstack/valet <https://git.openstack.org/cgit/openstack/valet>`__
vitrage `git://git.openstack.org/openstack/vitrage <https://git.openstack.org/cgit/openstack/vitrage>`__
vitrage-dashboard `git://git.openstack.org/openstack/vitrage-dashboard <https://git.openstack.org/cgit/openstack/vitrage-dashboard>`__
+vitrage-tempest-plugin `git://git.openstack.org/openstack/vitrage-tempest-plugin <https://git.openstack.org/cgit/openstack/vitrage-tempest-plugin>`__
vmware-nsx `git://git.openstack.org/openstack/vmware-nsx <https://git.openstack.org/cgit/openstack/vmware-nsx>`__
vmware-vspc `git://git.openstack.org/openstack/vmware-vspc <https://git.openstack.org/cgit/openstack/vmware-vspc>`__
watcher `git://git.openstack.org/openstack/watcher <https://git.openstack.org/cgit/openstack/watcher>`__
diff --git a/files/rpms/cinder b/files/rpms/cinder
index 3bc4e7a..e6addc6 100644
--- a/files/rpms/cinder
+++ b/files/rpms/cinder
@@ -1,5 +1,5 @@
iscsi-initiator-utils
lvm2
qemu-img
-scsi-target-utils # not:rhel7,f24,f25,f26 NOPRIME
-targetcli # dist:rhel7,f24,f25,f26 NOPRIME
+scsi-target-utils # not:rhel7,f25,f26,f27 NOPRIME
+targetcli # dist:rhel7,f25,f26,f27 NOPRIME
diff --git a/files/rpms/general b/files/rpms/general
index f3f8708..878b935 100644
--- a/files/rpms/general
+++ b/files/rpms/general
@@ -9,9 +9,9 @@
graphviz # needed only for docs
httpd
httpd-devel
-iptables-services # NOPRIME f23,f24,f25,f26
+iptables-services # NOPRIME f25,f26,f27
java-1.7.0-openjdk-headless # NOPRIME rhel7
-java-1.8.0-openjdk-headless # NOPRIME f23,f24,f25,f26
+java-1.8.0-openjdk-headless # NOPRIME f25,f26,f27
libffi-devel
libjpeg-turbo-devel # Pillow 3.0.0
libxml2-devel # lxml
diff --git a/files/rpms/nova b/files/rpms/nova
index 64ed480..9fb7282 100644
--- a/files/rpms/nova
+++ b/files/rpms/nova
@@ -7,7 +7,7 @@
genisoimage # required for config_drive
iptables
iputils
-kernel-modules # dist:f23,f24,f25,f26
+kernel-modules # dist:f25,f26,f27
kpartx
libxml2-python
m2crypto
diff --git a/files/rpms/swift b/files/rpms/swift
index 2e09cec..be0db14 100644
--- a/files/rpms/swift
+++ b/files/rpms/swift
@@ -2,7 +2,7 @@
liberasurecode-devel
memcached
pyxattr
-rsync-daemon # dist:f23,f24,f25,f26
+rsync-daemon # dist:f25,f26,f27
sqlite
xfsprogs
xinetd
diff --git a/inc/python b/inc/python
index 2e4eff0..e074ea4 100644
--- a/inc/python
+++ b/inc/python
@@ -49,7 +49,11 @@
fi
$xtrace
- if is_fedora || is_suse; then
+ if python3_enabled && [ "$os_VENDOR" = "Fedora" -a $os_RELEASE -gt 26 ]; then
+ # Default Python 3 install prefix changed to /usr/local in Fedora 27:
+ # https://fedoraproject.org/wiki/Changes/Making_sudo_pip_safe
+ echo "/usr/local/bin"
+ elif is_fedora || is_suse; then
echo "/usr/bin"
else
echo "/usr/local/bin"
diff --git a/lib/apache b/lib/apache
index 3af3411..84cec73 100644
--- a/lib/apache
+++ b/lib/apache
@@ -133,8 +133,9 @@
sudo rm -f /etc/httpd/conf.d/000-*
install_package httpd mod_wsgi
# For consistency with Ubuntu, switch to the worker mpm, as
- # the default is prefork
+ # the default is event
sudo sed -i '/mod_mpm_prefork.so/s/^/#/g' /etc/httpd/conf.modules.d/00-mpm.conf
+ sudo sed -i '/mod_mpm_event.so/s/^/#/g' /etc/httpd/conf.modules.d/00-mpm.conf
sudo sed -i '/mod_mpm_worker.so/s/^#//g' /etc/httpd/conf.modules.d/00-mpm.conf
elif is_suse; then
install_package apache2 apache2-mod_wsgi
diff --git a/lib/cinder b/lib/cinder
index a966a4b..655908c 100644
--- a/lib/cinder
+++ b/lib/cinder
@@ -206,8 +206,6 @@
function configure_cinder {
sudo install -d -o $STACK_USER -m 755 $CINDER_CONF_DIR
- cp -p $CINDER_DIR/etc/cinder/policy.json $CINDER_CONF_DIR
-
rm -f $CINDER_CONF
configure_rootwrap cinder
@@ -430,7 +428,7 @@
setup_develop $CINDER_DIR
if [[ "$CINDER_ISCSI_HELPER" == "tgtadm" ]]; then
install_package tgt
- elif [[ "$CINDER_ISCI_HELPER" == "lioadm" ]]; then
+ elif [[ "$CINDER_ISCSI_HELPER" == "lioadm" ]]; then
install_package targetcli
fi
}
diff --git a/lib/glance b/lib/glance
index ad286ba..95d2450 100644
--- a/lib/glance
+++ b/lib/glance
@@ -56,6 +56,7 @@
GLANCE_POLICY_JSON=$GLANCE_CONF_DIR/policy.json
GLANCE_SCHEMA_JSON=$GLANCE_CONF_DIR/schema-image.json
GLANCE_SWIFT_STORE_CONF=$GLANCE_CONF_DIR/glance-swift-store.conf
+GLANCE_IMAGE_IMPORT_CONF=$GLANCE_CONF_DIR/glance-image-import.conf
GLANCE_V1_ENABLED=${GLANCE_V1_ENABLED:-False}
if is_service_enabled tls-proxy; then
@@ -232,6 +233,11 @@
# Store specific confs
iniset $GLANCE_CACHE_CONF glance_store filesystem_store_datadir $GLANCE_IMAGE_DIR/
+ # Set default configuration options for the glance-image-import
+ iniset $GLANCE_IMAGE_IMPORT_CONF image_import_opts image_import_plugins []
+ iniset $GLANCE_IMAGE_IMPORT_CONF inject_metadata_properties ignore_user_roles admin
+ iniset $GLANCE_IMAGE_IMPORT_CONF inject_metadata_properties inject
+
cp -p $GLANCE_DIR/etc/policy.json $GLANCE_POLICY_JSON
cp -p $GLANCE_DIR/etc/schema-image.json $GLANCE_SCHEMA_JSON
diff --git a/lib/neutron_plugins/services/l3 b/lib/neutron_plugins/services/l3
index 98315b7..41a467d 100644
--- a/lib/neutron_plugins/services/l3
+++ b/lib/neutron_plugins/services/l3
@@ -188,7 +188,7 @@
if [ -z $SUBNETPOOL_V4_ID ]; then
fixed_range_v4=$FIXED_RANGE
fi
- SUBNET_ID=$(openstack --os-cloud devstack-admin --os-region "$REGION_NAME" subnet create --project $project_id --ip-version 4 ${ALLOCATION_POOL:+--allocation-pool $ALLOCATION_POOL} $PROVIDER_SUBNET_NAME --gateway $NETWORK_GATEWAY ${SUBNETPOOL_V4_ID:+--subnet-pool $SUBNETPOOL_V4_ID} --network $NET_ID --subnet-range $fixed_range_v4 | grep ' id ' | get_field 2)
+ SUBNET_ID=$(openstack --os-cloud devstack-admin --os-region "$REGION_NAME" subnet create --project $project_id --ip-version 4 ${ALLOCATION_POOL:+--allocation-pool $ALLOCATION_POOL} $PROVIDER_SUBNET_NAME --gateway $NETWORK_GATEWAY ${SUBNETPOOL_V4_ID:+--subnet-pool $SUBNETPOOL_V4_ID} --network $NET_ID ${fixed_range_v4:+--subnet-range $fixed_range_v4} | grep ' id ' | get_field 2)
die_if_not_set $LINENO SUBNET_ID "Failure creating SUBNET_ID for $PROVIDER_SUBNET_NAME $project_id"
fi
@@ -198,7 +198,7 @@
if [ -z $SUBNETPOOL_V6_ID ]; then
fixed_range_v6=$IPV6_PROVIDER_FIXED_RANGE
fi
- IPV6_SUBNET_ID=$(openstack --os-cloud devstack-admin --os-region "$REGION_NAME" subnet create --project $project_id --ip-version 6 --gateway $IPV6_PROVIDER_NETWORK_GATEWAY $IPV6_PROVIDER_SUBNET_NAME ${SUBNETPOOL_V6_ID:+--subnet-pool $SUBNETPOOL_V6_ID} --network $NET_ID --subnet-range $fixed_range_v6 | grep ' id ' | get_field 2)
+ IPV6_SUBNET_ID=$(openstack --os-cloud devstack-admin --os-region "$REGION_NAME" subnet create --project $project_id --ip-version 6 --gateway $IPV6_PROVIDER_NETWORK_GATEWAY $IPV6_PROVIDER_SUBNET_NAME ${SUBNETPOOL_V6_ID:+--subnet-pool $SUBNETPOOL_V6_ID} --network $NET_ID ${fixed_range_v6:+--subnet-range $fixed_range_v6} | grep ' id ' | get_field 2)
die_if_not_set $LINENO IPV6_SUBNET_ID "Failure creating IPV6_SUBNET_ID for $IPV6_PROVIDER_SUBNET_NAME $project_id"
fi
diff --git a/lib/nova b/lib/nova
index 50deeeb..c40c43a 100644
--- a/lib/nova
+++ b/lib/nova
@@ -600,6 +600,12 @@
else
rpc_backend_add_vhost $vhost
iniset_rpc_backend nova $conf DEFAULT $vhost
+ # When running in superconductor mode, the cell conductor
+ # must be configured to talk to the placement service for
+ # reschedules to work.
+ if is_service_enabled placement placement-client; then
+ configure_placement_nova_compute $conf
+ fi
fi
# Format logging
setup_logging $conf
diff --git a/lib/placement b/lib/placement
index 1875857..1d68f8a 100644
--- a/lib/placement
+++ b/lib/placement
@@ -103,14 +103,16 @@
}
function configure_placement_nova_compute {
- iniset $NOVA_CONF placement auth_type "password"
- iniset $NOVA_CONF placement auth_url "$KEYSTONE_SERVICE_URI"
- iniset $NOVA_CONF placement username placement
- iniset $NOVA_CONF placement password "$SERVICE_PASSWORD"
- iniset $NOVA_CONF placement user_domain_name "$SERVICE_DOMAIN_NAME"
- iniset $NOVA_CONF placement project_name "$SERVICE_TENANT_NAME"
- iniset $NOVA_CONF placement project_domain_name "$SERVICE_DOMAIN_NAME"
- iniset $NOVA_CONF placement os_region_name "$REGION_NAME"
+ # Use the provided config file path or default to $NOVA_CONF.
+ local conf=${1:-$NOVA_CONF}
+ iniset $conf placement auth_type "password"
+ iniset $conf placement auth_url "$KEYSTONE_SERVICE_URI"
+ iniset $conf placement username placement
+ iniset $conf placement password "$SERVICE_PASSWORD"
+ iniset $conf placement user_domain_name "$SERVICE_DOMAIN_NAME"
+ iniset $conf placement project_name "$SERVICE_TENANT_NAME"
+ iniset $conf placement project_domain_name "$SERVICE_DOMAIN_NAME"
+ iniset $conf placement os_region_name "$REGION_NAME"
# TODO(cdent): auth_strategy, which is common to see in these
# blocks is not currently used here. For the time being the
# placement api uses the auth_strategy configuration setting
diff --git a/playbooks/post.yaml b/playbooks/post.yaml
index 6f5126f..0c5e83b 100644
--- a/playbooks/post.yaml
+++ b/playbooks/post.yaml
@@ -1,4 +1,72 @@
- hosts: all
+ become: True
+ vars:
+ devstack_log_dir: "{{ devstack_base_dir|default('/opt/stack') }}/logs/"
+ devstack_conf_dir: "{{ devstack_base_dir|default('/opt/stack') }}/devstack/"
+ devstack_full_log: "{{ devstack_early_log|default('/opt/stack/logs/devstack-early.txt') }}"
+ tasks:
+ # NOTE(andreaf) If the tempest service is enabled, a tempest.log is
+ # generated as part of lib/tempest, as a result of verify_tempest_config
+ - name: Check if a tempest log exits
+ stat:
+ path: "{{ devstack_conf_dir }}/tempest.log"
+ register: tempest_log
+ - name: Link post-devstack tempest.log
+ file:
+ src: "{{ devstack_conf_dir }}/tempest.log"
+ dest: "{{ stage_dir }}/verify_tempest_conf.log"
+ state: hard
+ when: tempest_log.stat.exists
roles:
- export-devstack-journal
- - fetch-devstack-log-dir
+ - apache-logs-conf
+ - devstack-project-conf
+ # capture-system-logs should be the last role before stage-output
+ - capture-system-logs
+ - role: stage-output
+ zuul_copy_output:
+ { '{{ devstack_conf_dir }}/local.conf': 'logs',
+ '{{ devstack_conf_dir }}/localrc': 'logs',
+ '{{ devstack_conf_dir }}/.stackenv': 'logs' ,
+ '{{ devstack_log_dir }}/dstat-csv.log': 'logs',
+ '{{ devstack_log_dir }}/devstacklog.txt': 'logs',
+ '{{ devstack_log_dir }}/devstacklog.txt.summary': 'logs',
+ '{{ devstack_full_log}}': 'logs',
+ '{{ stage_dir }}/verify_tempest_conf.log': 'logs',
+ '{{ stage_dir }}/apache': 'logs',
+ '{{ stage_dir }}/apache_config': 'logs',
+ '{{ stage_dir }}/etc': 'logs',
+ '/var/log/rabbitmq': 'logs',
+ '/var/log/postgresql': 'logs',
+ '/var/log/mysql.err': 'logs',
+ '/var/log/mysql.log': 'logs',
+ '/var/log/libvirt': 'logs',
+ '/etc/sudoers': 'logs',
+ '/etc/sudoers.d': 'logs',
+ '{{ stage_dir }}/iptables.txt': 'logs',
+ '{{ stage_dir }}/df.txt': 'logs',
+ '{{ stage_dir }}/pip2-freeze.txt': 'logs',
+ '{{ stage_dir }}/pip3-freeze.txt': 'logs',
+ '{{ stage_dir }}/dpkg-l.txt': 'logs',
+ '{{ stage_dir }}/rpm-qa.txt': 'logs',
+ '{{ stage_dir }}/core': 'logs',
+ '{{ stage_dir }}/listen53.txt': 'logs',
+ '{{ stage_dir }}/deprecations.log': 'logs',
+ '/var/log/ceph': 'logs',
+ '/var/log/openvswitch': 'logs',
+ '/var/log/glusterfs': 'logs',
+ '/etc/glusterfs/glusterd.vol': 'logs',
+ '/etc/resolv.conf': 'logs',
+ '/var/log/unbound.log': 'logs' }
+ extensions_to_txt:
+ - conf
+ - log
+ - localrc
+ - stackenv
+ - summary
+ # NOTE(andreaf) We need fetch-devstack-log-dir only as long as the base job
+ # starts pulling logs for us from {{ ansible_user_dir }}/logs.
+ # Meanwhile we already store things in ansible_user_dir and use
+ # fetch-devstack-log-dir setting devstack_base_dir
+ - role: fetch-devstack-log-dir
+ devstack_base_dir: "{{ ansible_user_dir }}"
diff --git a/playbooks/pre.yaml b/playbooks/pre.yaml
index 4d07960..d61fd45 100644
--- a/playbooks/pre.yaml
+++ b/playbooks/pre.yaml
@@ -1,3 +1,13 @@
+- hosts: controller
+ roles:
+ - role: test-matrix
+ test_matrix_role: primary
+
+- hosts: subnode
+ roles:
+ - role: test-matrix
+ test_matrix_role: subnode
+
- hosts: all
roles:
- configure-swap
@@ -8,7 +18,7 @@
- setup-devstack-cache
- start-fresh-logging
- write-devstack-local-conf
- # TODO(jeblair): remove when configure-mirrors is fixed
+ # TODO(jeblair): remove when configure-mirrors is fixed
tasks:
- name: Hack mirror_info
shell:
diff --git a/playbooks/tox/post.yaml b/playbooks/tox/post.yaml
new file mode 100644
index 0000000..7f0cb19
--- /dev/null
+++ b/playbooks/tox/post.yaml
@@ -0,0 +1,4 @@
+- hosts: all
+ roles:
+ - fetch-tox-output
+ - fetch-subunit-output
diff --git a/playbooks/tox/pre.yaml b/playbooks/tox/pre.yaml
new file mode 100644
index 0000000..d7e4670
--- /dev/null
+++ b/playbooks/tox/pre.yaml
@@ -0,0 +1,8 @@
+- hosts: all
+ roles:
+ # Run bindep and test-setup after devstack so that they won't interfere
+ - role: bindep
+ bindep_profile: test
+ bindep_dir: "{{ zuul_work_dir }}"
+ - test-setup
+ - ensure-tox
diff --git a/playbooks/tox/run-both.yaml b/playbooks/tox/run-both.yaml
new file mode 100644
index 0000000..e85c2ee
--- /dev/null
+++ b/playbooks/tox/run-both.yaml
@@ -0,0 +1,10 @@
+- hosts: all
+ roles:
+ - run-devstack
+ # Run bindep and test-setup after devstack so that they won't interfere
+ - role: bindep
+ bindep_profile: test
+ bindep_dir: "{{ zuul_work_dir }}"
+ - test-setup
+ - ensure-tox
+ - tox
diff --git a/playbooks/tox/run.yaml b/playbooks/tox/run.yaml
new file mode 100644
index 0000000..22f8209
--- /dev/null
+++ b/playbooks/tox/run.yaml
@@ -0,0 +1,3 @@
+- hosts: all
+ roles:
+ - tox
diff --git a/roles/apache-logs-conf/README.rst b/roles/apache-logs-conf/README.rst
new file mode 100644
index 0000000..eccee40
--- /dev/null
+++ b/roles/apache-logs-conf/README.rst
@@ -0,0 +1,12 @@
+Prepare apache configs and logs for staging
+
+Make sure apache config files and log files are available in a linux flavor
+independent location. Note that this relies on hard links, to the staging
+directory must be in the same partition where the logs and configs are.
+
+**Role Variables**
+
+.. zuul:rolevar:: stage_dir
+ :default: {{ ansible_user_dir }}
+
+ The base stage directory.
diff --git a/roles/apache-logs-conf/defaults/main.yaml b/roles/apache-logs-conf/defaults/main.yaml
new file mode 100644
index 0000000..1fb04fe
--- /dev/null
+++ b/roles/apache-logs-conf/defaults/main.yaml
@@ -0,0 +1,2 @@
+devstack_base_dir: /opt/stack
+stage_dir: "{{ ansible_user_dir }}"
diff --git a/roles/apache-logs-conf/tasks/main.yaml b/roles/apache-logs-conf/tasks/main.yaml
new file mode 100644
index 0000000..7fd490e
--- /dev/null
+++ b/roles/apache-logs-conf/tasks/main.yaml
@@ -0,0 +1,80 @@
+- name: Ensure {{ stage_dir }}/apache exists
+ file:
+ path: "{{ stage_dir }}/apache"
+ state: directory
+
+- name: Link apache logs on Debian/SuSE
+ block:
+ - name: Find logs
+ find:
+ path: "/var/log/apache2"
+ file_type: any
+ register: debian_suse_apache_logs
+ - name: Dereference files
+ stat:
+ path: "{{ item.path }}"
+ with_items: "{{ debian_suse_apache_logs.files }}"
+ register: debian_suse_apache_deref_logs
+ - name: Create hard links
+ file:
+ src: "{{ item.stat.lnk_source | default(item.stat.path) }}"
+ dest: "{{ stage_dir }}/apache/{{ item.stat.path | basename }}"
+ state: hard
+ with_items: "{{ debian_suse_apache_deref_logs.results }}"
+ when:
+ - item.stat.isreg or item.stat.islnk
+ when: ansible_os_family in ('Debian', 'Suse')
+
+- name: Link apache logs on RedHat
+ block:
+ - name: Find logs
+ find:
+ path: "/var/log/httpd"
+ file_type: any
+ register: redhat_apache_logs
+ - name: Dereference files
+ stat:
+ path: "{{ item.path }}"
+ with_items: "{{ redhat_apache_logs.files }}"
+ register: redhat_apache_deref_logs
+ - name: Create hard links
+ file:
+ src: "{{ item.stat.lnk_source | default(item.stat.path) }}"
+ dest: "{{ stage_dir }}/apache/{{ item.stat.path | basename }}"
+ state: hard
+ with_items: "{{ redhat_apache_deref_logs.results }}"
+ when:
+ - item.stat.isreg or item.stat.islnk
+ when: ansible_os_family == 'Redhat'
+
+- name: Ensure {{ stage_dir }}/apache_config apache_config exists
+ file:
+ path: "{{ stage_dir }}/apache_config"
+ state: directory
+
+- name: Define config paths
+ set_fact:
+ apache_config_paths:
+ 'Debian': '/etc/apache2/sites-enabled/'
+ 'Suse': '/etc/apache2/conf.d/'
+ 'Redhat': '/etc/httpd/conf.d/'
+
+- name: Discover configurations
+ find:
+ path: "{{ apache_config_paths[ansible_os_family] }}"
+ file_type: any
+ register: apache_configs
+
+- name: Dereference configurations
+ stat:
+ path: "{{ item.path }}"
+ with_items: "{{ apache_configs.files }}"
+ register: apache_configs_deref
+
+- name: Link configurations
+ file:
+ src: "{{ item.stat.lnk_source | default(item.stat.path) }}"
+ dest: "{{ stage_dir }}/apache_config/{{ item.stat.path | basename }}"
+ state: hard
+ with_items: "{{ apache_configs_deref.results }}"
+ when: item.stat.isreg or item.stat.islnk
diff --git a/roles/capture-system-logs/README.rst b/roles/capture-system-logs/README.rst
new file mode 100644
index 0000000..c284124
--- /dev/null
+++ b/roles/capture-system-logs/README.rst
@@ -0,0 +1,20 @@
+Stage a number of system type logs
+
+Stage a number of different logs / reports:
+- snapshot of iptables
+- disk space available
+- pip[2|3] freeze
+- installed packages (dpkg/rpm)
+- ceph, openswitch, gluster
+- coredumps
+- dns resolver
+- listen53
+- unbound.log
+- deprecation messages
+
+**Role Variables**
+
+.. zuul:rolevar:: stage_dir
+ :default: {{ ansible_user_dir }}
+
+ The base stage directory.
diff --git a/roles/capture-system-logs/defaults/main.yaml b/roles/capture-system-logs/defaults/main.yaml
new file mode 100644
index 0000000..fea05c8
--- /dev/null
+++ b/roles/capture-system-logs/defaults/main.yaml
@@ -0,0 +1 @@
+devstack_base_dir: /opt/stack
diff --git a/roles/capture-system-logs/tasks/main.yaml b/roles/capture-system-logs/tasks/main.yaml
new file mode 100644
index 0000000..cd8f4f0
--- /dev/null
+++ b/roles/capture-system-logs/tasks/main.yaml
@@ -0,0 +1,38 @@
+# TODO(andreaf) Make this into proper Ansible
+- name: Stage various logs and reports
+ shell:
+ cmd: |
+ sudo iptables-save > {{ stage_dir }}/iptables.txt
+ df -h > {{ stage_dir }}/df.txt
+
+ for py_ver in 2 3; do
+ if [[ `which python${py_ver}` ]]; then
+ python${py_ver} -m pip freeze > {{ stage_dir }}/pip${py_ver}-freeze.txt
+ fi
+ done
+
+ if [ `command -v dpkg` ]; then
+ dpkg -l> {{ stage_dir }}/dpkg-l.txt
+ fi
+ if [ `command -v rpm` ]; then
+ rpm -qa | sort > {{ stage_dir }}/rpm-qa.txt
+ fi
+
+ # gzip and save any coredumps in /var/core
+ if [ -d /var/core ]; then
+ sudo gzip -r /var/core
+ sudo cp -r /var/core {{ stage_dir }}/
+ fi
+
+ sudo ss -lntup | grep ':53' > {{ stage_dir }}/listen53.txt
+
+ # NOTE(andreaf) Service logs are already in logs/ thanks for the
+ # export-devstack-journal log. Apache logs are under apache/ thans to the
+ # apache-logs-conf role.
+ grep -i deprecat {{ stage_dir }}/logs/*.txt {{ stage_dir }}/apache/*.log | \
+ sed -r 's/[0-9]{1,2}\:[0-9]{1,2}\:[0-9]{1,2}\.[0-9]{1,3}/ /g' | \
+ sed -r 's/[0-9]{1,2}\:[0-9]{1,2}\:[0-9]{1,2}/ /g' | \
+ sed -r 's/[0-9]{1,4}-[0-9]{1,2}-[0-9]{1,4}/ /g' |
+ sed -r 's/\[.*\]/ /g' | \
+ sed -r 's/\s[0-9]+\s/ /g' | \
+ awk '{if ($0 in seen) {seen[$0]++} else {out[++n]=$0;seen[$0]=1}} END { for (i=1; i<=n; i++) print seen[out[i]]" :: " out[i] }' > {{ stage_dir }}/deprecations.log
diff --git a/roles/devstack-project-conf/README.rst b/roles/devstack-project-conf/README.rst
new file mode 100644
index 0000000..3f2d4c9
--- /dev/null
+++ b/roles/devstack-project-conf/README.rst
@@ -0,0 +1,11 @@
+Prepare OpenStack project configurations for staging
+
+Prepare all relevant config files for staging.
+This is helpful to avoid staging the entire /etc.
+
+**Role Variables**
+
+.. zuul:rolevar:: stage_dir
+ :default: {{ ansible_user_dir }}
+
+ The base stage directory.
diff --git a/roles/devstack-project-conf/defaults/main.yaml b/roles/devstack-project-conf/defaults/main.yaml
new file mode 100644
index 0000000..f8fb8de
--- /dev/null
+++ b/roles/devstack-project-conf/defaults/main.yaml
@@ -0,0 +1 @@
+stage_dir: "{{ ansible_user_dir }}"
diff --git a/roles/devstack-project-conf/tasks/main.yaml b/roles/devstack-project-conf/tasks/main.yaml
new file mode 100644
index 0000000..9c6e06b
--- /dev/null
+++ b/roles/devstack-project-conf/tasks/main.yaml
@@ -0,0 +1,24 @@
+- name: Ensure {{ stage_dir }}/etc exists
+ file:
+ path: "{{ stage_dir }}/etc"
+ state: directory
+
+- name: Check which projects have a config folder
+ stat:
+ path: "/etc/{{ item.value.short_name }}"
+ with_dict: "{{ zuul.projects }}"
+ register: project_configs
+
+- name: Copy configuration files
+ command: cp -pRL {{ item.stat.path }} {{ stage_dir }}/etc/{{ item.item.value.short_name }}
+ when: item.stat.exists
+ with_items: "{{ project_configs.results }}"
+
+- name: Check if openstack has a config folder
+ stat:
+ path: "/etc/openstack"
+ register: openstack_configs
+
+- name: Copy configuration files
+ command: cp -pRL /etc/openstack {{ stage_dir }}/etc/
+ when: openstack_configs.stat.exists
diff --git a/roles/export-devstack-journal/README.rst b/roles/export-devstack-journal/README.rst
index 5f00592..a34e070 100644
--- a/roles/export-devstack-journal/README.rst
+++ b/roles/export-devstack-journal/README.rst
@@ -5,11 +5,17 @@
kernal and sudo messages.
Writes the output to the ``logs/`` subdirectory of
-``devstack_base_dir``.
+``stage_dir``.
**Role Variables**
.. zuul:rolevar:: devstack_base_dir
:default: /opt/stack
- The devstack base directory.
+ The devstack base directory. This is used to obtain the
+ ``log-start-timestamp.txt``, used to filter the systemd journal.
+
+.. zuul:rolevar:: stage_dir
+ :default: {{ ansible_user_dir }}
+
+ The base stage directory.
diff --git a/roles/export-devstack-journal/defaults/main.yaml b/roles/export-devstack-journal/defaults/main.yaml
index fea05c8..1fb04fe 100644
--- a/roles/export-devstack-journal/defaults/main.yaml
+++ b/roles/export-devstack-journal/defaults/main.yaml
@@ -1 +1,2 @@
devstack_base_dir: /opt/stack
+stage_dir: "{{ ansible_user_dir }}"
diff --git a/roles/export-devstack-journal/tasks/main.yaml b/roles/export-devstack-journal/tasks/main.yaml
index b9af02a..6e760c1 100644
--- a/roles/export-devstack-journal/tasks/main.yaml
+++ b/roles/export-devstack-journal/tasks/main.yaml
@@ -1,3 +1,11 @@
+# NOTE(andreaf) This bypasses the stage-output role
+- name: Ensure {{ stage_dir }}/logs exists
+ become: true
+ file:
+ path: "{{ stage_dir }}/logs"
+ state: directory
+ owner: "{{ ansible_user }}"
+
# TODO: convert this to ansible
- name: Export journal files
become: true
@@ -7,7 +15,7 @@
name=""
for u in `systemctl list-unit-files | grep devstack | awk '{print $1}'`; do
name=$(echo $u | sed 's/devstack@/screen-/' | sed 's/\.service//')
- journalctl -o short-precise --unit $u | tee {{ devstack_base_dir }}/logs/$name.txt > /dev/null
+ journalctl -o short-precise --unit $u | gzip - > {{ stage_dir }}/logs/$name.txt.gz
done
# Export the journal in export format to make it downloadable
@@ -16,7 +24,7 @@
# debugging much easier. We don't do the native conversion here as
# some distros do not package that tooling.
journalctl -u 'devstack@*' -o export | \
- xz --threads=0 - > {{ devstack_base_dir }}/logs/devstack.journal.xz
+ xz --threads=0 - > {{ stage_dir }}/logs/devstack.journal.xz
# The journal contains everything running under systemd, we'll
# build an old school version of the syslog with just the
@@ -26,4 +34,4 @@
-t sudo \
--no-pager \
--since="$(cat {{ devstack_base_dir }}/log-start-timestamp.txt)" \
- | tee {{ devstack_base_dir }}/logs/syslog.txt > /dev/null
+ | gzip - > {{ stage_dir }}/logs/syslog.txt.gz
diff --git a/roles/run-devstack/README.rst b/roles/run-devstack/README.rst
index d77eb15..e53f060 100644
--- a/roles/run-devstack/README.rst
+++ b/roles/run-devstack/README.rst
@@ -6,3 +6,9 @@
:default: /opt/stack
The devstack base directory.
+
+.. zuul:rolevar:: devstack_early_log
+ :default: /opt/stack/log/devstack-early.txt
+
+ The full devstack log that includes the whatever stack.sh logs before
+ the LOGFILE variable in local.conf is honoured.
diff --git a/roles/run-devstack/defaults/main.yaml b/roles/run-devstack/defaults/main.yaml
index fea05c8..dc4528f 100644
--- a/roles/run-devstack/defaults/main.yaml
+++ b/roles/run-devstack/defaults/main.yaml
@@ -1 +1,2 @@
devstack_base_dir: /opt/stack
+devstack_early_log: /opt/stack/logs/devstack-early.txt
diff --git a/roles/run-devstack/tasks/main.yaml b/roles/run-devstack/tasks/main.yaml
index bafebaf..f532129 100644
--- a/roles/run-devstack/tasks/main.yaml
+++ b/roles/run-devstack/tasks/main.yaml
@@ -1,5 +1,5 @@
- name: Run devstack
- command: ./stack.sh
+ shell: ./stack.sh 2>&1 {{ devstack_early_log }}
args:
chdir: "{{devstack_base_dir}}/devstack"
become: true
diff --git a/roles/write-devstack-local-conf/README.rst b/roles/write-devstack-local-conf/README.rst
index e30dfa1..73f9f0d 100644
--- a/roles/write-devstack-local-conf/README.rst
+++ b/roles/write-devstack-local-conf/README.rst
@@ -47,13 +47,27 @@
This is a dictionary of key-value pairs which comprise
this section of the INI file.
+.. zuul:rolevar:: devstack_base_services
+ :type: list
+ :default: {{ base_services | default(omit) }}
+
+ A list of base services which are enabled. Services can be added or removed
+ from this list via the ``devstack_services`` variable. This is ignored if
+ ``base`` is set to ``False`` in ``devstack_services``.
+
.. zuul:rolevar:: devstack_services
:type: dict
A dictionary mapping service names to boolean values. If the
boolean value is ``false``, a ``disable_service`` line will be
emitted for the service name. If it is ``true``, then
- ``enable_service`` will be emitted. All other values are ignored.
+ ``enable_service`` will be emitted. All other values are ignored.
+
+ The special key ``base`` can be used to enable or disable the base set of
+ services enabled by default. If ``base`` is found, it will processed before
+ all other keys. If its value is ``False`` a ``disable_all_services`` will be
+ emitted; if its value is ``True`` services from ``devstack_base_services``
+ will be emitted via ``ENABLED_SERVICES``.
.. zuul:rolevar:: devstack_plugins
:type: dict
diff --git a/roles/write-devstack-local-conf/defaults/main.yaml b/roles/write-devstack-local-conf/defaults/main.yaml
index 491fa0f..7bc1dec 100644
--- a/roles/write-devstack-local-conf/defaults/main.yaml
+++ b/roles/write-devstack-local-conf/defaults/main.yaml
@@ -1,2 +1,3 @@
devstack_base_dir: /opt/stack
devstack_local_conf_path: "{{ devstack_base_dir }}/devstack/local.conf"
+devstack_base_services: "{{ enabled_services | default(omit) }}"
diff --git a/roles/write-devstack-local-conf/library/devstack_local_conf.py b/roles/write-devstack-local-conf/library/devstack_local_conf.py
index 4134beb..55ba4af 100644
--- a/roles/write-devstack-local-conf/library/devstack_local_conf.py
+++ b/roles/write-devstack-local-conf/library/devstack_local_conf.py
@@ -106,13 +106,13 @@
class LocalConf(object):
- def __init__(self, localrc, localconf, services, plugins):
+ def __init__(self, localrc, localconf, base_services, services, plugins):
self.localrc = []
self.meta_sections = {}
if plugins:
self.handle_plugins(plugins)
- if services:
- self.handle_services(services)
+ if services or base_services:
+ self.handle_services(base_services, services or {})
if localrc:
self.handle_localrc(localrc)
if localconf:
@@ -123,7 +123,13 @@
if v:
self.localrc.append('enable_plugin {} {}'.format(k, v))
- def handle_services(self, services):
+ def handle_services(self, base_services, services):
+ enable_base_services = services.pop('base', True)
+ if enable_base_services and base_services:
+ self.localrc.append('ENABLED_SERVICES={}'.format(
+ ",".join(base_services)))
+ else:
+ self.localrc.append('disable_all_services')
for k, v in services.items():
if v is False:
self.localrc.append('disable_service {}'.format(k))
@@ -161,6 +167,7 @@
module = AnsibleModule(
argument_spec=dict(
plugins=dict(type='dict'),
+ base_services=dict(type='list'),
services=dict(type='dict'),
localrc=dict(type='dict'),
local_conf=dict(type='dict'),
@@ -171,6 +178,7 @@
p = module.params
lc = LocalConf(p.get('localrc'),
p.get('local_conf'),
+ p.get('base_services'),
p.get('services'),
p.get('plugins'))
lc.write(p['path'])
diff --git a/roles/write-devstack-local-conf/tasks/main.yaml b/roles/write-devstack-local-conf/tasks/main.yaml
index 1d67616..cc21426 100644
--- a/roles/write-devstack-local-conf/tasks/main.yaml
+++ b/roles/write-devstack-local-conf/tasks/main.yaml
@@ -4,6 +4,7 @@
devstack_local_conf:
path: "{{ devstack_local_conf_path }}"
plugins: "{{ devstack_plugins|default(omit) }}"
+ base_services: "{{ devstack_base_services|default(omit) }}"
services: "{{ devstack_services|default(omit) }}"
localrc: "{{ devstack_localrc|default(omit) }}"
local_conf: "{{ devstack_local_conf|default(omit) }}"
diff --git a/stack.sh b/stack.sh
index a125d4a..32eb43f 100755
--- a/stack.sh
+++ b/stack.sh
@@ -221,7 +221,7 @@
# Warn users who aren't on an explicitly supported distro, but allow them to
# override check and attempt installation with ``FORCE=yes ./stack``
-if [[ ! ${DISTRO} =~ (xenial|yakkety|zesty|stretch|jessie|f24|f25|f26|opensuse-42.2|opensuse-42.3|rhel7) ]]; then
+if [[ ! ${DISTRO} =~ (xenial|zesty|artful|stretch|jessie|f25|f26|f27|opensuse-42.2|opensuse-42.3|rhel7) ]]; then
echo "WARNING: this script has not been tested on $DISTRO"
if [[ "$FORCE" != "yes" ]]; then
die $LINENO "If you wish to run this script anyway run with FORCE=yes"
diff --git a/tools/worlddump.py b/tools/worlddump.py
index 6fff149..7506082 100755
--- a/tools/worlddump.py
+++ b/tools/worlddump.py
@@ -164,8 +164,7 @@
_header("Network Dump")
_dump_cmd("brctl show")
- _dump_cmd("arp -n")
- ip_cmds = ["addr", "link", "route"]
+ ip_cmds = ["neigh", "addr", "link", "route"]
for cmd in ip_cmds + ['netns']:
_dump_cmd("ip %s" % cmd)
for netns_ in _netns_list():
diff --git a/tox.ini b/tox.ini
index 46b15f4..74436b0 100644
--- a/tox.ini
+++ b/tox.ini
@@ -34,16 +34,7 @@
-print0 | xargs -0 bashate -v -iE006 -eE005,E042"
[testenv:docs]
-deps =
- Pygments
- docutils
- sphinx>=1.6.2
- pbr>=2.0.0,!=2.1.0
- openstackdocstheme>=1.11.0
- nwdiag
- blockdiag
- sphinxcontrib-blockdiag
- sphinxcontrib-nwdiag
+deps = -r{toxinidir}/doc/requirements.txt
whitelist_externals = bash
setenv =
TOP_DIR={toxinidir}
@@ -51,11 +42,5 @@
python setup.py build_sphinx
[testenv:venv]
-deps =
- pbr>=2.0.0,!=2.1.0
- sphinx>=1.6.2
- openstackdocstheme>=1.11.0
- blockdiag
- sphinxcontrib-blockdiag
- sphinxcontrib-nwdiag
+deps = -r{toxinidir}/doc/requirements.txt
commands = {posargs}
diff --git a/unstack.sh b/unstack.sh
index 5d3672e..ccea0ef 100755
--- a/unstack.sh
+++ b/unstack.sh
@@ -45,6 +45,10 @@
# Configure Projects
# ==================
+# Determine what system we are running on. This provides ``os_VENDOR``,
+# ``os_RELEASE``, ``os_PACKAGE``, ``os_CODENAME`` and ``DISTRO``
+GetDistro
+
# Plugin Phase 0: override_defaults - allow plugins to override
# defaults before other services are run
run_phase override_defaults
@@ -83,10 +87,6 @@
load_plugin_settings
-# Determine what system we are running on. This provides ``os_VENDOR``,
-# ``os_RELEASE``, ``os_PACKAGE``, ``os_CODENAME``
-GetOSVersion
-
set -o xtrace
# Run extras