Merge "Add a function to get an available random port"
diff --git a/README.rst b/README.rst
index dfa68b9..b4240bd 100644
--- a/README.rst
+++ b/README.rst
@@ -92,5 +92,5 @@
 `local.conf`.  It is likely that you will need to provide and modify
 this file if you want anything other than the most basic setup.  Start
 by reading the `configuration guide
-<https://docs.openstack.org/developer/devstack/configuration.html>_`
+<https://docs.openstack.org/developer/devstack/configuration.html>`_
 for details of the configuration file and the many available options.
diff --git a/clean.sh b/clean.sh
index ef38fbf..9ffe3be 100755
--- a/clean.sh
+++ b/clean.sh
@@ -125,6 +125,13 @@
     sudo rm -rf $SCREEN_LOGDIR
 fi
 
+# Clean out the sytemd user unit files if systemd was used.
+if [[ "$USE_SYSTEMD" = "True" ]]; then
+    sudo find $SYSTEMD_DIR -type f -name '*devstack@*service' -delete
+    # Make systemd aware of the deletion.
+    $SYSTEMCTL daemon-reload
+fi
+
 # Clean up venvs
 DIRS_TO_CLEAN="$WHEELHOUSE ${PROJECT_VENV[@]} .config/openstack"
 rm -rf $DIRS_TO_CLEAN
diff --git a/doc/source/configuration.rst b/doc/source/configuration.rst
index 66b8702..064bf51 100644
--- a/doc/source/configuration.rst
+++ b/doc/source/configuration.rst
@@ -195,6 +195,9 @@
 Setting it here also makes it available for ``openrc`` to set ``OS_AUTH_URL``.
 ``HOST_IPV6`` is not set by default.
 
+For architecture specific configurations which differ from the x86 default
+here, see `arch-configuration`_.
+
 Historical Notes
 ================
 
@@ -749,3 +752,60 @@
     ::
 
         TERMINATE_TIMEOUT=30
+
+
+.. _arch-configuration:
+
+Architectures
+-------------
+
+The upstream CI runs exclusively on nodes with x86 architectures, but
+OpenStack supports even more architectures. Some of them need to configure
+Devstack in a certain way.
+
+KVM on s390x (IBM z Systems)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+KVM on s390x (IBM z Systems) is supported since the *Kilo* release. For
+an all-in-one setup, these minimal settings in the ``local.conf`` file
+are needed::
+
+    [[local|localrc]]
+    ADMIN_PASSWORD=secret
+    DATABASE_PASSWORD=$ADMIN_PASSWORD
+    RABBIT_PASSWORD=$ADMIN_PASSWORD
+    SERVICE_PASSWORD=$ADMIN_PASSWORD
+
+    DOWNLOAD_DEFAULT_IMAGES=False
+    IMAGE_URLS="https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-s390x-disk1.img"
+
+    enable_service n-sproxy
+    disable_service n-novnc
+    disable_service etcd3  # https://bugs.launchpad.net/devstack/+bug/1693192
+
+    [[post-config|$NOVA_CONF]]
+
+    [serial_console]
+    base_url=ws://$HOST_IP:6083/  # optional
+
+Reasoning:
+
+* The default image of Devstack is x86 only, so we deactivate the download
+  with ``DOWNLOAD_DEFAULT_IMAGES``. The referenced guest image
+  in the code above (``IMAGE_URLS``) serves as an example. The list of
+  possible s390x guest images is not limited to that.
+
+* This platform doesn't support a graphical console like VNC or SPICE.
+  The technical reason is the missing framebuffer on the platform. This
+  means we rely on the substitute feature *serial console* which needs the
+  proxy service ``n-sproxy``. We also disable VNC's proxy ``n-novnc`` for
+  that reason . The configuration in the ``post-config`` section is only
+  needed if you want to use the *serial console* outside of the all-in-one
+  setup.
+
+* The service ``etcd3`` needs to be disabled as long as bug report
+  https://bugs.launchpad.net/devstack/+bug/1693192 is not resolved.
+
+.. note:: To run *Tempest* against this *Devstack* all-in-one, you'll need
+   to use a guest image which is smaller than 1GB when uncompressed.
+   The example image from above is bigger than that!
diff --git a/doc/source/guides/devstack-with-lbaas-v2.rst b/doc/source/guides/devstack-with-lbaas-v2.rst
index 21bea99..4ed64bf 100644
--- a/doc/source/guides/devstack-with-lbaas-v2.rst
+++ b/doc/source/guides/devstack-with-lbaas-v2.rst
@@ -45,7 +45,7 @@
     # Horizon
     ENABLED_SERVICES+=,horizon
     # Nova
-    ENABLED_SERVICES+=,n-api,n-crt,n-cpu,n-cond,n-sch
+    ENABLED_SERVICES+=,n-api,n-cpu,n-cond,n-sch
     # Glance
     ENABLED_SERVICES+=,g-api,g-reg
     # Neutron
diff --git a/doc/source/plugin-registry.rst b/doc/source/plugin-registry.rst
index 96a2733..c1c66b9 100644
--- a/doc/source/plugin-registry.rst
+++ b/doc/source/plugin-registry.rst
@@ -48,6 +48,7 @@
 devstack-plugin-glusterfs              `git://git.openstack.org/openstack/devstack-plugin-glusterfs <https://git.openstack.org/cgit/openstack/devstack-plugin-glusterfs>`__
 devstack-plugin-hdfs                   `git://git.openstack.org/openstack/devstack-plugin-hdfs <https://git.openstack.org/cgit/openstack/devstack-plugin-hdfs>`__
 devstack-plugin-kafka                  `git://git.openstack.org/openstack/devstack-plugin-kafka <https://git.openstack.org/cgit/openstack/devstack-plugin-kafka>`__
+devstack-plugin-libvirt-qemu           `git://git.openstack.org/openstack/devstack-plugin-libvirt-qemu <https://git.openstack.org/cgit/openstack/devstack-plugin-libvirt-qemu>`__
 devstack-plugin-mariadb                `git://git.openstack.org/openstack/devstack-plugin-mariadb <https://git.openstack.org/cgit/openstack/devstack-plugin-mariadb>`__
 devstack-plugin-nfs                    `git://git.openstack.org/openstack/devstack-plugin-nfs <https://git.openstack.org/cgit/openstack/devstack-plugin-nfs>`__
 devstack-plugin-pika                   `git://git.openstack.org/openstack/devstack-plugin-pika <https://git.openstack.org/cgit/openstack/devstack-plugin-pika>`__
@@ -62,7 +63,6 @@
 fuxi                                   `git://git.openstack.org/openstack/fuxi <https://git.openstack.org/cgit/openstack/fuxi>`__
 gce-api                                `git://git.openstack.org/openstack/gce-api <https://git.openstack.org/cgit/openstack/gce-api>`__
 glare                                  `git://git.openstack.org/openstack/glare <https://git.openstack.org/cgit/openstack/glare>`__
-gnocchi                                `git://git.openstack.org/openstack/gnocchi <https://git.openstack.org/cgit/openstack/gnocchi>`__
 group-based-policy                     `git://git.openstack.org/openstack/group-based-policy <https://git.openstack.org/cgit/openstack/group-based-policy>`__
 heat                                   `git://git.openstack.org/openstack/heat <https://git.openstack.org/cgit/openstack/heat>`__
 horizon-mellanox                       `git://git.openstack.org/openstack/horizon-mellanox <https://git.openstack.org/cgit/openstack/horizon-mellanox>`__
@@ -136,6 +136,7 @@
 os-xenapi                              `git://git.openstack.org/openstack/os-xenapi <https://git.openstack.org/cgit/openstack/os-xenapi>`__
 osprofiler                             `git://git.openstack.org/openstack/osprofiler <https://git.openstack.org/cgit/openstack/osprofiler>`__
 panko                                  `git://git.openstack.org/openstack/panko <https://git.openstack.org/cgit/openstack/panko>`__
+patrole                                `git://git.openstack.org/openstack/patrole <https://git.openstack.org/cgit/openstack/patrole>`__
 picasso                                `git://git.openstack.org/openstack/picasso <https://git.openstack.org/cgit/openstack/picasso>`__
 rally                                  `git://git.openstack.org/openstack/rally <https://git.openstack.org/cgit/openstack/rally>`__
 sahara                                 `git://git.openstack.org/openstack/sahara <https://git.openstack.org/cgit/openstack/sahara>`__
@@ -145,8 +146,10 @@
 searchlight-ui                         `git://git.openstack.org/openstack/searchlight-ui <https://git.openstack.org/cgit/openstack/searchlight-ui>`__
 senlin                                 `git://git.openstack.org/openstack/senlin <https://git.openstack.org/cgit/openstack/senlin>`__
 solum                                  `git://git.openstack.org/openstack/solum <https://git.openstack.org/cgit/openstack/solum>`__
+stackube                               `git://git.openstack.org/openstack/stackube <https://git.openstack.org/cgit/openstack/stackube>`__
 tacker                                 `git://git.openstack.org/openstack/tacker <https://git.openstack.org/cgit/openstack/tacker>`__
 tap-as-a-service                       `git://git.openstack.org/openstack/tap-as-a-service <https://git.openstack.org/cgit/openstack/tap-as-a-service>`__
+tap-as-a-service-dashboard             `git://git.openstack.org/openstack/tap-as-a-service-dashboard <https://git.openstack.org/cgit/openstack/tap-as-a-service-dashboard>`__
 tricircle                              `git://git.openstack.org/openstack/tricircle <https://git.openstack.org/cgit/openstack/tricircle>`__
 trio2o                                 `git://git.openstack.org/openstack/trio2o <https://git.openstack.org/cgit/openstack/trio2o>`__
 trove                                  `git://git.openstack.org/openstack/trove <https://git.openstack.org/cgit/openstack/trove>`__
diff --git a/files/debs/zookeeper b/files/debs/zookeeper
deleted file mode 100644
index f41b559..0000000
--- a/files/debs/zookeeper
+++ /dev/null
@@ -1 +0,0 @@
-zookeeperd
diff --git a/files/rpms-suse/general b/files/rpms-suse/general
index 1044c25..370f240 100644
--- a/files/rpms-suse/general
+++ b/files/rpms-suse/general
@@ -1,3 +1,5 @@
+apache2
+apache2-devel
 bc
 bridge-utils
 ca-certificates-mozilla
@@ -23,9 +25,11 @@
 python-devel  # pyOpenSSL
 python-xml
 screen
+systemd-devel # for systemd-python
 tar
 tcpdump
 unzip
 util-linux
 wget
+which
 zlib-devel
diff --git a/files/rpms/zookeeper b/files/rpms/zookeeper
deleted file mode 100644
index 1bfac53..0000000
--- a/files/rpms/zookeeper
+++ /dev/null
@@ -1 +0,0 @@
-zookeeper
diff --git a/files/zookeeper/environment b/files/zookeeper/environment
deleted file mode 100644
index afa2d2f..0000000
--- a/files/zookeeper/environment
+++ /dev/null
@@ -1,36 +0,0 @@
-#
-# (C) Copyright 2015 Hewlett Packard Enterprise Development Company LP
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Modified from http://packages.ubuntu.com/saucy/zookeeperd
-NAME=zookeeper
-ZOOCFGDIR=/etc/zookeeper/conf
-
-# seems, that log4j requires the log4j.properties file to be in the classpath
-CLASSPATH="$ZOOCFGDIR:/usr/share/java/jline.jar:/usr/share/java/log4j-1.2.jar:/usr/share/java/xercesImpl.jar:/usr/share/java/xmlParserAPIs.jar:/usr/share/java/netty.jar:/usr/share/java/slf4j-api.jar:/usr/share/java/slf4j-log4j12.jar:/usr/share/java/zookeeper.jar"
-
-ZOOCFG="$ZOOCFGDIR/zoo.cfg"
-ZOO_LOG_DIR=/var/log/zookeeper
-USER=$NAME
-GROUP=$NAME
-PIDDIR=/var/run/$NAME
-PIDFILE=$PIDDIR/$NAME.pid
-SCRIPTNAME=/etc/init.d/$NAME
-JAVA=/usr/bin/java
-ZOOMAIN="org.apache.zookeeper.server.quorum.QuorumPeerMain"
-ZOO_LOG4J_PROP="INFO,ROLLINGFILE"
-JMXLOCALONLY=false
-JAVA_OPTS=""
diff --git a/files/zookeeper/log4j.properties b/files/zookeeper/log4j.properties
deleted file mode 100644
index 6c45a4a..0000000
--- a/files/zookeeper/log4j.properties
+++ /dev/null
@@ -1,69 +0,0 @@
-#
-# (C) Copyright 2015 Hewlett Packard Enterprise Development Company LP
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# From http://packages.ubuntu.com/saucy/zookeeperd
-
-# ZooKeeper Logging Configuration
-#
-
-# Format is "<default threshold> (, <appender>)+
-
-log4j.rootLogger=${zookeeper.root.logger}
-
-# Example: console appender only
-# log4j.rootLogger=INFO, CONSOLE
-
-# Example with rolling log file
-#log4j.rootLogger=DEBUG, CONSOLE, ROLLINGFILE
-
-# Example with rolling log file and tracing
-#log4j.rootLogger=TRACE, CONSOLE, ROLLINGFILE, TRACEFILE
-
-#
-# Log INFO level and above messages to the console
-#
-log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-log4j.appender.CONSOLE.Threshold=INFO
-log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
-log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n
-
-#
-# Add ROLLINGFILE to rootLogger to get log file output
-#    Log DEBUG level and above messages to a log file
-log4j.appender.ROLLINGFILE=org.apache.log4j.RollingFileAppender
-log4j.appender.ROLLINGFILE.Threshold=WARN
-log4j.appender.ROLLINGFILE.File=${zookeeper.log.dir}/zookeeper.log
-
-# Max log file size of 10MB
-log4j.appender.ROLLINGFILE.MaxFileSize=10MB
-# uncomment the next line to limit number of backup files
-#log4j.appender.ROLLINGFILE.MaxBackupIndex=10
-
-log4j.appender.ROLLINGFILE.layout=org.apache.log4j.PatternLayout
-log4j.appender.ROLLINGFILE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n
-
-
-#
-# Add TRACEFILE to rootLogger to get log file output
-#    Log DEBUG level and above messages to a log file
-log4j.appender.TRACEFILE=org.apache.log4j.FileAppender
-log4j.appender.TRACEFILE.Threshold=TRACE
-log4j.appender.TRACEFILE.File=${zookeeper.log.dir}/zookeeper_trace.log
-
-log4j.appender.TRACEFILE.layout=org.apache.log4j.PatternLayout
-### Notice we are including log4j's NDC here (%x)
-log4j.appender.TRACEFILE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L][%x] - %m%n
diff --git a/files/zookeeper/myid b/files/zookeeper/myid
deleted file mode 100644
index c227083..0000000
--- a/files/zookeeper/myid
+++ /dev/null
@@ -1 +0,0 @@
-0
\ No newline at end of file
diff --git a/files/zookeeper/zoo.cfg b/files/zookeeper/zoo.cfg
deleted file mode 100644
index b8f5582..0000000
--- a/files/zookeeper/zoo.cfg
+++ /dev/null
@@ -1,74 +0,0 @@
-#
-# (C) Copyright 2015 Hewlett Packard Enterprise Development Company LP
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# http://hadoop.apache.org/zookeeper/docs/current/zookeeperAdmin.html
-
-# The number of milliseconds of each tick
-tickTime=2000
-# The number of ticks that the initial
-# synchronization phase can take
-initLimit=10
-# The number of ticks that can pass between
-# sending a request and getting an acknowledgement
-syncLimit=5
-# the directory where the snapshot is stored.
-dataDir=/var/lib/zookeeper
-# Place the dataLogDir to a separate physical disc for better performance
-# dataLogDir=/disk2/zookeeper
-
-# the port at which the clients will connect
-clientPort=2181
-
-# Maximum number of clients that can connect from one client
-maxClientCnxns=60
-
-# specify all zookeeper servers
-# The fist port is used by followers to connect to the leader
-# The second one is used for leader election
-
-server.0=127.0.0.1:2888:3888
-
-# To avoid seeks ZooKeeper allocates space in the transaction log file in
-# blocks of preAllocSize kilobytes. The default block size is 64M. One reason
-# for changing the size of the blocks is to reduce the block size if snapshots
-# are taken more often. (Also, see snapCount).
-#preAllocSize=65536
-
-# Clients can submit requests faster than ZooKeeper can process them,
-# especially if there are a lot of clients. To prevent ZooKeeper from running
-# out of memory due to queued requests, ZooKeeper will throttle clients so that
-# there is no more than globalOutstandingLimit outstanding requests in the
-# system. The default limit is 1,000.ZooKeeper logs transactions to a
-# transaction log. After snapCount transactions are written to a log file a
-# snapshot is started and a new transaction log file is started. The default
-# snapCount is 10,000.
-#snapCount=1000
-
-# If this option is defined, requests will be will logged to a trace file named
-# traceFile.year.month.day.
-#traceFile=
-
-# Leader accepts client connections. Default value is "yes". The leader machine
-# coordinates updates. For higher update throughput at thes slight expense of
-# read throughput the leader can be configured to not accept clients and focus
-# on coordination.
-#leaderServes=yes
-
-# Autopurge every hour to avoid using lots of disk in bursts
-# Order of the next 2 properties matters.
-# autopurge.snapRetainCount must be before autopurge.purgeInterval.
-autopurge.snapRetainCount=3
-autopurge.purgeInterval=1
\ No newline at end of file
diff --git a/functions b/functions
index 4d2703f..e056c3f 100644
--- a/functions
+++ b/functions
@@ -323,7 +323,7 @@
         *.vhd|*.vhdx|*.vhd.gz|*.vhdx.gz)
             local extension="${image_fname#*.}"
             image_name=$(basename "$image" ".$extension")
-            disk_format=vhd
+            disk_format=$(echo $image_fname | grep -oP '(?<=\.)vhdx?(?=\.|$)')
             container_format=bare
             if [ "${image_fname##*.}" == "gz" ]; then
                 unpack=zcat
@@ -615,7 +615,7 @@
     # native systemd path, which provides for things like search on
     # request-id. However, there may be an eventlet interaction here,
     # so going off for now.
-    USE_JOURNAL=$(trueorfalse USE_JOURNAL False)
+    USE_JOURNAL=$(trueorfalse False USE_JOURNAL)
     local pidstr=""
     if [[ "$USE_JOURNAL" == "True" ]]; then
         iniset $conf_file $conf_section use_journal "True"
@@ -625,7 +625,7 @@
     fi
     iniset $conf_file $conf_section logging_debug_format_suffix "{{${pidstr}%(funcName)s %(pathname)s:%(lineno)d}}"
 
-    iniset $conf_file $conf_section logging_context_format_string "%(color)s%(levelname)s %(name)s [%(request_id)s %(project_name)s %(user_name)s%(color)s] %(instance)s%(color)s%(message)s"
+    iniset $conf_file $conf_section logging_context_format_string "%(color)s%(levelname)s %(name)s [%(global_request_id)s %(request_id)s %(project_name)s %(user_name)s%(color)s] %(instance)s%(color)s%(message)s"
     iniset $conf_file $conf_section logging_default_format_string "%(color)s%(levelname)s %(name)s [-%(color)s] %(instance)s%(color)s%(message)s"
     iniset $conf_file $conf_section logging_exception_prefix "ERROR %(name)s %(instance)s"
 }
diff --git a/functions-common b/functions-common
index 13559da..65db681 100644
--- a/functions-common
+++ b/functions-common
@@ -93,7 +93,7 @@
         --os-region-name $REGION_NAME \
         --os-identity-api-version 3 \
         $CA_CERT_ARG \
-        --os-auth-url $KEYSTONE_AUTH_URI \
+        --os-auth-url $KEYSTONE_SERVICE_URI \
         --os-username demo \
         --os-password $ADMIN_PASSWORD \
         --os-project-name demo
@@ -105,7 +105,7 @@
         --os-region-name $REGION_NAME \
         --os-identity-api-version 3 \
         $CA_CERT_ARG \
-        --os-auth-url $KEYSTONE_AUTH_URI \
+        --os-auth-url $KEYSTONE_SERVICE_URI \
         --os-username alt_demo \
         --os-password $ADMIN_PASSWORD \
         --os-project-name alt_demo
@@ -117,7 +117,7 @@
         --os-region-name $REGION_NAME \
         --os-identity-api-version 3 \
         $CA_CERT_ARG \
-        --os-auth-url $KEYSTONE_AUTH_URI \
+        --os-auth-url $KEYSTONE_SERVICE_URI \
         --os-username admin \
         --os-password $ADMIN_PASSWORD \
         --os-project-name admin
@@ -1508,8 +1508,13 @@
 
 }
 
-# Helper function to build a basic unit file and run it under systemd.
-function _run_under_systemd {
+# Defines a systemd service which can be enabled and started later on.
+# arg1: The openstack service name ('n-cpu', 'c-sch', ...).
+# arg2: The command to start (e.g. path to service binary + config files).
+# arg3: The group which owns the process.
+# arg4: The user which owns the process.
+# Returns: The systemd service name which got defined.
+function _define_systemd_service {
     local service=$1
     local command="$2"
     local cmd=$command
@@ -1524,9 +1529,7 @@
     else
         write_user_unit_file $systemd_service "$cmd" "$group" "$user"
     fi
-
-    $SYSTEMCTL enable $systemd_service
-    $SYSTEMCTL start $systemd_service
+    echo $systemd_service
 }
 
 # Helper to remove the ``*.failure`` files under ``$SERVICE_DIR/$SCREEN_NAME``.
@@ -1567,11 +1570,19 @@
     local user=$4
 
     local name=$service
+    local systemd_service
 
     time_start "run_process"
+    # Note we deliberately make all service files, even if the service
+    # isn't enabled, so it can be enabled by a dev manually on command
+    # line.
+    if [[ "$USE_SYSTEMD" = "True" ]]; then
+        systemd_service=$(_define_systemd_service "$name" "$command" "$group" "$user")
+    fi
     if is_service_enabled $service; then
         if [[ "$USE_SYSTEMD" = "True" ]]; then
-            _run_under_systemd "$name" "$command" "$group" "$user"
+            $SYSTEMCTL enable $systemd_service
+            $SYSTEMCTL start $systemd_service
         elif [[ "$USE_SCREEN" = "True" ]]; then
             if [[ "$user" == "root" ]]; then
                 command="sudo $command"
@@ -2491,7 +2502,7 @@
     if [[ -n "$start_time" ]]; then
         die $LINENO "Trying to start the clock on $name, but it's already been started"
     fi
-    _TIME_START[$name]=$(date +%s)
+    _TIME_START[$name]=$(date +%s%3N)
 }
 
 # time_stop $name
@@ -2512,7 +2523,7 @@
     if [[ -z "$start_time" ]]; then
         die $LINENO "Trying to stop the clock on $name, but it was never started"
     fi
-    end_time=$(date +%s)
+    end_time=$(date +%s%3N)
     elapsed_time=$(($end_time - $start_time))
     total=${_TIME_TOTAL[$name]:-0}
     # reset the clock so we can start it in the future
@@ -2520,6 +2531,49 @@
     _TIME_TOTAL[$name]=$(($total + $elapsed_time))
 }
 
+function oscwrap {
+    local out
+    local rc
+    local start
+    local end
+    # Cannot use timer_start and timer_stop as we run in subshells
+    # and those rely on modifying vars in the same process (which cannot
+    # happen from a subshell.
+    start=$(date +%s%3N)
+    out=$(command openstack "$@")
+    rc=$?
+    end=$(date +%s%3N)
+    echo $((end - start)) >> $OSCWRAP_TIMER_FILE
+
+    echo "$out"
+    return $rc
+}
+
+function install_oscwrap {
+    # File to accumulate our timing data
+    OSCWRAP_TIMER_FILE=$(mktemp)
+    # Bash by default doesn't expand aliases, allow it for the aliases
+    # we want to whitelist.
+    shopt -s expand_aliases
+    # Remove all aliases that might be expanded to preserve old unexpanded
+    # behavior
+    unalias -a
+    # Add only the alias we want for openstack
+    alias openstack=oscwrap
+}
+
+function cleanup_oscwrap {
+    local total=0
+    if python3_enabled ; then
+        local python=python3
+    else
+        local python=python
+    fi
+    total=$(cat $OSCWRAP_TIMER_FILE | $python -c "import sys; print(sum(int(l) for l in sys.stdin))")
+    _TIME_TOTAL["osc"]=$total
+    rm $OSCWRAP_TIMER_FILE
+}
+
 # time_totals
 #  Print out total time summary
 function time_totals {
@@ -2538,6 +2592,8 @@
         fi
     done
 
+    cleanup_oscwrap
+
     xtrace=$(set +o | grep xtrace)
     set +o xtrace
 
@@ -2549,6 +2605,8 @@
     echo
     for t in ${!_TIME_TOTAL[*]}; do
         local v=${_TIME_TOTAL[$t]}
+        # because we're recording in milliseconds
+        v=$(($v / 1000))
         printf "%-${len}s %3d\n" "$t" "$v"
     done
     echo "========================="
diff --git a/inc/python b/inc/python
index 4c443d6..718cbb2 100644
--- a/inc/python
+++ b/inc/python
@@ -320,6 +320,14 @@
     fi
 
     $xtrace
+
+    # Also install test requirements
+    local install_test_reqs=""
+    local test_req="${!#}/test-requirements.txt"
+    if [[ -e "$test_req" ]]; then
+        install_test_reqs="-r $test_req"
+    fi
+
     # adding SETUPTOOLS_SYS_PATH_TECHNIQUE is a workaround to keep
     # the same behaviour of setuptools before version 25.0.0.
     # related issue: https://github.com/pypa/pip/issues/3874
@@ -329,24 +337,10 @@
         no_proxy="${no_proxy:-}" \
         PIP_FIND_LINKS=$PIP_FIND_LINKS \
         SETUPTOOLS_SYS_PATH_TECHNIQUE=rewrite \
-        $cmd_pip $upgrade \
+        $cmd_pip $upgrade $install_test_reqs \
         $@
     result=$?
 
-    # Also install test requirements
-    local test_req="${!#}/test-requirements.txt"
-    if [[ $result == 0 ]] && [[ -e "$test_req" ]]; then
-        echo "Installing test-requirements for $test_req"
-        $sudo_pip \
-            http_proxy=${http_proxy:-} \
-            https_proxy=${https_proxy:-} \
-            no_proxy=${no_proxy:-} \
-            PIP_FIND_LINKS=$PIP_FIND_LINKS \
-            $cmd_pip $upgrade \
-            -r $test_req
-        result=$?
-    fi
-
     time_stop "pip_install"
     return $result
 }
diff --git a/lib/apache b/lib/apache
index 34ac660..c1b6bf8 100644
--- a/lib/apache
+++ b/lib/apache
@@ -53,8 +53,15 @@
 function enable_apache_mod {
     local mod=$1
     # Apache installation, because we mark it NOPRIME
-    if is_ubuntu || is_suse ; then
-        if ! a2query -m $mod ; then
+    if is_ubuntu; then
+        # Skip mod_version as it is not a valid mod to enable
+        # on debuntu, instead it is built in.
+        if [[ "$mod" != "version" ]] && ! a2query -m $mod ; then
+            sudo a2enmod $mod
+            restart_apache_server
+        fi
+    elif is_suse; then
+        if ! a2enmod -q $mod ; then
             sudo a2enmod $mod
             restart_apache_server
         fi
@@ -96,7 +103,7 @@
     # delete the temp directory
     sudo rm -rf $dir
 
-    if is_ubuntu; then
+    if is_ubuntu || is_suse ; then
         # we've got to enable proxy and proxy_uwsgi for this to work
         sudo a2enmod proxy
         sudo a2enmod proxy_uwsgi
@@ -171,6 +178,8 @@
 # enable_apache_site() - Enable a particular apache site
 function enable_apache_site {
     local site=$@
+    # Many of our sites use mod version. Just enable it.
+    enable_apache_mod version
     if is_ubuntu; then
         sudo a2ensite ${site}
     elif is_fedora || is_suse; then
@@ -229,7 +238,13 @@
     # create a home for the sockets; note don't use /tmp -- apache has
     # a private view of it on some platforms.
     local socket_dir='/var/run/uwsgi'
-    sudo install -d -o $STACK_USER -m 755 $socket_dir
+
+    # /var/run will be empty on ubuntu after reboot, so we can use systemd-temptiles
+    # to automatically create $socket_dir.
+    sudo mkdir -p /etc/tmpfiles.d/
+    echo "d $socket_dir 0755 $STACK_USER root" | sudo tee /etc/tmpfiles.d/uwsgi.conf
+    sudo systemd-tmpfiles --create /etc/tmpfiles.d/uwsgi.conf
+
     local socket="$socket_dir/${name}.socket"
 
     # always cleanup given that we are using iniset here
diff --git a/lib/cinder b/lib/cinder
index e3a687b..2068812 100644
--- a/lib/cinder
+++ b/lib/cinder
@@ -84,20 +84,6 @@
 # CINDER_ENABLED_BACKENDS=${CINDER_ENABLED_BACKENDS:-lvm:lvmdriver-1,lvm:lvmdriver-2}
 CINDER_ENABLED_BACKENDS=${CINDER_ENABLED_BACKENDS:-lvm:lvmdriver-1}
 
-
-# Should cinder perform secure deletion of volumes?
-# Defaults to zero. Can also be set to none or shred.
-# This was previously CINDER_SECURE_DELETE (True or False).
-# Equivalents using CINDER_VOLUME_CLEAR are zero and none, respectively.
-# Set to none to avoid this bug when testing:
-# https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1023755
-if [[ -n $CINDER_SECURE_DELETE ]]; then
-    CINDER_SECURE_DELETE=$(trueorfalse True CINDER_SECURE_DELETE)
-    if [[ $CINDER_SECURE_DELETE == "False" ]]; then
-        CINDER_VOLUME_CLEAR_DEFAULT="none"
-    fi
-    deprecated "Configure secure Cinder volume deletion using CINDER_VOLUME_CLEAR instead of CINDER_SECURE_DELETE."
-fi
 CINDER_VOLUME_CLEAR=${CINDER_VOLUME_CLEAR:-${CINDER_VOLUME_CLEAR_DEFAULT:-zero}}
 CINDER_VOLUME_CLEAR=$(echo ${CINDER_VOLUME_CLEAR} | tr '[:upper:]' '[:lower:]')
 
@@ -143,6 +129,7 @@
 # Test if any Cinder services are enabled
 # is_cinder_enabled
 function is_cinder_enabled {
+    [[ ,${DISABLED_SERVICES} =~ ,"cinder" ]] && return 1
     [[ ,${ENABLED_SERVICES} =~ ,"c-" ]] && return 0
     return 1
 }
@@ -303,6 +290,9 @@
                 default_name=$be_name
             fi
             enabled_backends+=$be_name,
+
+            iniset $CINDER_CONF $be_name volume_clear $CINDER_VOLUME_CLEAR
+
         done
         iniset $CINDER_CONF DEFAULT enabled_backends ${enabled_backends%,*}
         if [[ -n "$default_name" ]]; then
@@ -332,8 +322,6 @@
 
     iniset_rpc_backend cinder $CINDER_CONF
 
-    iniset $CINDER_CONF DEFAULT volume_clear $CINDER_VOLUME_CLEAR
-
     # Format logging
     setup_logging $CINDER_CONF $CINDER_USE_MOD_WSGI
 
@@ -363,11 +351,10 @@
     iniset $CINDER_CONF DEFAULT os_privileged_user_tenant "$SERVICE_PROJECT_NAME"
     iniset $CINDER_CONF DEFAULT graceful_shutdown_timeout "$SERVICE_GRACEFUL_SHUTDOWN_TIMEOUT"
 
-    # Set the backend url according to the configured dlm backend
-    if is_dlm_enabled; then
-        if [[ "$(dlm_backend)" == "zookeeper" ]]; then
-            iniset $CINDER_CONF coordination backend_url "zookeeper://${SERVICE_HOST}:2181"
-        fi
+    if [[ ! -z "$CINDER_COORDINATION_URL" ]]; then
+        iniset $CINDER_CONF coordination backend_url "$CINDER_COORDINATION_URL"
+    elif is_service_enabled etcd3; then
+        iniset $CINDER_CONF coordination backend_url "etcd3+http://${SERVICE_HOST}:2379"
     fi
 }
 
@@ -420,8 +407,10 @@
         # (Re)create cinder database
         recreate_database cinder
 
+        time_start "dbsync"
         # Migrate cinder database
         $CINDER_BIN_DIR/cinder-manage --config-file $CINDER_CONF db sync
+        time_stop "dbsync"
     fi
 
     if is_service_enabled c-vol && [[ -n "$CINDER_ENABLED_BACKENDS" ]]; then
@@ -512,17 +501,24 @@
         fi
     fi
 
-    if [ "$CINDER_USE_MOD_WSGI" == "True" ]; then
-        enable_apache_site osapi-volume
-        restart_apache_server
-        tail_log c-api /var/log/$APACHE_NAME/c-api.log
-    else
-        run_process c-api "$CINDER_BIN_DIR/cinder-api --config-file $CINDER_CONF"
-    fi
+    if is_service_enabled c-api ; then
+        if [ "$CINDER_USE_MOD_WSGI" == "True" ]; then
+            enable_apache_site osapi-volume
+            restart_apache_server
+            tail_log c-api /var/log/$APACHE_NAME/c-api.log
+        else
+            run_process c-api "$CINDER_BIN_DIR/cinder-api --config-file $CINDER_CONF"
+        fi
 
-    echo "Waiting for Cinder API to start..."
-    if ! wait_for_service $SERVICE_TIMEOUT $service_protocol://$CINDER_SERVICE_HOST:$service_port; then
-        die $LINENO "c-api did not start"
+        echo "Waiting for Cinder API to start..."
+        if ! wait_for_service $SERVICE_TIMEOUT $service_protocol://$CINDER_SERVICE_HOST:$service_port; then
+            die $LINENO "c-api did not start"
+        fi
+
+        # Start proxies if enabled
+        if is_service_enabled tls-proxy; then
+            start_tls_proxy cinder '*' $CINDER_SERVICE_PORT $CINDER_SERVICE_HOST $CINDER_SERVICE_PORT_INT
+        fi
     fi
 
     run_process c-sch "$CINDER_BIN_DIR/cinder-scheduler --config-file $CINDER_CONF"
@@ -532,11 +528,6 @@
     # NOTE(jdg): For cinder, startup order matters.  To ensure that repor_capabilities is received
     # by the scheduler start the cinder-volume service last (or restart it) after the scheduler
     # has started.  This is a quick fix for lp bug/1189595
-
-    # Start proxies if enabled
-    if is_service_enabled c-api && is_service_enabled tls-proxy; then
-        start_tls_proxy cinder '*' $CINDER_SERVICE_PORT $CINDER_SERVICE_HOST $CINDER_SERVICE_PORT_INT
-    fi
 }
 
 # stop_cinder() - Stop running processes
diff --git a/lib/cinder_backends/lvm b/lib/cinder_backends/lvm
index d927f9c..03e1880 100644
--- a/lib/cinder_backends/lvm
+++ b/lib/cinder_backends/lvm
@@ -53,9 +53,6 @@
     iniset $CINDER_CONF $be_name iscsi_helper "$CINDER_ISCSI_HELPER"
     iniset $CINDER_CONF $be_name lvm_type "$CINDER_LVM_TYPE"
 
-    if [[ "$CINDER_SECURE_DELETE" == "False" ]]; then
-        iniset $CINDER_CONF $be_name volume_clear none
-    fi
 }
 
 # init_cinder_backend_lvm - Initialize volume group
diff --git a/lib/dlm b/lib/dlm
deleted file mode 100644
index b5ac0f5..0000000
--- a/lib/dlm
+++ /dev/null
@@ -1,111 +0,0 @@
-#!/bin/bash
-#
-# lib/dlm
-#
-# Functions to control the installation and configuration of software
-# that provides a dlm (and possibly other functions). The default is
-# **zookeeper**, and is going to be the only backend supported in the
-# devstack tree.
-
-# Dependencies:
-#
-# - ``functions`` file
-
-# ``stack.sh`` calls the entry points in this order:
-#
-# - is_dlm_enabled
-# - install_dlm
-# - configure_dlm
-# - cleanup_dlm
-
-# Save trace setting
-_XTRACE_DLM=$(set +o | grep xtrace)
-set +o xtrace
-
-
-# Defaults
-# --------
-
-# <define global variables here that belong to this project>
-
-# Set up default directories
-ZOOKEEPER_DATA_DIR=$DEST/data/zookeeper
-ZOOKEEPER_CONF_DIR=/etc/zookeeper
-
-
-# Entry Points
-# ------------
-#
-# NOTE(sdague): it is expected that when someone wants to implement
-# another one of these out of tree, they'll implement the following
-# functions:
-#
-# - dlm_backend
-# - install_dlm
-# - configure_dlm
-# - cleanup_dlm
-
-# This should be declared in the settings file of any plugin or
-# service that needs to have a dlm in their environment.
-function use_dlm {
-    enable_service $(dlm_backend)
-}
-
-# A function to return the name of the backend in question, some users
-# are going to need to know this.
-function dlm_backend {
-    echo "zookeeper"
-}
-
-# Test if a dlm is enabled (defaults to a zookeeper specific check)
-function is_dlm_enabled {
-    [[ ,${ENABLED_SERVICES}, =~ ,"$(dlm_backend)", ]] && return 0
-    return 1
-}
-
-# cleanup_dlm() - Remove residual data files, anything left over from previous
-# runs that a clean run would need to clean up
-function cleanup_dlm {
-    # NOTE(sdague): we don't check for is_enabled here because we
-    # should just delete this regardless. Some times users updated
-    # their service list before they run cleanup.
-    sudo rm -rf $ZOOKEEPER_DATA_DIR
-}
-
-# configure_dlm() - Set config files, create data dirs, etc
-function configure_dlm {
-    if is_dlm_enabled; then
-        sudo cp $FILES/zookeeper/* $ZOOKEEPER_CONF_DIR
-        sudo sed -i -e 's|.*dataDir.*|dataDir='$ZOOKEEPER_DATA_DIR'|' $ZOOKEEPER_CONF_DIR/zoo.cfg
-        # clean up from previous (possibly aborted) runs
-        # create required data files
-        sudo rm -rf $ZOOKEEPER_DATA_DIR
-        sudo mkdir -p $ZOOKEEPER_DATA_DIR
-        # restart after configuration, there is no reason to make this
-        # another step, because having data files that don't match the
-        # zookeeper running is just going to cause tears.
-        restart_service zookeeper
-    fi
-}
-
-# install_dlm() - Collect source and prepare
-function install_dlm {
-    if is_dlm_enabled; then
-        pip_install_gr_extras tooz zookeeper
-        if is_ubuntu; then
-            install_package zookeeperd
-        elif is_fedora; then
-            install_package zookeeper
-        else
-            die $LINENO "Don't know how to install zookeeper on this platform"
-        fi
-    fi
-}
-
-# Restore xtrace
-$_XTRACE_DLM
-
-# Tell emacs to use shell-script-mode
-## Local variables:
-## mode: shell-script
-## End:
diff --git a/lib/etcd3 b/lib/etcd3
new file mode 100644
index 0000000..75c2ae9
--- /dev/null
+++ b/lib/etcd3
@@ -0,0 +1,148 @@
+#!/bin/bash
+#
+# lib/etcd3
+#
+# Functions to control the installation and configuration of etcd 3.x
+# that provides a key-value store (and possibly other functions).
+
+# Dependencies:
+#
+# - ``functions`` file
+
+# ``stack.sh`` calls the entry points in this order:
+#
+# - start_etcd3
+# - stop_etcd3
+# - cleanup_etcd3
+
+# Save trace setting
+_XTRACE_ETCD3=$(set +o | grep xtrace)
+set +o xtrace
+
+
+# Defaults
+# --------
+
+# Set up default values for etcd
+ETCD_DOWNLOAD_URL=${ETCD_DOWNLOAD_URL:-https://github.com/coreos/etcd/releases/download}
+ETCD_VERSION=${ETCD_VERSION:-v3.1.7}
+ETCD_DATA_DIR="$DEST/data/etcd"
+ETCD_SYSTEMD_SERVICE="devstack@etcd.service"
+ETCD_BIN_DIR="$DEST/bin"
+ETCD_SHA256_AMD64="4fde194bbcd259401e2b5c462dfa579ee7f6af539f13f130b8f5b4f52e3b3c52"
+# NOTE(sdague): etcd v3.1.7 doesn't have anything for these architectures, though 3.2.0 does.
+ETCD_SHA256_ARM64=""
+ETCD_SHA256_PPC64=""
+ETCD_PORT=2379
+
+if is_ubuntu ; then
+    UBUNTU_RELEASE_BASE_NUM=`lsb_release -r | awk '{print $2}' | cut -d '.' -f 1`
+fi
+
+# start_etcd3() - Starts to run the etcd process
+function start_etcd3 {
+    # Don't install in sub nodes (multinode scenario)
+    if [ "$SERVICE_HOST" != "$HOST_IP" ]; then
+        return
+    fi
+
+    _install_etcd
+
+    local cmd="$ETCD_BIN_DIR/etcd"
+    cmd+=" --name $HOSTNAME --data-dir $ETCD_DATA_DIR"
+    cmd+=" --initial-cluster-state new --initial-cluster-token etcd-cluster-01"
+    cmd+=" --initial-cluster $HOSTNAME=http://$SERVICE_HOST:2380"
+    cmd+=" --initial-advertise-peer-urls http://$SERVICE_HOST:2380"
+    cmd+=" --advertise-client-urls http://${HOST_IP}:$ETCD_PORT"
+    cmd+=" --listen-peer-urls http://0.0.0.0:2380 "
+    cmd+=" --listen-client-urls http://${HOST_IP}:$ETCD_PORT"
+
+    local unitfile="$SYSTEMD_DIR/$ETCD_SYSTEMD_SERVICE"
+    write_user_unit_file $ETCD_SYSTEMD_SERVICE "$cmd" "" "root"
+
+    iniset -sudo $unitfile "Unit" "After" "network.target"
+    iniset -sudo $unitfile "Service" "Type" "notify"
+    iniset -sudo $unitfile "Service" "Restart" "on-failure"
+    iniset -sudo $unitfile "Service" "LimitNOFILE" "65536"
+
+    $SYSTEMCTL daemon-reload
+    $SYSTEMCTL enable $ETCD_SYSTEMD_SERVICE
+    $SYSTEMCTL start $ETCD_SYSTEMD_SERVICE
+}
+
+# stop_etcd3() stops the etcd3 process
+function stop_etcd3 {
+    # Don't install in sub nodes (multinode scenario)
+    if [ "$SERVICE_HOST" != "$HOST_IP" ]; then
+        return
+    fi
+
+    $SYSTEMCTL stop $ETCD_SYSTEMD_SERVICE
+}
+
+function cleanup_etcd3 {
+    # Don't install in sub nodes (multinode scenario)
+    if [ "$SERVICE_HOST" != "$HOST_IP" ]; then
+        return
+    fi
+
+    $SYSTEMCTL disable $ETCD_SYSTEMD_SERVICE
+
+    local unitfile="$SYSTEMD_DIR/$ETCD_SYSTEMD_SERVICE"
+    sudo rm -f $unitfile
+
+    $SYSTEMCTL daemon-reload
+
+    sudo rm -rf $ETCD_DATA_DIR
+}
+
+function _install_etcd {
+    echo "Installing etcd"
+
+    # Make sure etcd3 downloads the correct architecture
+    if is_arch "x86_64"; then
+        ETCD_ARCH="amd64"
+        ETCD_SHA256=${ETCD_SHA256:-$ETCD_SHA256_AMD64}
+    elif is_arch "aarch64"; then
+        ETCD_ARCH="arm64"
+        ETCD_SHA256=${ETCD_SHA256:-$ETCD_SHA256_ARM64}
+    elif is_arch "ppc64le"; then
+        ETCD_ARCH="ppc64le"
+        ETCD_SHA256=${ETCD_SHA256:-$ETCD_SHA256_PPC64}
+    else
+        exit_distro_not_supported "invalid hardware type - $ETCD_ARCH"
+    fi
+
+    ETCD_NAME=etcd-$ETCD_VERSION-linux-$ETCD_ARCH
+
+    # Install the libraries needed. Note: tooz for example does not have a hard dependency on these libraries
+    pip_install etcd3
+    pip_install etcd3gw
+
+    # Create the necessary directories
+    sudo mkdir -p $ETCD_BIN_DIR
+    sudo mkdir -p $ETCD_DATA_DIR
+
+    # Download and cache the etcd tgz for subsequent use
+    if [ ! -f "$FILES/etcd-$ETCD_VERSION-linux-$ETCD_ARCH/etcd" ]; then
+        ETCD_DOWNLOAD_FILE=$ETCD_NAME.tar.gz
+        wget $ETCD_DOWNLOAD_URL/$ETCD_VERSION/$ETCD_DOWNLOAD_FILE -O $FILES/$ETCD_DOWNLOAD_FILE
+        echo "${ETCD_SHA256} $FILES/${ETCD_DOWNLOAD_FILE}" > $FILES/etcd.sha256sum
+        # NOTE(sdague): this should go fatal if this fails
+        sha256sum -c $FILES/etcd.sha256sum
+
+        tar xzvf $FILES/$ETCD_DOWNLOAD_FILE -C $FILES
+        sudo cp $FILES/$ETCD_NAME/etcd $ETCD_BIN_DIR/etcd
+    fi
+    if [ ! -f "$ETCD_BIN_DIR/etcd" ]; then
+        sudo cp $FILES/$ETCD_NAME/etcd $ETCD_BIN_DIR/etcd
+    fi
+}
+
+# Restore xtrace
+$_XTRACE_ETCD3
+
+# Tell emacs to use shell-script-mode
+## Local variables:
+## mode: shell-script
+## End:
diff --git a/lib/glance b/lib/glance
index d6438a6..baf8c61 100644
--- a/lib/glance
+++ b/lib/glance
@@ -78,6 +78,7 @@
 # Test if any Glance services are enabled
 # is_glance_enabled
 function is_glance_enabled {
+    [[ ,${DISABLED_SERVICES} =~ ,"glance" ]] && return 1
     [[ ,${ENABLED_SERVICES} =~ ,"g-" ]] && return 0
     return 1
 }
@@ -284,11 +285,13 @@
     # (Re)create glance database
     recreate_database glance
 
+    time_start "dbsync"
     # Migrate glance database
     $GLANCE_BIN_DIR/glance-manage --config-file $GLANCE_CONF_DIR/glance-api.conf db_sync
 
     # Load metadata definitions
     $GLANCE_BIN_DIR/glance-manage --config-file $GLANCE_CONF_DIR/glance-api.conf db_load_metadefs
+    time_stop "dbsync"
 
     create_glance_cache_dir
 }
diff --git a/lib/keystone b/lib/keystone
index 4bb6893..eb46526 100644
--- a/lib/keystone
+++ b/lib/keystone
@@ -124,9 +124,15 @@
 # Security compliance
 KEYSTONE_SECURITY_COMPLIANCE_ENABLED=${KEYSTONE_SECURITY_COMPLIANCE_ENABLED:-True}
 KEYSTONE_LOCKOUT_FAILURE_ATTEMPTS=${KEYSTONE_LOCKOUT_FAILURE_ATTEMPTS:-2}
-KEYSTONE_LOCKOUT_DURATION=${KEYSTONE_LOCKOUT_DURATION:-5}
+KEYSTONE_LOCKOUT_DURATION=${KEYSTONE_LOCKOUT_DURATION:-10}
 KEYSTONE_UNIQUE_LAST_PASSWORD_COUNT=${KEYSTONE_UNIQUE_LAST_PASSWORD_COUNT:-2}
 
+# Number of bcrypt hashing rounds, increasing number exponentially increases required
+# resources to generate password hash. This is very effective way to protect from
+# bruteforce attacks. 4 is minimal value that can be specified for bcrypt and
+# it works way faster than default 12. Minimal value is great for CI and development
+# however may not be suitable for real production.
+KEYSTONE_PASSWORD_HASH_ROUNDS=${KEYSTONE_PASSWORD_HASH_ROUNDS:-4}
 
 # Functions
 # ---------
@@ -134,6 +140,7 @@
 # Test if Keystone is enabled
 # is_keystone_enabled
 function is_keystone_enabled {
+    [[ ,${DISABLED_SERVICES} =~ ,"keystone" ]] && return 1
     [[ ,${ENABLED_SERVICES}, =~ ,"key", ]] && return 0
     return 1
 }
@@ -224,6 +231,7 @@
     fi
 
     iniset $KEYSTONE_CONF identity driver "$KEYSTONE_IDENTITY_BACKEND"
+    iniset $KEYSTONE_CONF identity password_hash_rounds $KEYSTONE_PASSWORD_HASH_ROUNDS
     iniset $KEYSTONE_CONF assignment driver "$KEYSTONE_ASSIGNMENT_BACKEND"
     iniset $KEYSTONE_CONF role driver "$KEYSTONE_ROLE_BACKEND"
     iniset $KEYSTONE_CONF resource driver "$KEYSTONE_RESOURCE_BACKEND"
@@ -457,8 +465,10 @@
         recreate_database keystone
     fi
 
+    time_start "dbsync"
     # Initialize keystone database
     $KEYSTONE_BIN_DIR/keystone-manage --config-file $KEYSTONE_CONF db_sync
+    time_stop "dbsync"
 
     if [[ "$KEYSTONE_TOKEN_FORMAT" == "pki" || "$KEYSTONE_TOKEN_FORMAT" == "pkiz" ]]; then
         # Set up certificates
diff --git a/lib/neutron b/lib/neutron
index 941a697..2a660ec 100644
--- a/lib/neutron
+++ b/lib/neutron
@@ -91,6 +91,7 @@
 # Test if any Neutron services are enabled
 # is_neutron_enabled
 function is_neutron_enabled {
+    [[ ,${DISABLED_SERVICES} =~ ,"neutron" ]] && return 1
     [[ ,${ENABLED_SERVICES} =~ ,"neutron-" || ,${ENABLED_SERVICES} =~ ,"q-" ]] && return 0
     return 1
 }
@@ -98,6 +99,7 @@
 # Test if any Neutron services are enabled
 # is_neutron_enabled
 function is_neutron_legacy_enabled {
+    [[ ,${DISABLED_SERVICES} =~ ,"neutron" ]] && return 1
     [[ ,${ENABLED_SERVICES} =~ ,"q-" ]] && return 0
     return 1
 }
@@ -135,7 +137,11 @@
 
     mkdir -p $NEUTRON_CORE_PLUGIN_CONF_PATH
 
-    cp $NEUTRON_DIR/etc/neutron/plugins/$NEUTRON_CORE_PLUGIN/$NEUTRON_CORE_PLUGIN_CONF_FILENAME.sample $NEUTRON_CORE_PLUGIN_CONF
+    # NOTE(yamamoto): A decomposed plugin should prepare the config file in
+    # its devstack plugin.
+    if [ -f $NEUTRON_DIR/etc/neutron/plugins/$NEUTRON_CORE_PLUGIN/$NEUTRON_CORE_PLUGIN_CONF_FILENAME.sample ]; then
+        cp $NEUTRON_DIR/etc/neutron/plugins/$NEUTRON_CORE_PLUGIN/$NEUTRON_CORE_PLUGIN_CONF_FILENAME.sample $NEUTRON_CORE_PLUGIN_CONF
+    fi
 
     iniset $NEUTRON_CONF database connection `database_connection_url neutron`
     iniset $NEUTRON_CONF DEFAULT state_path $NEUTRON_STATE_PATH
@@ -330,8 +336,10 @@
 
     recreate_database neutron
 
+    time_start "dbsync"
     # Run Neutron db migrations
     $NEUTRON_BIN_DIR/neutron-db-manage upgrade heads
+    time_stop "dbsync"
 
     create_neutron_cache_dir
 }
diff --git a/lib/neutron-legacy b/lib/neutron-legacy
index 67cf110..784f3a8 100644
--- a/lib/neutron-legacy
+++ b/lib/neutron-legacy
@@ -417,8 +417,10 @@
 # init_mutnauq() - Initialize databases, etc.
 function init_mutnauq {
     recreate_database $Q_DB_NAME
+    time_start "dbsync"
     # Run Neutron db migrations
     $NEUTRON_BIN_DIR/neutron-db-manage --config-file $NEUTRON_CONF --config-file /$Q_PLUGIN_CONF_FILE upgrade head
+    time_stop "dbsync"
 }
 
 # install_mutnauq() - Collect source and prepare
diff --git a/lib/nova b/lib/nova
index de053ab..18715fc 100644
--- a/lib/nova
+++ b/lib/nova
@@ -175,6 +175,7 @@
 # Test if any Nova services are enabled
 # is_nova_enabled
 function is_nova_enabled {
+    [[ ,${DISABLED_SERVICES} =~ ,"nova" ]] && return 1
     [[ ,${ENABLED_SERVICES} =~ ,"n-" ]] && return 0
     return 1
 }
@@ -453,8 +454,8 @@
     fi
     iniset $NOVA_CONF wsgi api_paste_config "$NOVA_API_PASTE_INI"
     iniset $NOVA_CONF DEFAULT rootwrap_config "$NOVA_CONF_DIR/rootwrap.conf"
-    iniset $NOVA_CONF DEFAULT scheduler_driver "$SCHEDULER"
-    iniset $NOVA_CONF DEFAULT scheduler_default_filters "$FILTERS"
+    iniset $NOVA_CONF scheduler driver "$SCHEDULER"
+    iniset $NOVA_CONF filter_scheduler enabled_filters "$FILTERS"
     iniset $NOVA_CONF DEFAULT default_floating_pool "$PUBLIC_NETWORK_NAME"
     if [[ $SERVICE_IP_VERSION == 6 ]]; then
         iniset $NOVA_CONF DEFAULT my_ip "$HOST_IPV6"
@@ -648,7 +649,9 @@
             iniset $NOVA_CELLS_CONF DEFAULT enabled_apis metadata
         fi
 
+        time_start "dbsync"
         $NOVA_BIN_DIR/nova-manage --config-file $NOVA_CELLS_CONF db sync
+        time_stop "dbsync"
         $NOVA_BIN_DIR/nova-manage --config-file $NOVA_CELLS_CONF cell create --name=region --cell_type=parent --username=$RABBIT_USERID --hostname=$RABBIT_HOST --port=5672 --password=$RABBIT_PASSWORD --virtual_host=/ --woffset=0 --wscale=1
         $NOVA_BIN_DIR/nova-manage cell create --name=child --cell_type=child --username=$RABBIT_USERID --hostname=$RABBIT_HOST --port=5672 --password=$RABBIT_PASSWORD --virtual_host=child_cell --woffset=0 --wscale=1
 
@@ -878,7 +881,6 @@
     run_process n-cond "$NOVA_BIN_DIR/nova-conductor --config-file $compute_cell_conf"
     run_process n-cell-region "$NOVA_BIN_DIR/nova-cells --config-file $api_cell_conf"
     run_process n-cell-child "$NOVA_BIN_DIR/nova-cells --config-file $compute_cell_conf"
-    run_process n-crt "$NOVA_BIN_DIR/nova-cert --config-file $api_cell_conf"
 
     if is_service_enabled n-net; then
         if ! running_in_container; then
@@ -929,7 +931,7 @@
     # Kill the nova screen windows
     # Some services are listed here twice since more than one instance
     # of a service may be running in certain configs.
-    for serv in n-api n-crt n-net n-sch n-novnc n-xvnc n-cauth n-spice n-cond n-cell n-cell n-api-meta n-sproxy; do
+    for serv in n-api n-net n-sch n-novnc n-xvnc n-cauth n-spice n-cond n-cell n-cell n-api-meta n-sproxy; do
         stop_process $serv
     done
 }
diff --git a/lib/nova_plugins/hypervisor-fake b/lib/nova_plugins/hypervisor-fake
index f9b95c1..49c8dee 100644
--- a/lib/nova_plugins/hypervisor-fake
+++ b/lib/nova_plugins/hypervisor-fake
@@ -49,7 +49,7 @@
     iniset $NOVA_CONF DEFAULT quota_security_groups -1
     iniset $NOVA_CONF DEFAULT quota_security_group_rules -1
     iniset $NOVA_CONF DEFAULT quota_key_pairs -1
-    iniset $NOVA_CONF DEFAULT scheduler_default_filters "RetryFilter,AvailabilityZoneFilter,ComputeFilter,ComputeCapabilitiesFilter,ImagePropertiesFilter,CoreFilter,RamFilter,DiskFilter"
+    iniset $NOVA_CONF filter_scheduler enabled_filters "RetryFilter,AvailabilityZoneFilter,ComputeFilter,ComputeCapabilitiesFilter,ImagePropertiesFilter,CoreFilter,RamFilter,DiskFilter"
 }
 
 # install_nova_hypervisor() - Install external components
diff --git a/lib/nova_plugins/hypervisor-xenserver b/lib/nova_plugins/hypervisor-xenserver
index 880b87f..6f79e4f 100644
--- a/lib/nova_plugins/hypervisor-xenserver
+++ b/lib/nova_plugins/hypervisor-xenserver
@@ -84,14 +84,6 @@
 * * * * * /root/rotate_xen_guest_logs.sh >/dev/null 2>&1
 CRONTAB
 
-    # Create directories for kernels and images
-    {
-        echo "set -eux"
-        cat $TOP_DIR/tools/xen/functions
-        echo "create_directory_for_images"
-        echo "create_directory_for_kernels"
-        echo "install_conntrack_tools"
-    } | $ssh_dom0
 }
 
 # install_nova_hypervisor() - Install external components
diff --git a/lib/oslo b/lib/oslo
index 2895503..d15a303 100644
--- a/lib/oslo
+++ b/lib/oslo
@@ -54,6 +54,11 @@
 GITDIR["stevedore"]=$DEST/stevedore
 GITDIR["taskflow"]=$DEST/taskflow
 GITDIR["tooz"]=$DEST/tooz
+# TODO(mriedem): This is a common pattern so even though os-traits isn't
+# officially an oslo library, it is nice to re-use this script for non-oslo
+# things like os-traits. We should rename this script to be more generic
+# and then fold os-brick into it also.
+GITDIR["os-traits"]=$DEST/os-traits
 
 # Support entry points installation of console scripts
 OSLO_BIN_DIR=$(get_python_exec_prefix)
@@ -104,6 +109,7 @@
     _do_install_oslo_lib "stevedore"
     _do_install_oslo_lib "taskflow"
     _do_install_oslo_lib "tooz"
+    _do_install_oslo_lib "os-traits"
 }
 
 # Restore xtrace
diff --git a/lib/placement b/lib/placement
index ad12824..8adbbde 100644
--- a/lib/placement
+++ b/lib/placement
@@ -149,7 +149,9 @@
 function init_placement {
     if [ "$PLACEMENT_DB_ENABLED" != False ]; then
         recreate_database placement
+        time_start "dbsync"
         $NOVA_BIN_DIR/nova-manage --config-file $NOVA_CONF api_db sync
+        time_stop "dbsync"
     fi
     create_placement_accounts
 }
diff --git a/lib/swift b/lib/swift
index 8fad6b8..e247f15 100644
--- a/lib/swift
+++ b/lib/swift
@@ -174,6 +174,7 @@
 # Test if any Swift services are enabled
 # is_swift_enabled
 function is_swift_enabled {
+    [[ ,${DISABLED_SERVICES} =~ ,"swift" ]] && return 1
     [[ ,${ENABLED_SERVICES} =~ ,"s-" ]] && return 0
     return 1
 }
@@ -541,6 +542,10 @@
         else
             iniset ${testfile} func_test auth_prefix /identity/v2.0/
         fi
+        if is_service_enabled tls-proxy; then
+            iniset ${testfile} func_test cafile ${SSL_BUNDLE_FILE}
+            iniset ${testfile} func_test web_front_end apache2
+        fi
     fi
 
     local user_group
diff --git a/lib/tempest b/lib/tempest
index 47785ec..cc65ec7 100644
--- a/lib/tempest
+++ b/lib/tempest
@@ -413,7 +413,7 @@
         TEMPEST_SSH_NETWORK_NAME=$PHYSICAL_NETWORK
     fi
     # Validation
-    iniset $TEMPEST_CONFIG validation run_validation ${TEMPEST_RUN_VALIDATION:-False}
+    iniset $TEMPEST_CONFIG validation run_validation ${TEMPEST_RUN_VALIDATION:-True}
     iniset $TEMPEST_CONFIG validation ip_version_for_ssh 4
     iniset $TEMPEST_CONFIG validation ssh_timeout $BUILD_TIMEOUT
     iniset $TEMPEST_CONFIG validation image_ssh_user ${DEFAULT_INSTANCE_USER:-cirros}
diff --git a/lib/template b/lib/template
index b92fb40..25d653c 100644
--- a/lib/template
+++ b/lib/template
@@ -41,6 +41,7 @@
 # Test if any XXXX services are enabled
 # is_XXXX_enabled
 function is_XXXX_enabled {
+    [[ ,${DISABLED_SERVICES} =~ ,"XXXX" ]] && return 1
     [[ ,${ENABLED_SERVICES} =~ ,"XX-" ]] && return 0
     return 1
 }
diff --git a/lib/tls b/lib/tls
index 238687c..6d67c90 100644
--- a/lib/tls
+++ b/lib/tls
@@ -113,11 +113,11 @@
 certificate             = \$dir/cacert.pem
 private_key             = \$dir/private/cacert.key
 RANDFILE                = \$dir/private/.rand
-default_md              = default
+default_md              = sha256
 
 [ req ]
-default_bits            = 1024
-default_md              = sha1
+default_bits            = 2048
+default_md              = sha256
 
 prompt                  = no
 distinguished_name      = ca_distinguished_name
@@ -212,6 +212,9 @@
     if is_fedora; then
         sudo cp $INT_CA_DIR/ca-chain.pem /usr/share/pki/ca-trust-source/anchors/devstack-chain.pem
         sudo update-ca-trust
+    elif is_suse; then
+        sudo cp $INT_CA_DIR/ca-chain.pem /usr/share/pki/trust/anchors/devstack-chain.pem
+        sudo update-ca-certificates
     elif is_ubuntu; then
         sudo cp $INT_CA_DIR/ca-chain.pem /usr/local/share/ca-certificates/devstack-int.crt
         sudo cp $ROOT_CA_DIR/cacert.pem /usr/local/share/ca-certificates/devstack-root.crt
@@ -345,7 +348,8 @@
 function fix_system_ca_bundle_path {
     if is_service_enabled tls-proxy; then
         local capath
-        capath=$(python -c $'try:\n from requests import certs\n print certs.where()\nexcept ImportError: pass')
+        local python_cmd=${1:-python}
+        capath=$($python_cmd -c $'try:\n from requests import certs\n print (certs.where())\nexcept ImportError: pass')
 
         if [[ ! $capath == "" && ! $capath =~ ^/etc/.* && ! -L $capath ]]; then
             if is_fedora; then
@@ -354,6 +358,9 @@
             elif is_ubuntu; then
                 sudo rm -f $capath
                 sudo ln -s /etc/ssl/certs/ca-certificates.crt $capath
+            elif is_suse; then
+                sudo rm -f $capath
+                sudo ln -s /etc/ssl/ca-bundle.pem $capath
             else
                 echo "Don't know how to set the CA bundle, expect the install to fail."
             fi
@@ -416,6 +423,9 @@
 
     if is_ubuntu; then
         sudo a2enmod ssl
+    elif is_suse; then
+        sudo a2enmod ssl
+        sudo a2enflag SSL
     elif is_fedora; then
         # Fedora enables mod_ssl by default
         :
@@ -522,6 +532,9 @@
     LogFormat "%v %h %l %u %t \"%r\" %>s %b"
 </VirtualHost>
 EOF
+    if is_suse ; then
+        sudo a2enflag SSL
+    fi
     for mod in ssl proxy proxy_http; do
         enable_apache_mod $mod
     done
diff --git a/setup.cfg b/setup.cfg
index 3487f65..73d22b5 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -15,6 +15,7 @@
 all_files = 1
 build-dir = doc/build
 source-dir = doc/source
+warning-is-error = 1
 
 [pbr]
 warnerrors = True
diff --git a/stack.sh b/stack.sh
index e83eaea..cfdc2c4c 100755
--- a/stack.sh
+++ b/stack.sh
@@ -27,11 +27,37 @@
 # Make sure custom grep options don't get in the way
 unset GREP_OPTIONS
 
-# Sanitize language settings to avoid commands bailing out
-# with "unsupported locale setting" errors.
+# NOTE(sdague): why do we explicitly set locale when running stack.sh?
+#
+# Devstack is written in bash, and many functions used throughout
+# devstack process text comming off a command (like the ip command)
+# and do transforms using grep, sed, cut, awk on the strings that are
+# returned. Many of these programs are interationalized, which is
+# great for end users, but means that the strings that devstack
+# functions depend upon might not be there in other locales. We thus
+# need to pin the world to an english basis during the runs.
+#
+# Previously we used the C locale for this, every system has it, and
+# it gives us a stable sort order. It does however mean that we
+# effectively drop unicode support.... boo!  :(
+#
+# With python3 being more unicode aware by default, that's not the
+# right option. While there is a C.utf8 locale, some distros are
+# shipping it as C.UTF8 for extra confusingness. And it's support
+# isn't super clear across distros. This is made more challenging when
+# trying to support both out of the box distros, and the gate which
+# uses diskimage builder to build disk images in a different way than
+# the distros do.
+#
+# So... en_US.utf8 it is. That's existed for a very long time. It is a
+# compromise position, but it is the least worse idea at the time of
+# this comment.
+#
+# We also have to unset other variables that might impact LC_ALL
+# taking effect.
 unset LANG
 unset LANGUAGE
-LC_ALL=C
+LC_ALL=en_US.utf8
 export LC_ALL
 
 # Make sure umask is sane
@@ -192,7 +218,7 @@
 
 # Warn users who aren't on an explicitly supported distro, but allow them to
 # override check and attempt installation with ``FORCE=yes ./stack``
-if [[ ! ${DISTRO} =~ (xenial|yakkety|zesty|stretch|jessie|f24|f25|rhel7|kvmibm1) ]]; then
+if [[ ! ${DISTRO} =~ (xenial|yakkety|zesty|stretch|jessie|f24|f25|opensuse-42.2|rhel7|kvmibm1) ]]; then
     echo "WARNING: this script has not been tested on $DISTRO"
     if [[ "$FORCE" != "yes" ]]; then
         die $LINENO "If you wish to run this script anyway run with FORCE=yes"
@@ -498,6 +524,11 @@
         kill 2>&1 $jobs
     fi
 
+    #Remove timing data file
+    if [ -f "$OSCWRAP_TIMER_FILE" ] ; then
+        rm "$OSCWRAP_TIMER_FILE"
+    fi
+
     # Kill the last spinner process
     kill_spinner
 
@@ -573,7 +604,7 @@
 source $TOP_DIR/lib/neutron
 source $TOP_DIR/lib/ldap
 source $TOP_DIR/lib/dstat
-source $TOP_DIR/lib/dlm
+source $TOP_DIR/lib/etcd3
 source $TOP_DIR/lib/os_brick
 
 # Extras Source
@@ -771,10 +802,6 @@
 install_rpc_backend
 restart_rpc_backend
 
-# NOTE(sdague): dlm install is conditional on one being enabled by configuration
-install_dlm
-configure_dlm
-
 if is_service_enabled $DATABASE_BACKENDS; then
     install_database
 fi
@@ -898,6 +925,9 @@
 
 if is_service_enabled tls-proxy; then
     fix_system_ca_bundle_path
+    if python3_enabled ; then
+        fix_system_ca_bundle_path python3
+    fi
 fi
 
 # Extras Install
@@ -914,6 +944,10 @@
     pip_install_gr python-openstackclient
 fi
 
+# Installs alias for osc so that we can collect timing for all
+# osc commands. Alias dies with stack.sh.
+install_oscwrap
+
 if [[ $TRACK_DEPENDS = True ]]; then
     $DEST/.venv/bin/pip freeze > $DEST/requires-post-pip
     if ! diff -Nru $DEST/requires-pre-pip $DEST/requires-post-pip > $DEST/requires.diff; then
@@ -1043,6 +1077,13 @@
 # A better kind of sysstat, with the top process per time slice
 start_dstat
 
+# Etcd
+# -----
+
+# etcd is a distributed key value store that provides a reliable way to store data across a cluster of machines
+if is_service_enabled etcd3; then
+    start_etcd3
+fi
 
 # Keystone
 # --------
@@ -1256,8 +1297,13 @@
 fi
 
 # Create a randomized default value for the key manager's fixed_key
+# NOTE(lyarwood): This is currently set to 36 as a workaround to the following
+# libvirt bug that incorrectly pads passphrases that are a multiple of 16 bytes
+# in length.
+# Unable to use LUKS passphrase that is exactly 16 bytes long
+# https://bugzilla.redhat.com/show_bug.cgi?id=1447297
 if is_service_enabled nova; then
-    iniset $NOVA_CONF key_manager fixed_key $(generate_hex_string 32)
+    iniset $NOVA_CONF key_manager fixed_key $(generate_hex_string 36)
 fi
 
 # Launch the nova-api and wait for it to answer before continuing
@@ -1289,6 +1335,13 @@
     $NOVA_BIN_DIR/nova-manage --config-file $NM_CONF floating create --ip_range=$TEST_FLOATING_RANGE --pool=$TEST_FLOATING_POOL
 fi
 
+# Start placement before any of the service that are likely to want
+# to use it to manage resource providers.
+if is_service_enabled placement; then
+    echo_summary "Starting Placement"
+    start_placement
+fi
+
 if is_service_enabled neutron; then
     start_neutron
 fi
@@ -1303,10 +1356,6 @@
     start_nova
     create_flavors
 fi
-if is_service_enabled placement; then
-    echo_summary "Starting Placement"
-    start_placement
-fi
 if is_service_enabled cinder; then
     echo_summary "Starting Cinder"
     start_cinder
diff --git a/stackrc b/stackrc
index 41ff268..cfe2496 100644
--- a/stackrc
+++ b/stackrc
@@ -65,7 +65,7 @@
     # Dashboard
     ENABLED_SERVICES+=,horizon
     # Additional services
-    ENABLED_SERVICES+=,rabbit,tempest,mysql,dstat
+    ENABLED_SERVICES+=,rabbit,tempest,mysql,etcd3,dstat
 fi
 
 # Global toggle for enabling services under mod_wsgi. If this is set to
@@ -570,6 +570,10 @@
 GITBRANCH["neutron-lib"]=${NEUTRON_LIB_BRANCH:-master}
 GITDIR["neutron-lib"]=$DEST/neutron-lib
 
+# os-traits library for resource provider traits in the placement service
+GITREPO["os-traits"]=${OS_TRAITS_REPO:-${GIT_BASE}/openstack/os-traits.git}
+GITBRANCH["os-traits"]=${OS_TRAITS_BRANCH:-master}
+
 ##################
 #
 #  TripleO / Heat Agent Components
@@ -607,7 +611,7 @@
 
 # a websockets/html5 or flash powered VNC console for vm instances
 NOVNC_REPO=${NOVNC_REPO:-https://github.com/kanaka/noVNC.git}
-NOVNC_BRANCH=${NOVNC_BRANCH:-master}
+NOVNC_BRANCH=${NOVNC_BRANCH:-stable/v0.6}
 
 # a websockets/html5 or flash powered SPICE console for vm instances
 SPICE_REPO=${SPICE_REPO:-http://anongit.freedesktop.org/git/spice/spice-html5.git}
@@ -655,6 +659,8 @@
         ;;
 esac
 
+# By default, devstack will use Ubuntu Cloud Archive.
+ENABLE_UBUNTU_CLOUD_ARCHIVE=$(trueorfalse True ENABLE_UBUNTU_CLOUD_ARCHIVE)
 
 # Images
 # ------
diff --git a/tests/test_libs_from_pypi.sh b/tests/test_libs_from_pypi.sh
index 608ef6a..1f2d3c2 100755
--- a/tests/test_libs_from_pypi.sh
+++ b/tests/test_libs_from_pypi.sh
@@ -39,7 +39,7 @@
 ALL_LIBS+=" python-openstackclient osc-lib os-client-config oslo.rootwrap"
 ALL_LIBS+=" oslo.i18n oslo.utils python-openstacksdk python-swiftclient"
 ALL_LIBS+=" python-neutronclient tooz ceilometermiddleware oslo.policy"
-ALL_LIBS+=" debtcollector os-brick automaton futurist oslo.service"
+ALL_LIBS+=" debtcollector os-brick os-traits automaton futurist oslo.service"
 ALL_LIBS+=" oslo.cache oslo.reports osprofiler cursive"
 ALL_LIBS+=" keystoneauth ironic-lib neutron-lib oslo.privsep"
 ALL_LIBS+=" diskimage-builder os-vif python-brick-cinderclient-ext"
diff --git a/tests/test_refs.sh b/tests/test_refs.sh
index bccca5d..65848cd 100755
--- a/tests/test_refs.sh
+++ b/tests/test_refs.sh
@@ -15,7 +15,7 @@
 
 echo "Ensuring we don't have crazy refs"
 
-REFS=`grep BRANCH stackrc | grep -v -- '-master'`
+REFS=`grep BRANCH stackrc | grep -v -- '-master' | grep -v 'NOVNC_BRANCH'`
 rc=$?
 if [[ $rc -eq 0 ]]; then
     echo "Branch defaults must be master. Found:"
diff --git a/tools/fixup_stuff.sh b/tools/fixup_stuff.sh
index d07d267..0b78bde 100755
--- a/tools/fixup_stuff.sh
+++ b/tools/fixup_stuff.sh
@@ -72,7 +72,10 @@
 # We've found that Libvirt on Xenial is flaky and crashes enough to be
 # a regular top e-r bug. Opt into Ubuntu Cloud Archive if on Xenial to
 # get newer Libvirt.
-if [[ "$DISTRO" = "xenial" ]]; then
+# Make it possible to switch this based on an environment variable as
+# libvirt 2.5.0 doesn't handle nested virtualization quite well and this
+# is required for the trove development environment.
+if [[ "${ENABLE_UBUNTU_CLOUD_ARCHIVE}" == "True" && "$DISTRO" = "xenial" ]]; then
     # This pulls in apt-add-repository
     install_package "software-properties-common"
     # Use UCA for newer libvirt. Should give us libvirt 2.5.0.
diff --git a/tox.ini b/tox.ini
index 55a06d0..cc7c544 100644
--- a/tox.ini
+++ b/tox.ini
@@ -37,8 +37,8 @@
 deps =
    Pygments
    docutils
-   sphinx>=1.1.2,<1.2
-   pbr>=0.6,!=0.7,<1.0
+   sphinx>=1.5.1,<1.6.1
+   pbr>=2.0.0,!=2.1.0
    oslosphinx
    nwdiag
    blockdiag
@@ -52,8 +52,8 @@
 
 [testenv:venv]
 deps =
-   pbr>=0.6,!=0.7,<1.0
-   sphinx>=1.1.2,<1.2
+   pbr>=2.0.0,!=2.1.0
+   sphinx>=1.5.1,<1.6.1
    oslosphinx
    blockdiag
    sphinxcontrib-blockdiag
diff --git a/unstack.sh b/unstack.sh
index 485fed7..77a151f 100755
--- a/unstack.sh
+++ b/unstack.sh
@@ -69,7 +69,7 @@
 source $TOP_DIR/lib/neutron
 source $TOP_DIR/lib/ldap
 source $TOP_DIR/lib/dstat
-source $TOP_DIR/lib/dlm
+source $TOP_DIR/lib/etcd3
 
 # Extras Source
 # --------------
@@ -162,6 +162,11 @@
     cleanup_neutron
 fi
 
+if is_service_enabled etcd3; then
+    stop_etcd3
+    cleanup_etcd3
+fi
+
 if is_service_enabled dstat; then
     stop_dstat
 fi