diff --git a/HACKING.rst b/HACKING.rst
index d66687e..d763c75 100644
--- a/HACKING.rst
+++ b/HACKING.rst
@@ -328,7 +328,7 @@
 your change
 
 * **Is it passing tests** -- your change will not be reviewed
-  throughly unless the official CI has run successfully against it.
+  thoroughly unless the official CI has run successfully against it.
 
 * **Does this belong in DevStack** -- DevStack reviewers have a
   default position of "no" but are ready to be convinced by your
diff --git a/clean.sh b/clean.sh
index ae28aa9..fc6f80d 100755
--- a/clean.sh
+++ b/clean.sh
@@ -50,7 +50,6 @@
 source $TOP_DIR/lib/swift
 source $TOP_DIR/lib/heat
 source $TOP_DIR/lib/neutron-legacy
-source $TOP_DIR/lib/ironic
 
 
 # Extras Source
diff --git a/doc/source/configuration.rst b/doc/source/configuration.rst
index d87809a..386fbbb 100644
--- a/doc/source/configuration.rst
+++ b/doc/source/configuration.rst
@@ -44,6 +44,7 @@
    before they are started
 -  **extra** - runs after services are started and before any files in
    ``extra.d`` are executed
+-  **post-extra** - runs after files in ``extra.d`` are executed
 
 The file is processed strictly in sequence; meta-sections may be
 specified more than once but if any settings are duplicated the last to
diff --git a/doc/source/guides/devstack-with-lbaas-v2.rst b/doc/source/guides/devstack-with-lbaas-v2.rst
index 4e5f874..f3bd2fe 100644
--- a/doc/source/guides/devstack-with-lbaas-v2.rst
+++ b/doc/source/guides/devstack-with-lbaas-v2.rst
@@ -1,13 +1,17 @@
-Configure Load-Balancer in Kilo
+Configure Load-Balancer Version 2
 =================================
 
-The Kilo release of OpenStack will support Version 2 of the neutron load balancer. Until now, using OpenStack `LBaaS V2 <http://docs.openstack.org/api/openstack-network/2.0/content/lbaas_ext.html>`_ has required a good understanding of neutron and LBaaS architecture and several manual steps.
+Starting in the OpenStack Liberty release, the
+`neutron LBaaS v2 API <http://developer.openstack.org/api-ref-networking-v2-ext.html>`_
+is now stable while the LBaaS v1 API has been deprecated.  The LBaaS v2 reference
+driver is based on Octavia.
 
 
 Phase 1: Create DevStack + 2 nova instances
 --------------------------------------------
 
-First, set up a vm of your choice with at least 8 GB RAM and 16 GB disk space, make sure it is updated. Install git and any other developer tools you find useful.
+First, set up a vm of your choice with at least 8 GB RAM and 16 GB disk space,
+make sure it is updated. Install git and any other developer tools you find useful.
 
 Install devstack
 
@@ -24,6 +28,7 @@
     [[local|localrc]]
     # Load the external LBaaS plugin.
     enable_plugin neutron-lbaas https://git.openstack.org/openstack/neutron-lbaas
+    enable_plugin octavia https://git.openstack.org/openstack/octavia
 
     # ===== BEGIN localrc =====
     DATABASE_PASSWORD=password
@@ -42,13 +47,13 @@
     ENABLED_SERVICES+=,horizon
     # Nova
     ENABLED_SERVICES+=,n-api,n-crt,n-obj,n-cpu,n-cond,n-sch
-    IMAGE_URLS+=",https://launchpad.net/cirros/trunk/0.3.0/+download/cirros-0.3.0-x86_64-disk.img"
     # Glance
     ENABLED_SERVICES+=,g-api,g-reg
     # Neutron
     ENABLED_SERVICES+=,q-svc,q-agt,q-dhcp,q-l3,q-meta
-    # Enable LBaaS V2
+    # Enable LBaaS v2
     ENABLED_SERVICES+=,q-lbaasv2
+    ENABLED_SERVICES+=,octavia,o-cw,o-hk,o-hm,o-api
     # Cinder
     ENABLED_SERVICES+=,c-api,c-vol,c-sch
     # Tempest
@@ -69,11 +74,11 @@
   ::
 
     #create nova instances on private network
-    nova boot --image $(nova image-list | awk '/ cirros-0.3.0-x86_64-disk / {print $2}') --flavor 1 --nic net-id=$(neutron net-list | awk '/ private / {print $2}') node1
-    nova boot --image $(nova image-list | awk '/ cirros-0.3.0-x86_64-disk / {print $2}') --flavor 1 --nic net-id=$(neutron net-list | awk '/ private / {print $2}') node2
+    nova boot --image $(nova image-list | awk '/ cirros-.*-x86_64-uec / {print $2}') --flavor 1 --nic net-id=$(neutron net-list | awk '/ private / {print $2}') node1
+    nova boot --image $(nova image-list | awk '/ cirros-.*-x86_64-uec / {print $2}') --flavor 1 --nic net-id=$(neutron net-list | awk '/ private / {print $2}') node2
     nova list # should show the nova instances just created
 
-    #add secgroup rule to allow ssh etc..
+    #add secgroup rules to allow ssh etc..
     neutron security-group-rule-create default --protocol icmp
     neutron security-group-rule-create default --protocol tcp --port-range-min 22 --port-range-max 22
     neutron security-group-rule-create default --protocol tcp --port-range-min 80 --port-range-max 80
@@ -91,9 +96,16 @@
  ::
 
     neutron lbaas-loadbalancer-create --name lb1 private-subnet
+    neutron lbaas-loadbalancer-show lb1  # Wait for the provisioning_status to be ACTIVE.
     neutron lbaas-listener-create --loadbalancer lb1 --protocol HTTP --protocol-port 80 --name listener1
+    sleep 10  # Sleep since LBaaS actions can take a few seconds depending on the environment.
     neutron lbaas-pool-create --lb-algorithm ROUND_ROBIN --listener listener1 --protocol HTTP --name pool1
+    sleep 10
     neutron lbaas-member-create  --subnet private-subnet --address 10.0.0.3 --protocol-port 80 pool1
+    sleep 10
     neutron lbaas-member-create  --subnet private-subnet --address 10.0.0.5 --protocol-port 80 pool1
 
-Please note here that the "10.0.0.3" and "10.0.0.5" in the above commands are the IPs of the nodes (in my test run-thru, they were actually 10.2 and 10.4), and the address of the created LB will be reported as "vip_address" from the lbaas-loadbalancer-create, and a quick test of that LB is "curl that-lb-ip", which should alternate between showing the IPs of the two nodes.
+Please note here that the "10.0.0.3" and "10.0.0.5" in the above commands are the IPs of the nodes
+(in my test run-thru, they were actually 10.2 and 10.4), and the address of the created LB will be
+reported as "vip_address" from the lbaas-loadbalancer-create, and a quick test of that LB is
+"curl that-lb-ip", which should alternate between showing the IPs of the two nodes.
diff --git a/doc/source/guides/neutron.rst b/doc/source/guides/neutron.rst
index 996c7d1..a72b6f9 100644
--- a/doc/source/guides/neutron.rst
+++ b/doc/source/guides/neutron.rst
@@ -354,8 +354,6 @@
         HOST_IP=10.0.0.2
         SERVICE_HOST=10.0.0.2
         MYSQL_HOST=10.0.0.2
-        SERVICE_HOST=10.0.0.2
-        MYSQL_HOST=10.0.0.2
         RABBIT_HOST=10.0.0.2
         GLANCE_HOSTPORT=10.0.0.2:9292
         PUBLIC_INTERFACE=eth1
@@ -385,6 +383,7 @@
         ## Neutron Networking options used to create Neutron Subnets
 
         FIXED_RANGE="203.0.113.0/24"
+        NETWORK_GATEWAY=203.0.113.1
         PROVIDER_SUBNET_NAME="provider_net"
         PROVIDER_NETWORK_TYPE="vlan"
         SEGMENTATION_ID=2010
@@ -397,7 +396,7 @@
 allocated to you, so that you could access your instances from the
 public internet.
 
-The following is the DevStack configuration on 
+The following is the DevStack configuration on
 compute node 1.
 
 ::
@@ -405,8 +404,6 @@
         HOST_IP=10.0.0.3
         SERVICE_HOST=10.0.0.2
         MYSQL_HOST=10.0.0.2
-        SERVICE_HOST=10.0.0.2
-        MYSQL_HOST=10.0.0.2
         RABBIT_HOST=10.0.0.2
         GLANCE_HOSTPORT=10.0.0.2:9292
         ADMIN_PASSWORD=secrete
diff --git a/doc/source/index.rst b/doc/source/index.rst
index b65730f..2622436 100644
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -44,6 +44,18 @@
 
    We recommend at least a :ref:`minimal-configuration` be set up.
 
+#. Add Stack User
+
+   Devstack should be run as a non-root user with sudo enabled
+   (standard logins to cloud images such as "ubuntu" or "cloud-user"
+   are usually fine).
+
+   You can quickly create a separate `stack` user to run DevStack with
+
+   ::
+
+       devstack/tools/create-stack-user.sh; su stack
+
 #. Start the install
 
    ::
@@ -162,7 +174,6 @@
 * `lib/heat <lib/heat.html>`__
 * `lib/horizon <lib/horizon.html>`__
 * `lib/infra <lib/infra.html>`__
-* `lib/ironic <lib/ironic.html>`__
 * `lib/keystone <lib/keystone.html>`__
 * `lib/ldap <lib/ldap.html>`__
 * `lib/neutron-legacy <lib/neutron-legacy.html>`__
@@ -177,7 +188,6 @@
 * `clean.sh <clean.sh.html>`__
 * `run\_tests.sh <run_tests.sh.html>`__
 
-* `extras.d/50-ironic.sh <extras.d/50-ironic.sh.html>`__
 * `extras.d/60-ceph.sh <extras.d/60-ceph.sh.html>`__
 * `extras.d/70-tuskar.sh <extras.d/70-tuskar.sh.html>`__
 * `extras.d/80-tempest.sh <extras.d/80-tempest.sh.html>`__
diff --git a/doc/source/plugin-registry.rst b/doc/source/plugin-registry.rst
index 49b3a7f..7682def 100644
--- a/doc/source/plugin-registry.rst
+++ b/doc/source/plugin-registry.rst
@@ -26,8 +26,12 @@
 +------------------+---------------------------------------------+--------------------+
 |ceilometer        |git://git.openstack.org/openstack/ceilometer | metering           |
 +------------------+---------------------------------------------+--------------------+
+|congress          |git://git.openstack.org/openstack/congress   | governance         |
++------------------+---------------------------------------------+--------------------+
 |gnocchi           |git://git.openstack.org/openstack/gnocchi    | metric             |
 +------------------+---------------------------------------------+--------------------+
+|ironic            |git://git.openstack.org/openstack/ironic     | baremetal          |
++------------------+---------------------------------------------+--------------------+
 |magnum            |git://git.openstack.org/openstack/magnum     |                    |
 +------------------+---------------------------------------------+--------------------+
 |manila            |git://git.openstack.org/openstack/manila     | file shares        |
diff --git a/doc/source/plugins.rst b/doc/source/plugins.rst
index 8bd3797..83e5609 100644
--- a/doc/source/plugins.rst
+++ b/doc/source/plugins.rst
@@ -19,7 +19,16 @@
 external repositories. The plugin interface assumes the following:
 
 An external git repository that includes a ``devstack/`` top level
-directory. Inside this directory there can be 2 files.
+directory. Inside this directory there can be 3 files.
+
+- ``override-defaults`` - a file containing global variables that
+  will be sourced before the lib/* files. This allows the plugin
+  to override the defaults that are otherwise set in the lib/*
+  files.
+
+  For example, override-defaults may export CINDER_ENABLED_BACKENDS
+  to include the plugin-specific storage backend and thus be able
+  to override the default lvm only storage backend for Cinder.
 
 - ``settings`` - a file containing global variables that will be
   sourced very early in the process. This is helpful if other plugins
@@ -38,7 +47,7 @@
 
 - ``plugin.sh`` - the actual plugin. It is executed by devstack at
   well defined points during a ``stack.sh`` run. The plugin.sh
-  internal structure is discussed bellow.
+  internal structure is discussed below.
 
 
 Plugins are registered by adding the following to the localrc section
diff --git a/extras.d/50-ironic.sh b/extras.d/50-ironic.sh
deleted file mode 100644
index 3b8e3d5..0000000
--- a/extras.d/50-ironic.sh
+++ /dev/null
@@ -1,43 +0,0 @@
-# ironic.sh - Devstack extras script to install ironic
-
-if is_service_enabled ir-api ir-cond; then
-    if [[ "$1" == "source" ]]; then
-        # Initial source
-        source $TOP_DIR/lib/ironic
-    elif [[ "$1" == "stack" && "$2" == "install" ]]; then
-        echo_summary "Installing Ironic"
-        install_ironic
-        install_ironicclient
-        cleanup_ironic
-    elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then
-        echo_summary "Configuring Ironic"
-        configure_ironic
-
-        if is_service_enabled key; then
-            create_ironic_accounts
-        fi
-
-    elif [[ "$1" == "stack" && "$2" == "extra" ]]; then
-        # Initialize ironic
-        init_ironic
-
-        # Start the ironic API and ironic taskmgr components
-        echo_summary "Starting Ironic"
-        start_ironic
-
-        if [[ "$IRONIC_BAREMETAL_BASIC_OPS" = "True" ]]; then
-            prepare_baremetal_basic_ops
-        fi
-    fi
-
-    if [[ "$1" == "unstack" ]]; then
-        stop_ironic
-        if [[ "$IRONIC_BAREMETAL_BASIC_OPS" = "True" ]]; then
-            cleanup_baremetal_basic_ops
-        fi
-    fi
-
-    if [[ "$1" == "clean" ]]; then
-        cleanup_ironic
-    fi
-fi
diff --git a/extras.d/README.md b/extras.d/README.md
index 7c2e4fe..4cec14b 100644
--- a/extras.d/README.md
+++ b/extras.d/README.md
@@ -14,10 +14,13 @@
 entire `stack.sh` variable space is available.  The scripts are
 sourced with one or more arguments, the first of which defines the hook phase:
 
-    source | stack | unstack | clean
+    override_defaults | source | stack | unstack | clean
 
-    source: always called first in any of the scripts, used to set the
-        initial defaults in a lib/* script or similar
+    override_defaults: always called first in any of the scripts, used to
+        override defaults (if need be) that are otherwise set in lib/* scripts
+
+    source: called by stack.sh. Used to set the initial defaults in a lib/*
+        script or similar
 
     stack: called by stack.sh.  There are four possible values for
         the second arg to distinguish the phase stack.sh is in:
diff --git a/files/apache-ironic.template b/files/apache-ironic.template
deleted file mode 100644
index 8864194..0000000
--- a/files/apache-ironic.template
+++ /dev/null
@@ -1,12 +0,0 @@
-Listen %PUBLICPORT%
-
-<VirtualHost *:%PUBLICPORT%>
-    DocumentRoot "%HTTPROOT%"
-    <Directory "%HTTPROOT%">
-        Options Indexes FollowSymLinks
-        AllowOverride None
-        Order allow,deny
-        Allow from all
-        Require all granted
-    </Directory>
-</VirtualHost>
diff --git a/files/debs/ironic b/files/debs/ironic
deleted file mode 100644
index 4d5a6aa..0000000
--- a/files/debs/ironic
+++ /dev/null
@@ -1,19 +0,0 @@
-docker.io
-ipmitool
-iptables
-ipxe
-libguestfs0
-libvirt-bin
-open-iscsi
-openssh-client
-openvswitch-datapath-dkms
-openvswitch-switch
-python-libguestfs
-python-libvirt
-qemu
-qemu-kvm
-qemu-utils
-sgabios
-syslinux
-tftpd-hpa
-xinetd
diff --git a/files/debs/keystone b/files/debs/keystone
index 0795167..370e4aa 100644
--- a/files/debs/keystone
+++ b/files/debs/keystone
@@ -1,6 +1,5 @@
 libkrb5-dev
 libldap2-dev
 libsasl2-dev
-python-mysql.connector
 python-mysqldb
 sqlite3
diff --git a/files/debs/neutron b/files/debs/neutron
index 85145d3..e53cc68 100644
--- a/files/debs/neutron
+++ b/files/debs/neutron
@@ -8,7 +8,6 @@
 libmysqlclient-dev
 mysql-server #NOPRIME
 postgresql-server-dev-all
-python-mysql.connector
 python-mysqldb
 rabbitmq-server # NOPRIME
 radvd # NOPRIME
diff --git a/files/debs/nova b/files/debs/nova
index fe57fc4..58dad41 100644
--- a/files/debs/nova
+++ b/files/debs/nova
@@ -15,7 +15,6 @@
 mysql-server # NOPRIME
 parted
 pm-utils
-python-mysql.connector
 python-mysqldb
 qemu # dist:wheezy,jessie NOPRIME
 qemu-kvm # NOPRIME
diff --git a/files/debs/swift b/files/debs/swift
index 726786e..4b8ac3d 100644
--- a/files/debs/swift
+++ b/files/debs/swift
@@ -1,4 +1,5 @@
 curl
+liberasurecode-dev
 make
 memcached
 sqlite3
diff --git a/files/ebtables.workaround b/files/ebtables.workaround
new file mode 100644
index 0000000..c8af51f
--- /dev/null
+++ b/files/ebtables.workaround
@@ -0,0 +1,23 @@
+#!/bin/bash
+#
+# Copyright 2015 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+#
+# This is a terrible, terrible, truly terrible work around for
+# environments that have libvirt < 1.2.11. ebtables requires that you
+# specifically tell it you would like to not race and get punched in
+# the face when 2 run at the same time with a --concurrent flag.
+
+flock -w 300 /var/lock/ebtables.nova /sbin/ebtables.real $@
diff --git a/files/rpms-suse/swift b/files/rpms-suse/swift
index 52e0a99..3663b98 100644
--- a/files/rpms-suse/swift
+++ b/files/rpms-suse/swift
@@ -1,4 +1,5 @@
 curl
+liberasurecode-devel
 memcached
 sqlite3
 xfsprogs
diff --git a/files/rpms/general b/files/rpms/general
index 40b06f4..2804682 100644
--- a/files/rpms/general
+++ b/files/rpms/general
@@ -8,9 +8,9 @@
 gettext  # used for compiling message catalogs
 git-core
 graphviz # needed only for docs
-iptables-services  # NOPRIME f21,f22
+iptables-services  # NOPRIME f21,f22,f23
 java-1.7.0-openjdk-headless  # NOPRIME rhel7
-java-1.8.0-openjdk-headless  # NOPRIME f21,f22
+java-1.8.0-openjdk-headless  # NOPRIME f21,f22,f23
 libffi-devel
 libjpeg-turbo-devel # Pillow 3.0.0
 libxml2-devel # lxml
diff --git a/files/rpms/ironic b/files/rpms/ironic
deleted file mode 100644
index 2bf8bb3..0000000
--- a/files/rpms/ironic
+++ /dev/null
@@ -1,14 +0,0 @@
-docker-io
-ipmitool
-iptables
-ipxe-bootimgs
-libguestfs
-libvirt
-libvirt-python
-net-tools
-openssh-clients
-openvswitch
-sgabios
-syslinux
-tftp-server
-xinetd
diff --git a/files/rpms/neutron b/files/rpms/neutron
index 9683475..2e49a0c 100644
--- a/files/rpms/neutron
+++ b/files/rpms/neutron
@@ -4,7 +4,6 @@
 ebtables
 iptables
 iputils
-mysql-connector-python
 mysql-devel
 MySQL-python
 mysql-server # NOPRIME
diff --git a/files/rpms/nova b/files/rpms/nova
index 00e7596..4db9a06 100644
--- a/files/rpms/nova
+++ b/files/rpms/nova
@@ -15,7 +15,6 @@
 libvirt-python # NOPRIME
 libxml2-python
 m2crypto
-mysql-connector-python
 mysql-devel
 MySQL-python
 mysql-server # NOPRIME
diff --git a/files/rpms/swift b/files/rpms/swift
index f56a81b..46dc59d 100644
--- a/files/rpms/swift
+++ b/files/rpms/swift
@@ -1,4 +1,5 @@
 curl
+liberasurecode-devel
 memcached
 pyxattr
 rsync-daemon # dist:f22,f23
diff --git a/functions b/functions
index 34da1ba..762fc47 100644
--- a/functions
+++ b/functions
@@ -22,7 +22,7 @@
 source ${FUNC_DIR}/inc/rootwrap
 
 # Save trace setting
-XTRACE=$(set +o | grep xtrace)
+_XTRACE_FUNCTIONS=$(set +o | grep xtrace)
 set +o xtrace
 
 # Check if a function already exists
@@ -603,7 +603,7 @@
 }
 
 # Restore xtrace
-$XTRACE
+$_XTRACE_FUNCTIONS
 
 # Local variables:
 # mode: shell-script
diff --git a/functions-common b/functions-common
index 6a065ba..c3eef46 100644
--- a/functions-common
+++ b/functions-common
@@ -32,7 +32,7 @@
 #
 
 # Save trace setting
-XTRACE=$(set +o | grep xtrace)
+_XTRACE_FUNCTIONS_COMMON=$(set +o | grep xtrace)
 set +o xtrace
 
 # ensure we don't re-source this in the same environment
@@ -954,6 +954,15 @@
             -c URL -f value)
 }
 
+# check if we are using ironic with hardware
+# TODO(jroll) this is a kludge left behind when ripping ironic code
+# out of tree, as it is used by nova and neutron.
+# figure out a way to refactor nova/neutron code to eliminate this
+function is_ironic_hardware {
+    is_service_enabled ironic && [[ -n "${IRONIC_DEPLOY_DRIVER##*_ssh}" ]] && return 0
+    return 1
+}
+
 
 # Package Functions
 # =================
@@ -978,6 +987,34 @@
     echo "$pkg_dir"
 }
 
+# Wrapper for ``apt-get update`` to try multiple times on the update
+# to address bad package mirrors (which happen all the time).
+function apt_get_update {
+    # only do this once per run
+    if [[ "$REPOS_UPDATED" == "True" && "$RETRY_UPDATE" != "True" ]]; then
+        return
+    fi
+
+    # bail if we are offline
+    [[ "$OFFLINE" = "True" ]] && return
+
+    local sudo="sudo"
+    [[ "$(id -u)" = "0" ]] && sudo="env"
+
+    # time all the apt operations
+    time_start "apt-get-update"
+
+    local proxies="http_proxy=${http_proxy:-} https_proxy=${https_proxy:-} no_proxy=${no_proxy:-} "
+    local update_cmd="$sudo $proxies apt-get update"
+    if ! timeout 300 sh -c "while ! $update_cmd; do sleep 30; done"; then
+        die $LINENO "Failed to update apt repos, we're dead now"
+    fi
+
+    REPOS_UPDATED=True
+    # stop the clock
+    time_stop "apt-get-update"
+}
+
 # Wrapper for ``apt-get`` to set cache and proxy environment variables
 # Uses globals ``OFFLINE``, ``*_proxy``
 # apt_get operation package [package ...]
@@ -1158,16 +1195,7 @@
     fi
 
     if is_ubuntu; then
-        local xtrace
-        xtrace=$(set +o | grep xtrace)
-        set +o xtrace
-        if [[ "$REPOS_UPDATED" != "True" || "$RETRY_UPDATE" = "True" ]]; then
-            # if there are transient errors pulling the updates, that's fine.
-            # It may be secondary repositories that we don't really care about.
-            apt_get update  || /bin/true
-            REPOS_UPDATED=True
-        fi
-        $xtrace
+        apt_get_update
     fi
 }
 
@@ -1741,17 +1769,18 @@
     if [[ -d $TOP_DIR/extras.d ]]; then
         local extra_plugin_file_name
         for extra_plugin_file_name in $TOP_DIR/extras.d/*.sh; do
-            [[ -r $extra_plugin_file_name ]] && source $extra_plugin_file_name $mode $phase
-            # NOTE(sdague): generate a big warning about using
-            # extras.d in an unsupported way which will let us track
-            # unsupported usage in the gate.
-            local exceptions="50-ironic.sh 60-ceph.sh 80-tempest.sh"
+            # NOTE(sdague): only process extras.d for the 3 explicitly
+            # white listed elements in tree. We want these to move out
+            # over time as well, but they are in tree, so we need to
+            # manage that.
+            local exceptions="60-ceph.sh 80-tempest.sh"
             local extra
             extra=$(basename $extra_plugin_file_name)
             if [[ ! ( $exceptions =~ "$extra" ) ]]; then
-                deprecated "extras.d support is being removed in Mitaka-1"
-                deprecated "jobs for project $extra will break after that point"
-                deprecated "please move project to a supported devstack plugin model"
+                warn "use of extras.d is no longer supported"
+                warn "processing of project $extra is skipped"
+            else
+                [[ -r $extra_plugin_file_name ]] && source $extra_plugin_file_name $mode $phase
             fi
         done
     fi
@@ -1932,7 +1961,6 @@
         [[ ${service} == n-cpu-* && ${ENABLED_SERVICES} =~ "n-cpu" ]] && enabled=0
         [[ ${service} == "nova" && ${ENABLED_SERVICES} =~ "n-" ]] && enabled=0
         [[ ${service} == "glance" && ${ENABLED_SERVICES} =~ "g-" ]] && enabled=0
-        [[ ${service} == "ironic" && ${ENABLED_SERVICES} =~ "ir-" ]] && enabled=0
         [[ ${service} == "neutron" && ${ENABLED_SERVICES} =~ "q-" ]] && enabled=0
         [[ ${service} == "trove" && ${ENABLED_SERVICES} =~ "tr-" ]] && enabled=0
         [[ ${service} == "swift" && ${ENABLED_SERVICES} =~ "s-" ]] && enabled=0
@@ -2254,7 +2282,7 @@
 }
 
 # Restore xtrace
-$XTRACE
+$_XTRACE_FUNCTIONS_COMMON
 
 # Local variables:
 # mode: shell-script
diff --git a/inc/ini-config b/inc/ini-config
index 42a66c6..d2830d7 100644
--- a/inc/ini-config
+++ b/inc/ini-config
@@ -205,7 +205,7 @@
 }
 
 # Set a multiple line option in an INI file
-# iniset_multiline [-sudo] config-file section option value1 value2 valu3 ...
+# iniset_multiline [-sudo] config-file section option value1 value2 value3 ...
 function iniset_multiline {
     local xtrace
     xtrace=$(set +o | grep xtrace)
diff --git a/inc/meta-config b/inc/meta-config
index b9ab6b2..b6fe437 100644
--- a/inc/meta-config
+++ b/inc/meta-config
@@ -20,7 +20,7 @@
 # file-name is the destination of the config file
 
 # Save trace setting
-INC_META_XTRACE=$(set +o | grep xtrace)
+_XTRACE_INC_META=$(set +o | grep xtrace)
 set +o xtrace
 
 
@@ -197,7 +197,7 @@
 
 
 # Restore xtrace
-$INC_META_XTRACE
+$_XTRACE_INC_META
 
 # Local variables:
 # mode: shell-script
diff --git a/inc/python b/inc/python
index 91ceb44..c157604 100644
--- a/inc/python
+++ b/inc/python
@@ -17,7 +17,7 @@
 
 # Global Config Variables
 
-# PROJECT_VENV contains the name of the virtual enviromnet for each
+# PROJECT_VENV contains the name of the virtual environment for each
 # project.  A null value installs to the system Python directories.
 declare -A PROJECT_VENV
 
@@ -28,14 +28,17 @@
 # Get the path to the pip command.
 # get_pip_command
 function get_pip_command {
-    which pip || which pip-python
+    local version="$1"
+    # NOTE(dhellmann): I don't know if we actually get a pip3.4-python
+    # under any circumstances.
+    which pip${version} || which pip${version}-python
 
     if [ $? -ne 0 ]; then
-        die $LINENO "Unable to find pip; cannot continue"
+        die $LINENO "Unable to find pip${version}; cannot continue"
     fi
 }
 
-# Get the path to the direcotry where python executables are installed.
+# Get the path to the directory where python executables are installed.
 # get_python_exec_prefix
 function get_python_exec_prefix {
     local xtrace
@@ -66,6 +69,13 @@
     pip_install $clean_name
 }
 
+# Determine the python versions supported by a package
+function get_python_versions_for_package {
+    local name=$1
+    cd $name && python setup.py --classifiers \
+        | grep 'Language' | cut -f5 -d: | grep '\.' | tr '\n' ' '
+}
+
 # Wrapper for ``pip install`` to set cache and proxy environment variables
 # Uses globals ``OFFLINE``, ``PIP_VIRTUAL_ENV``,
 # ``PIP_UPGRADE``, ``TRACK_DEPENDS``, ``*_proxy``,
@@ -93,7 +103,7 @@
     fi
     if [[ $TRACK_DEPENDS = True && ! "$@" =~ virtualenv ]]; then
         # TRACK_DEPENDS=True installation creates a circular dependency when
-        # we attempt to install virtualenv into a virualenv, so we must global
+        # we attempt to install virtualenv into a virtualenv, so we must global
         # that installation.
         source $DEST/.venv/bin/activate
         local cmd_pip=$DEST/.venv/bin/pip
@@ -104,8 +114,22 @@
             local sudo_pip="env"
         else
             local cmd_pip
-            cmd_pip=$(get_pip_command)
+            cmd_pip=$(get_pip_command $PYTHON2_VERSION)
             local sudo_pip="sudo -H"
+            if python3_enabled; then
+                # Look at the package classifiers to find the python
+                # versions supported, and if we find the version of
+                # python3 we've been told to use, use that instead of the
+                # default pip
+                local package_dir=${!#}
+                local python_versions
+                if [[ -d "$package_dir" ]]; then
+                    python_versions=$(get_python_versions_for_package $package_dir)
+                    if [[ $python_versions =~ $PYTHON3_VERSION ]]; then
+                        cmd_pip=$(get_pip_command $PYTHON3_VERSION)
+                    fi
+                fi
+            fi
         fi
     fi
 
@@ -113,6 +137,8 @@
     # Always apply constraints
     cmd_pip="$cmd_pip -c $REQUIREMENTS_DIR/upper-constraints.txt"
 
+    # FIXME(dhellmann): Need to force multiple versions of pip for
+    # packages like setuptools?
     local pip_version
     pip_version=$(python -c "import pip; \
                         print(pip.__version__.strip('.')[0])")
@@ -199,7 +225,7 @@
     setup_install $dir
 }
 
-# setup a library by name in editiable mode. If we are trying to use
+# setup a library by name in editable mode. If we are trying to use
 # the library from git, we'll do a git based install, otherwise we'll
 # punt and the library should be installed by a requirements pull from
 # another project.
@@ -276,6 +302,21 @@
     fi
 }
 
+# Report whether python 3 should be used
+function python3_enabled {
+    if [[ $USE_PYTHON3 == "True" ]]; then
+        return 0
+    else
+        return 1
+    fi
+}
+
+# Install python3 packages
+function install_python3 {
+    if is_ubuntu; then
+        apt_get install python3.4 python3.4-dev
+    fi
+}
 
 # Restore xtrace
 $INC_PY_TRACE
diff --git a/inc/rootwrap b/inc/rootwrap
index 63ab59a..2a6e4b6 100644
--- a/inc/rootwrap
+++ b/inc/rootwrap
@@ -22,14 +22,14 @@
     local line
 
     # This is pretty simplistic for now - assume only the first line is used
-    if [[ -r SUDO_SECURE_PATH_FILE ]]; then
+    if [[ -r $SUDO_SECURE_PATH_FILE ]]; then
         line=$(head -1 $SUDO_SECURE_PATH_FILE)
     else
         line="Defaults:$STACK_USER secure_path=/usr/local/sbin:/usr/local/bin:/usr/sbin:/sbin:/usr/bin:/bin"
     fi
 
     # Only add ``dir`` if it is not already present
-    if [[ $line =~ $dir ]]; then
+    if [[ ! $line =~ $dir ]]; then
         echo "${line}:$dir" | sudo tee $SUDO_SECURE_PATH_FILE
         sudo chmod 400 $SUDO_SECURE_PATH_FILE
         sudo chown root:root $SUDO_SECURE_PATH_FILE
diff --git a/lib/apache b/lib/apache
index 17526c7..c9e02a2 100644
--- a/lib/apache
+++ b/lib/apache
@@ -19,7 +19,7 @@
 # - restart_apache_server
 
 # Save trace setting
-XTRACE=$(set +o | grep xtrace)
+_XTRACE_LIB_APACHE=$(set +o | grep xtrace)
 set +o xtrace
 
 # Allow overriding the default Apache user and group, default to
@@ -191,7 +191,7 @@
 }
 
 # Restore xtrace
-$XTRACE
+$_XTRACE_LIB_APACHE
 
 # Tell emacs to use shell-script-mode
 ## Local variables:
diff --git a/lib/ceph b/lib/ceph
index 29d2aca..4ac498a 100644
--- a/lib/ceph
+++ b/lib/ceph
@@ -18,7 +18,7 @@
 # - cleanup_ceph
 
 # Save trace setting
-XTRACE=$(set +o | grep xtrace)
+_XTRACE_LIB_CEPH=$(set +o | grep xtrace)
 set +o xtrace
 
 
@@ -116,7 +116,7 @@
 
 # check_os_support_ceph() - Check if the operating system provides a decent version of Ceph
 function check_os_support_ceph {
-    if [[ ! ${DISTRO} =~ (trusty|f21|f22) ]]; then
+    if [[ ! ${DISTRO} =~ (trusty|f21|f22|f23) ]]; then
         echo "WARNING: your distro $DISTRO does not provide (at least) the Firefly release. Please use Ubuntu Trusty or Fedora 20 (and higher)"
         if [[ "$FORCE_CEPH_INSTALL" != "yes" ]]; then
             die $LINENO "If you wish to install Ceph on this distribution anyway run with FORCE_CEPH_INSTALL=yes"
@@ -375,7 +375,7 @@
 
 
 # Restore xtrace
-$XTRACE
+$_XTRACE_LIB_CEPH
 
 ## Local variables:
 ## mode: shell-script
diff --git a/lib/cinder b/lib/cinder
index cc203ad..5bd940b 100644
--- a/lib/cinder
+++ b/lib/cinder
@@ -20,7 +20,7 @@
 # - cleanup_cinder
 
 # Save trace setting
-XTRACE=$(set +o | grep xtrace)
+_XTRACE_CINDER=$(set +o | grep xtrace)
 set +o xtrace
 
 
@@ -270,10 +270,6 @@
     iniset $CINDER_CONF DEFAULT state_path $CINDER_STATE_PATH
     iniset $CINDER_CONF oslo_concurrency lock_path $CINDER_STATE_PATH
     iniset $CINDER_CONF DEFAULT periodic_interval $CINDER_PERIODIC_INTERVAL
-    # NOTE(thingee): Cinder V1 API is deprecated and defaults to off as of
-    # Juno. Keep it enabled so we can continue testing while it's still
-    # supported.
-    iniset $CINDER_CONF DEFAULT enable_v1_api true
 
     iniset $CINDER_CONF DEFAULT os_region_name "$REGION_NAME"
 
@@ -357,7 +353,7 @@
     iniset $CINDER_CONF DEFAULT os_privileged_user_name nova
     iniset $CINDER_CONF DEFAULT os_privileged_user_password "$SERVICE_PASSWORD"
     iniset $CINDER_CONF DEFAULT os_privileged_user_tenant "$SERVICE_TENANT_NAME"
-
+    iniset $CINDER_CONF DEFAULT graceful_shutdown_timeout "$SERVICE_GRACEFUL_SHUTDOWN_TIMEOUT"
 }
 
 # create_cinder_accounts() - Set up common required cinder accounts
@@ -552,9 +548,7 @@
         local be be_name
         for be in ${CINDER_ENABLED_BACKENDS//,/ }; do
             be_name=${be##*:}
-            # FIXME(jamielennox): Remove --os-volume-api-version pinning when
-            # osc supports volume type create on v2 api. bug #1475060
-            openstack volume type create --os-volume-api-version 1 --property volume_backend_name="${be_name}" ${be_name}
+            openstack volume type create --property volume_backend_name="${be_name}" ${be_name}
         done
     fi
 }
@@ -569,7 +563,7 @@
 
 
 # Restore xtrace
-$XTRACE
+$_XTRACE_CINDER
 
 # Tell emacs to use shell-script-mode
 ## Local variables:
diff --git a/lib/cinder_backends/ceph b/lib/cinder_backends/ceph
index 7e9d2d3..c21350b 100644
--- a/lib/cinder_backends/ceph
+++ b/lib/cinder_backends/ceph
@@ -22,7 +22,7 @@
 
 
 # Save trace setting
-MY_XTRACE=$(set +o | grep xtrace)
+_XTRACE_CINDER_CEPH=$(set +o | grep xtrace)
 set +o xtrace
 
 
@@ -76,7 +76,7 @@
 }
 
 # Restore xtrace
-$MY_XTRACE
+$_XTRACE_CINDER_CEPH
 
 # Local variables:
 # mode: shell-script
diff --git a/lib/cinder_backends/glusterfs b/lib/cinder_backends/glusterfs
index 00c62e0..4e34f8e 100644
--- a/lib/cinder_backends/glusterfs
+++ b/lib/cinder_backends/glusterfs
@@ -19,7 +19,7 @@
 # configure_cinder_backend_glusterfs - Configure Cinder for GlusterFS backends
 
 # Save trace setting
-GLUSTERFS_XTRACE=$(set +o | grep xtrace)
+_XTRACE_CINDER_GLUSTERFS=$(set +o | grep xtrace)
 set +o xtrace
 
 
@@ -41,7 +41,7 @@
 
 
 # Restore xtrace
-$GLUSTERFS_XTRACE
+$_XTRACE_CINDER_GLUSTERFS
 
 # Local variables:
 # mode: shell-script
diff --git a/lib/cinder_backends/lvm b/lib/cinder_backends/lvm
index 411b82c..d927f9c 100644
--- a/lib/cinder_backends/lvm
+++ b/lib/cinder_backends/lvm
@@ -22,7 +22,7 @@
 
 
 # Save trace setting
-MY_XTRACE=$(set +o | grep xtrace)
+_XTRACE_CINDER_LVM=$(set +o | grep xtrace)
 set +o xtrace
 
 
@@ -68,7 +68,7 @@
 }
 
 # Restore xtrace
-$MY_XTRACE
+$_XTRACE_CINDER_LVM
 
 # mode: shell-script
 # End:
diff --git a/lib/cinder_backends/netapp_iscsi b/lib/cinder_backends/netapp_iscsi
index be9442e..5cce30a 100644
--- a/lib/cinder_backends/netapp_iscsi
+++ b/lib/cinder_backends/netapp_iscsi
@@ -20,7 +20,7 @@
 # configure_cinder_backend_netapp_iscsi - configure iSCSI
 
 # Save trace setting
-MY_XTRACE=$(set +o | grep xtrace)
+_XTRACE_CINDER_NETAPP=$(set +o | grep xtrace)
 set +o xtrace
 
 
@@ -59,7 +59,7 @@
 
 
 # Restore xtrace
-$MY_XTRACE
+$_XTRACE_CINDER_NETAPP
 
 # Local variables:
 # mode: shell-script
diff --git a/lib/cinder_backends/netapp_nfs b/lib/cinder_backends/netapp_nfs
index dc919ad..7ba36d2 100644
--- a/lib/cinder_backends/netapp_nfs
+++ b/lib/cinder_backends/netapp_nfs
@@ -20,7 +20,7 @@
 # configure_cinder_backend_netapp_nfs - configure NFS
 
 # Save trace setting
-MY_XTRACE=$(set +o | grep xtrace)
+_XTRACE_CINDER_NETAPP=$(set +o | grep xtrace)
 set +o xtrace
 
 
@@ -70,7 +70,7 @@
 
 
 # Restore xtrace
-$MY_XTRACE
+$_XTRACE_CINDER_NETAPP
 
 # Local variables:
 # mode: shell-script
diff --git a/lib/cinder_backends/nfs b/lib/cinder_backends/nfs
index fc51b2b..89a37a1 100644
--- a/lib/cinder_backends/nfs
+++ b/lib/cinder_backends/nfs
@@ -19,7 +19,7 @@
 # configure_cinder_backend_nfs - Configure Cinder for NFS backends
 
 # Save trace setting
-NFS_XTRACE=$(set +o | grep xtrace)
+_XTRACE_CINDER_NFS=$(set +o | grep xtrace)
 set +o xtrace
 
 
@@ -38,7 +38,7 @@
 
 
 # Restore xtrace
-$NFS_XTRACE
+$_XTRACE_CINDER_NFS
 
 # Local variables:
 # mode: shell-script
diff --git a/lib/cinder_backends/solidfire b/lib/cinder_backends/solidfire
index 7cc70fc..16bc527 100644
--- a/lib/cinder_backends/solidfire
+++ b/lib/cinder_backends/solidfire
@@ -17,7 +17,7 @@
 # configure_cinder_driver - make configuration changes, including those to other services
 
 # Save trace setting
-MY_XTRACE=$(set +o | grep xtrace)
+_XTRACE_CINDER_SOLIDFIRE=$(set +o | grep xtrace)
 set +o xtrace
 
 
@@ -42,7 +42,7 @@
 
 
 # Restore xtrace
-$MY_XTRACE
+$_XTRACE_CINDER_SOLIDFIRE
 
 # Local variables:
 # mode: shell-script
diff --git a/lib/cinder_backends/vmdk b/lib/cinder_backends/vmdk
index d5b9453..3a6a5cf 100644
--- a/lib/cinder_backends/vmdk
+++ b/lib/cinder_backends/vmdk
@@ -15,7 +15,7 @@
 # configure_cinder_backend_vmdk - Configure Cinder for VMware vmdk backends
 
 # Save trace setting
-VMDK_XTRACE=$(set +o | grep xtrace)
+_XTRACE_CINDER_VMDK=$(set +o | grep xtrace)
 set +o xtrace
 
 
@@ -40,7 +40,7 @@
 
 
 # Restore xtrace
-$VMDK_XTRACE
+$_XTRACE_CINDER_VMDK
 
 # Local variables:
 # mode: shell-script
diff --git a/lib/cinder_backends/xiv b/lib/cinder_backends/xiv
index 6eadaae..e8b5da0 100644
--- a/lib/cinder_backends/xiv
+++ b/lib/cinder_backends/xiv
@@ -42,7 +42,7 @@
 # configure_cinder_backend_xiv - Configure Cinder for xiv backends
 
 # Save trace setting
-XIV_XTRACE=$(set +o | grep xtrace)
+_XTRACE_CINDER_XIV=$(set +o | grep xtrace)
 set +o xtrace
 
 # Defaults
@@ -79,7 +79,7 @@
 }
 
 # Restore xtrace
-$XIV_XTRACE
+$_XTRACE_CINDER_XIV
 
 # Local variables:
 # mode: shell-script
diff --git a/lib/cinder_plugins/XenAPINFS b/lib/cinder_plugins/XenAPINFS
index f730695..92135e7 100644
--- a/lib/cinder_plugins/XenAPINFS
+++ b/lib/cinder_plugins/XenAPINFS
@@ -15,7 +15,7 @@
 # configure_cinder_driver - make configuration changes, including those to other services
 
 # Save trace setting
-MY_XTRACE=$(set +o | grep xtrace)
+_XTRACE_CINDER_XENAPINFS=$(set +o | grep xtrace)
 set +o xtrace
 
 
@@ -39,7 +39,7 @@
 }
 
 # Restore xtrace
-$MY_XTRACE
+$_XTRACE_CINDER_XENAPINFS
 
 # Local variables:
 # mode: shell-script
diff --git a/lib/cinder_plugins/glusterfs b/lib/cinder_plugins/glusterfs
index 35ceb27..329dd6c 100644
--- a/lib/cinder_plugins/glusterfs
+++ b/lib/cinder_plugins/glusterfs
@@ -15,7 +15,7 @@
 # configure_cinder_driver - make configuration changes, including those to other services
 
 # Save trace setting
-MY_XTRACE=$(set +o | grep xtrace)
+_XTRACE_CINDER_GLUSTERFS=$(set +o | grep xtrace)
 set +o xtrace
 
 
@@ -45,7 +45,7 @@
 }
 
 # Restore xtrace
-$MY_XTRACE
+$_XTRACE_CINDER_GLUSTERFS
 
 # Local variables:
 # mode: shell-script
diff --git a/lib/cinder_plugins/nfs b/lib/cinder_plugins/nfs
index 83b3993..6e4ffe0 100644
--- a/lib/cinder_plugins/nfs
+++ b/lib/cinder_plugins/nfs
@@ -15,7 +15,7 @@
 # configure_cinder_driver - make configuration changes, including those to other services
 
 # Save trace setting
-MY_XTRACE=$(set +o | grep xtrace)
+_XTRACE_CINDER_NFS=$(set +o | grep xtrace)
 set +o xtrace
 
 
@@ -36,7 +36,7 @@
 }
 
 # Restore xtrace
-$MY_XTRACE
+$_XTRACE_CINDER_NFS
 
 # Local variables:
 # mode: shell-script
diff --git a/lib/cinder_plugins/sheepdog b/lib/cinder_plugins/sheepdog
index ca343f7..558de46 100644
--- a/lib/cinder_plugins/sheepdog
+++ b/lib/cinder_plugins/sheepdog
@@ -15,7 +15,7 @@
 # configure_cinder_driver - make configuration changes, including those to other services
 
 # Save trace setting
-MY_XTRACE=$(set +o | grep xtrace)
+_XTRACE_CINDER_SHEEPDOG=$(set +o | grep xtrace)
 set +o xtrace
 
 
@@ -34,7 +34,7 @@
 }
 
 # Restore xtrace
-$MY_XTRACE
+$_XTRACE_CINDER_SHEEPDOG
 
 # Local variables:
 # mode: shell-script
diff --git a/lib/cinder_plugins/vsphere b/lib/cinder_plugins/vsphere
index f14ddf0..1b28ffe 100644
--- a/lib/cinder_plugins/vsphere
+++ b/lib/cinder_plugins/vsphere
@@ -15,7 +15,7 @@
 # configure_cinder_driver - make configuration changes, including those to other services
 
 # Save trace setting
-MY_XTRACE=$(set +o | grep xtrace)
+_XTRACE_CINDER_VSPHERE=$(set +o | grep xtrace)
 set +o xtrace
 
 
@@ -37,7 +37,7 @@
 }
 
 # Restore xtrace
-$MY_XTRACE
+$_XTRACE_CINDER_VSPHERE
 
 # Local variables:
 # mode: shell-script
diff --git a/lib/database b/lib/database
index 13740b9..0d72052 100644
--- a/lib/database
+++ b/lib/database
@@ -20,7 +20,7 @@
 # and call register_database $DATABASE_TYPE
 
 # Save trace setting
-XTRACE=$(set +o | grep xtrace)
+_XTRACE_LIB_DB=$(set +o | grep xtrace)
 set +o xtrace
 
 DATABASE_BACKENDS=""
@@ -137,7 +137,7 @@
 
 
 # Restore xtrace
-$XTRACE
+$_XTRACE_LIB_DB
 
 # Tell emacs to use shell-script-mode
 ## Local variables:
diff --git a/lib/databases/mysql b/lib/databases/mysql
index c2ab32e..1bbbd62 100644
--- a/lib/databases/mysql
+++ b/lib/databases/mysql
@@ -8,7 +8,7 @@
 # - DATABASE_{HOST,USER,PASSWORD} must be defined
 
 # Save trace setting
-MY_XTRACE=$(set +o | grep xtrace)
+_XTRACE_DB_MYSQL=$(set +o | grep xtrace)
 set +o xtrace
 
 MYSQL_DRIVER=${MYSQL_DRIVER:-PyMySQL}
@@ -143,7 +143,7 @@
 [client]
 user=$DATABASE_USER
 password=$DATABASE_PASSWORD
-host=$DATABASE_HOST
+host=$MYSQL_HOST
 EOF
         chmod 0600 $HOME/.my.cnf
     fi
@@ -178,7 +178,7 @@
 
 
 # Restore xtrace
-$MY_XTRACE
+$_XTRACE_DB_MYSQL
 
 # Local variables:
 # mode: shell-script
diff --git a/lib/databases/postgresql b/lib/databases/postgresql
index 78c7bed..913e8ff 100644
--- a/lib/databases/postgresql
+++ b/lib/databases/postgresql
@@ -8,7 +8,7 @@
 # - DATABASE_{HOST,USER,PASSWORD} must be defined
 
 # Save trace setting
-PG_XTRACE=$(set +o | grep xtrace)
+_XTRACE_PG=$(set +o | grep xtrace)
 set +o xtrace
 
 
@@ -119,7 +119,7 @@
 
 
 # Restore xtrace
-$PG_XTRACE
+$_XTRACE_PG
 
 # Local variables:
 # mode: shell-script
diff --git a/lib/dlm b/lib/dlm
index f68ee26..74eb67e 100644
--- a/lib/dlm
+++ b/lib/dlm
@@ -19,7 +19,7 @@
 # - cleanup_dlm
 
 # Save trace setting
-XTRACE=$(set +o | grep xtrace)
+_XTRACE_DLM=$(set +o | grep xtrace)
 set +o xtrace
 
 
@@ -46,7 +46,7 @@
 # - cleanup_dlm
 
 # This should be declared in the settings file of any plugin or
-# service that needs to have a dlm in their enviroment.
+# service that needs to have a dlm in their environment.
 function use_dlm {
     enable_service $(dlm_backend)
 }
@@ -100,7 +100,7 @@
 }
 
 # Restore xtrace
-$XTRACE
+$_XTRACE_DLM
 
 # Tell emacs to use shell-script-mode
 ## Local variables:
diff --git a/lib/dstat b/lib/dstat
index fe4790b..b705948 100644
--- a/lib/dstat
+++ b/lib/dstat
@@ -13,7 +13,7 @@
 # - stop_dstat
 
 # Save trace setting
-XTRACE=$(set +o | grep xtrace)
+_XTRACE_DSTAT=$(set +o | grep xtrace)
 set +o xtrace
 
 # start_dstat() - Start running processes, including screen
@@ -34,4 +34,4 @@
 }
 
 # Restore xtrace
-$XTRACE
+$_XTRACE_DSTAT
diff --git a/lib/glance b/lib/glance
index 5712943..19e7937 100644
--- a/lib/glance
+++ b/lib/glance
@@ -21,7 +21,7 @@
 # - cleanup_glance
 
 # Save trace setting
-XTRACE=$(set +o | grep xtrace)
+_XTRACE_GLANCE=$(set +o | grep xtrace)
 set +o xtrace
 
 
@@ -115,6 +115,7 @@
     configure_auth_token_middleware $GLANCE_REGISTRY_CONF glance $GLANCE_AUTH_CACHE_DIR/registry
     iniset $GLANCE_REGISTRY_CONF DEFAULT notification_driver messaging
     iniset_rpc_backend glance $GLANCE_REGISTRY_CONF
+    iniset $GLANCE_REGISTRY_CONF DEFAULT graceful_shutdown_timeout "$SERVICE_GRACEFUL_SHUTDOWN_TIMEOUT"
 
     cp $GLANCE_DIR/etc/glance-api.conf $GLANCE_API_CONF
     iniset $GLANCE_API_CONF DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL
@@ -149,6 +150,7 @@
         iniset $GLANCE_API_CONF glance_store swift_store_config_file $GLANCE_SWIFT_STORE_CONF
         iniset $GLANCE_API_CONF glance_store default_swift_reference ref1
         iniset $GLANCE_API_CONF glance_store stores "file, http, swift"
+        iniset $GLANCE_API_CONF DEFAULT graceful_shutdown_timeout "$SERVICE_GRACEFUL_SHUTDOWN_TIMEOUT"
 
         iniset $GLANCE_SWIFT_STORE_CONF ref1 user $SERVICE_TENANT_NAME:glance-swift
         iniset $GLANCE_SWIFT_STORE_CONF ref1 key $SERVICE_PASSWORD
@@ -406,7 +408,7 @@
 }
 
 # Restore xtrace
-$XTRACE
+$_XTRACE_GLANCE
 
 # Tell emacs to use shell-script-mode
 ## Local variables:
diff --git a/lib/heat b/lib/heat
index e42bdf0..3666776 100644
--- a/lib/heat
+++ b/lib/heat
@@ -23,7 +23,7 @@
 # - cleanup_heat
 
 # Save trace setting
-XTRACE=$(set +o | grep xtrace)
+_XTRACE_HEAT=$(set +o | grep xtrace)
 set +o xtrace
 
 
@@ -56,6 +56,10 @@
 HEAT_TEMPLATES_DIR=$HEAT_CONF_DIR/templates
 HEAT_API_HOST=${HEAT_API_HOST:-$HOST_IP}
 HEAT_API_PORT=${HEAT_API_PORT:-8004}
+HEAT_SERVICE_USER=${HEAT_SERVICE_USER:-heat}
+HEAT_TRUSTEE_USER=${HEAT_TRUSTEE_USER:-$HEAT_SERVICE_USER}
+HEAT_TRUSTEE_PASSWORD=${HEAT_TRUSTEE_PASSWORD:-$SERVICE_PASSWORD}
+HEAT_TRUSTEE_DOMAIN=${HEAT_TRUSTEE_DOMAIN:-default}
 
 # Support entry points installation of console scripts
 HEAT_BIN_DIR=$(get_python_exec_prefix)
@@ -65,9 +69,15 @@
     # for standalone, use defaults which require no service user
     HEAT_STACK_DOMAIN=$(trueorfalse False HEAT_STACK_DOMAIN)
     HEAT_DEFERRED_AUTH=${HEAT_DEFERRED_AUTH:-password}
+    if [[ ${HEAT_DEFERRED_AUTH} != "password" ]]; then
+        # Heat does not support keystone trusts when deployed in
+        # standalone mode
+        die $LINENO \
+            'HEAT_DEFERRED_AUTH can only be set to "password" when HEAT_STANDALONE is True.'
+    fi
 else
     HEAT_STACK_DOMAIN=$(trueorfalse True HEAT_STACK_DOMAIN)
-    HEAT_DEFERRED_AUTH=${HEAT_DEFERRED_AUTH:-trusts}
+    HEAT_DEFERRED_AUTH=${HEAT_DEFERRED_AUTH:-}
 fi
 HEAT_PLUGIN_DIR=${HEAT_PLUGIN_DIR:-$DATA_DIR/heat/plugins}
 ENABLE_HEAT_PLUGINS=${ENABLE_HEAT_PLUGINS:-}
@@ -128,30 +138,39 @@
         setup_colorized_logging $HEAT_CONF DEFAULT tenant user
     fi
 
-    iniset $HEAT_CONF DEFAULT deferred_auth_method $HEAT_DEFERRED_AUTH
+    if [ ! -z "$HEAT_DEFERRED_AUTH" ]; then
+        iniset $HEAT_CONF DEFAULT deferred_auth_method $HEAT_DEFERRED_AUTH
+    fi
 
     if [ "$HEAT_USE_MOD_WSGI" == "True" ]; then
         _config_heat_apache_wsgi
     fi
 
-    # NOTE(jamielennox): heat re-uses specific values from the
-    # keystone_authtoken middleware group and so currently fails when using the
-    # auth plugin setup. This should be fixed in heat.  Heat is also the only
-    # service that requires the auth_uri to include a /v2.0. Remove this custom
-    # setup when bug #1300246 is resolved.
-    iniset $HEAT_CONF keystone_authtoken auth_uri $KEYSTONE_SERVICE_URI/v2.0
     if [[ "$HEAT_STANDALONE" = "True" ]]; then
         iniset $HEAT_CONF paste_deploy flavor standalone
         iniset $HEAT_CONF clients_heat url "http://$HEAT_API_HOST:$HEAT_API_PORT/v1/%(tenant_id)s"
     else
         iniset $HEAT_CONF keystone_authtoken identity_uri $KEYSTONE_AUTH_URI
-        iniset $HEAT_CONF keystone_authtoken admin_user heat
+        iniset $HEAT_CONF keystone_authtoken admin_user $HEAT_SERVICE_USER
         iniset $HEAT_CONF keystone_authtoken admin_password $SERVICE_PASSWORD
         iniset $HEAT_CONF keystone_authtoken admin_tenant_name $SERVICE_TENANT_NAME
         iniset $HEAT_CONF keystone_authtoken cafile $SSL_BUNDLE_FILE
         iniset $HEAT_CONF keystone_authtoken signing_dir $HEAT_AUTH_CACHE_DIR
     fi
 
+    # If HEAT_DEFERRED_AUTH is unset or explicitly set to trusts, configure
+    # the section for the client plugin associated with the trustee
+    if [ -z "$HEAT_DEFERRED_AUTH" -o "trusts" == "$HEAT_DEFERRED_AUTH" ]; then
+        iniset $HEAT_CONF trustee auth_plugin password
+        iniset $HEAT_CONF trustee auth_url $KEYSTONE_AUTH_URI
+        iniset $HEAT_CONF trustee username $HEAT_TRUSTEE_USER
+        iniset $HEAT_CONF trustee password $HEAT_TRUSTEE_PASSWORD
+        iniset $HEAT_CONF trustee user_domain_id $HEAT_TRUSTEE_DOMAIN
+    fi
+
+    # clients_keystone
+    iniset $HEAT_CONF clients_keystone auth_uri $KEYSTONE_AUTH_URI
+
     # ec2authtoken
     iniset $HEAT_CONF ec2authtoken auth_uri $KEYSTONE_SERVICE_URI/v2.0
 
@@ -464,7 +483,7 @@
 }
 
 # Restore xtrace
-$XTRACE
+$_XTRACE_HEAT
 
 # Tell emacs to use shell-script-mode
 ## Local variables:
diff --git a/lib/horizon b/lib/horizon
index ff63b06..67181fc 100644
--- a/lib/horizon
+++ b/lib/horizon
@@ -19,7 +19,7 @@
 # - cleanup_horizon
 
 # Save trace setting
-XTRACE=$(set +o | grep xtrace)
+_XTRACE_HORIZON=$(set +o | grep xtrace)
 set +o xtrace
 
 
@@ -193,7 +193,7 @@
 
 
 # Restore xtrace
-$XTRACE
+$_XTRACE_HORIZON
 
 # Tell emacs to use shell-script-mode
 ## Local variables:
diff --git a/lib/infra b/lib/infra
index ab32efe..cf003cc 100644
--- a/lib/infra
+++ b/lib/infra
@@ -15,7 +15,7 @@
 # - install_infra
 
 # Save trace setting
-XTRACE=$(set +o | grep xtrace)
+_XTRACE_INFRA=$(set +o | grep xtrace)
 set +o xtrace
 
 
@@ -50,7 +50,7 @@
 }
 
 # Restore xtrace
-$XTRACE
+$_XTRACE_INFRA
 
 # Tell emacs to use shell-script-mode
 ## Local variables:
diff --git a/lib/ironic b/lib/ironic
deleted file mode 100644
index 016e639..0000000
--- a/lib/ironic
+++ /dev/null
@@ -1,864 +0,0 @@
-#!/bin/bash
-#
-# lib/ironic
-# Functions to control the configuration and operation of the **Ironic** service
-
-# Dependencies:
-#
-# - ``functions`` file
-# - ``DEST``, ``DATA_DIR``, ``STACK_USER`` must be defined
-# - ``SERVICE_{TENANT_NAME|PASSWORD}`` must be defined
-# - ``SERVICE_HOST``
-# - ``KEYSTONE_TOKEN_FORMAT`` must be defined
-
-# ``stack.sh`` calls the entry points in this order:
-#
-# - install_ironic
-# - install_ironicclient
-# - init_ironic
-# - start_ironic
-# - stop_ironic
-# - cleanup_ironic
-
-# Save trace and pipefail settings
-XTRACE=$(set +o | grep xtrace)
-PIPEFAIL=$(set +o | grep pipefail)
-set +o xtrace
-set +o pipefail
-
-# Defaults
-# --------
-
-# Set up default directories
-GITDIR["python-ironicclient"]=$DEST/python-ironicclient
-GITDIR["ironic-lib"]=$DEST/ironic-lib
-
-IRONIC_DIR=$DEST/ironic
-IRONIC_PYTHON_AGENT_DIR=$DEST/ironic-python-agent
-IRONIC_DATA_DIR=$DATA_DIR/ironic
-IRONIC_STATE_PATH=/var/lib/ironic
-IRONIC_AUTH_CACHE_DIR=${IRONIC_AUTH_CACHE_DIR:-/var/cache/ironic}
-IRONIC_CONF_DIR=${IRONIC_CONF_DIR:-/etc/ironic}
-IRONIC_CONF_FILE=$IRONIC_CONF_DIR/ironic.conf
-IRONIC_ROOTWRAP_CONF=$IRONIC_CONF_DIR/rootwrap.conf
-IRONIC_POLICY_JSON=$IRONIC_CONF_DIR/policy.json
-
-# Deploy callback timeout can be changed from its default (1800), if required.
-IRONIC_CALLBACK_TIMEOUT=${IRONIC_CALLBACK_TIMEOUT:-}
-
-# Deploy to hardware platform
-IRONIC_HW_NODE_CPU=${IRONIC_HW_NODE_CPU:-1}
-IRONIC_HW_NODE_RAM=${IRONIC_HW_NODE_RAM:-512}
-IRONIC_HW_NODE_DISK=${IRONIC_HW_NODE_DISK:-10}
-IRONIC_HW_EPHEMERAL_DISK=${IRONIC_HW_EPHEMERAL_DISK:-0}
-# The file is composed of multiple lines, each line includes four field
-# separated by white space: IPMI address, MAC address, IPMI username
-# and IPMI password.
-#
-#   192.168.110.107 00:1e:67:57:50:4c root otc123
-IRONIC_IPMIINFO_FILE=${IRONIC_IPMIINFO_FILE:-$IRONIC_DATA_DIR/hardware_info}
-
-# Set up defaults for functional / integration testing
-IRONIC_NODE_UUID=${IRONIC_NODE_UUID:-`uuidgen`}
-IRONIC_SCRIPTS_DIR=${IRONIC_SCRIPTS_DIR:-$TOP_DIR/tools/ironic/scripts}
-IRONIC_TEMPLATES_DIR=${IRONIC_TEMPLATES_DIR:-$TOP_DIR/tools/ironic/templates}
-IRONIC_BAREMETAL_BASIC_OPS=$(trueorfalse False IRONIC_BAREMETAL_BASIC_OPS)
-IRONIC_ENABLED_DRIVERS=${IRONIC_ENABLED_DRIVERS:-fake,pxe_ssh,pxe_ipmitool}
-IRONIC_SSH_USERNAME=${IRONIC_SSH_USERNAME:-`whoami`}
-IRONIC_SSH_TIMEOUT=${IRONIC_SSH_TIMEOUT:-15}
-IRONIC_SSH_KEY_DIR=${IRONIC_SSH_KEY_DIR:-$IRONIC_DATA_DIR/ssh_keys}
-IRONIC_SSH_KEY_FILENAME=${IRONIC_SSH_KEY_FILENAME:-ironic_key}
-IRONIC_KEY_FILE=${IRONIC_KEY_FILE:-$IRONIC_SSH_KEY_DIR/$IRONIC_SSH_KEY_FILENAME}
-IRONIC_SSH_VIRT_TYPE=${IRONIC_SSH_VIRT_TYPE:-virsh}
-IRONIC_TFTPBOOT_DIR=${IRONIC_TFTPBOOT_DIR:-$IRONIC_DATA_DIR/tftpboot}
-IRONIC_TFTPSERVER_IP=${IRONIC_TFTPSERVER_IP:-$HOST_IP}
-IRONIC_VM_SSH_PORT=${IRONIC_VM_SSH_PORT:-22}
-IRONIC_VM_SSH_ADDRESS=${IRONIC_VM_SSH_ADDRESS:-$HOST_IP}
-IRONIC_VM_COUNT=${IRONIC_VM_COUNT:-1}
-IRONIC_VM_SPECS_CPU=${IRONIC_VM_SPECS_CPU:-1}
-IRONIC_VM_SPECS_RAM=${IRONIC_VM_SPECS_RAM:-512}
-IRONIC_VM_SPECS_DISK=${IRONIC_VM_SPECS_DISK:-10}
-IRONIC_VM_EPHEMERAL_DISK=${IRONIC_VM_EPHEMERAL_DISK:-0}
-IRONIC_VM_EMULATOR=${IRONIC_VM_EMULATOR:-/usr/bin/qemu-system-x86_64}
-IRONIC_VM_NETWORK_BRIDGE=${IRONIC_VM_NETWORK_BRIDGE:-brbm}
-IRONIC_VM_NETWORK_RANGE=${IRONIC_VM_NETWORK_RANGE:-192.0.2.0/24}
-IRONIC_VM_MACS_CSV_FILE=${IRONIC_VM_MACS_CSV_FILE:-$IRONIC_DATA_DIR/ironic_macs.csv}
-IRONIC_AUTHORIZED_KEYS_FILE=${IRONIC_AUTHORIZED_KEYS_FILE:-$HOME/.ssh/authorized_keys}
-
-# By default, baremetal VMs will console output to file.
-IRONIC_VM_LOG_CONSOLE=${IRONIC_VM_LOG_CONSOLE:-True}
-IRONIC_VM_LOG_DIR=${IRONIC_VM_LOG_DIR:-$IRONIC_DATA_DIR/logs/}
-
-# Use DIB to create deploy ramdisk and kernel.
-IRONIC_BUILD_DEPLOY_RAMDISK=$(trueorfalse True IRONIC_BUILD_DEPLOY_RAMDISK)
-# If not use DIB, these files are used as deploy ramdisk/kernel.
-# (The value must be a absolute path)
-IRONIC_DEPLOY_RAMDISK=${IRONIC_DEPLOY_RAMDISK:-}
-IRONIC_DEPLOY_KERNEL=${IRONIC_DEPLOY_KERNEL:-}
-IRONIC_DEPLOY_ELEMENT=${IRONIC_DEPLOY_ELEMENT:-deploy-ironic}
-
-IRONIC_AGENT_KERNEL_URL=${IRONIC_AGENT_KERNEL_URL:-http://tarballs.openstack.org/ironic-python-agent/coreos/files/coreos_production_pxe.vmlinuz}
-IRONIC_AGENT_RAMDISK_URL=${IRONIC_AGENT_RAMDISK_URL:-http://tarballs.openstack.org/ironic-python-agent/coreos/files/coreos_production_pxe_image-oem.cpio.gz}
-
-# Which deploy driver to use - valid choices right now
-# are ``pxe_ssh``, ``pxe_ipmitool``, ``agent_ssh`` and ``agent_ipmitool``.
-IRONIC_DEPLOY_DRIVER=${IRONIC_DEPLOY_DRIVER:-pxe_ssh}
-
-# TODO(agordeev): replace 'ubuntu' with host distro name getting
-IRONIC_DEPLOY_FLAVOR=${IRONIC_DEPLOY_FLAVOR:-ubuntu $IRONIC_DEPLOY_ELEMENT}
-
-# Support entry points installation of console scripts
-IRONIC_BIN_DIR=$(get_python_exec_prefix)
-
-# Ironic connection info.  Note the port must be specified.
-IRONIC_SERVICE_PROTOCOL=http
-IRONIC_SERVICE_PORT=${IRONIC_SERVICE_PORT:-6385}
-IRONIC_HOSTPORT=${IRONIC_HOSTPORT:-$SERVICE_HOST:$IRONIC_SERVICE_PORT}
-
-# Enable iPXE
-IRONIC_IPXE_ENABLED=$(trueorfalse False IRONIC_IPXE_ENABLED)
-IRONIC_HTTP_DIR=${IRONIC_HTTP_DIR:-$IRONIC_DATA_DIR/httpboot}
-IRONIC_HTTP_SERVER=${IRONIC_HTTP_SERVER:-$HOST_IP}
-IRONIC_HTTP_PORT=${IRONIC_HTTP_PORT:-8088}
-
-# NOTE(lucasagomes): This flag is used to differentiate the nodes that
-# uses IPA as their deploy ramdisk from nodes that uses the agent_* drivers
-# (which also uses IPA but depends on Swift Temp URLs to work). At present,
-# all drivers that uses the iSCSI approach for their deployment supports
-# using both, IPA or bash ramdisks for the deployment. In the future we
-# want to remove the support for the bash ramdisk in favor of IPA, once
-# we get there this flag can be removed, and all conditionals that uses
-# it should just run by default.
-IRONIC_DEPLOY_DRIVER_ISCSI_WITH_IPA=$(trueorfalse False IRONIC_DEPLOY_DRIVER_ISCSI_WITH_IPA)
-
-# get_pxe_boot_file() - Get the PXE/iPXE boot file path
-function get_pxe_boot_file {
-    local relpath=syslinux/pxelinux.0
-    if [[ "$IRONIC_IPXE_ENABLED" == "True" ]] ; then
-        relpath=ipxe/undionly.kpxe
-    fi
-
-    local pxe_boot_file
-    if is_ubuntu; then
-        pxe_boot_file=/usr/lib/$relpath
-    elif is_fedora || is_suse; then
-        pxe_boot_file=/usr/share/$relpath
-    fi
-
-    echo $pxe_boot_file
-}
-
-# PXE boot image
-IRONIC_PXE_BOOT_IMAGE=${IRONIC_PXE_BOOT_IMAGE:-$(get_pxe_boot_file)}
-
-
-# Functions
-# ---------
-
-# Test if any Ironic services are enabled
-# is_ironic_enabled
-function is_ironic_enabled {
-    [[ ,${ENABLED_SERVICES} =~ ,"ir-" ]] && return 0
-    return 1
-}
-
-function is_ironic_hardware {
-    is_ironic_enabled && [[ -n "${IRONIC_DEPLOY_DRIVER##*_ssh}" ]] && return 0
-    return 1
-}
-
-function is_deployed_by_agent {
-    [[ -z "${IRONIC_DEPLOY_DRIVER%%agent*}" ]] && return 0
-    return 1
-}
-
-function is_deployed_with_ipa_ramdisk {
-    is_deployed_by_agent || [[ "$IRONIC_DEPLOY_DRIVER_ISCSI_WITH_IPA" == "True" ]] && return 0
-    return 1
-}
-
-# install_ironic() - Collect source and prepare
-function install_ironic {
-    # make sure all needed service were enabled
-    local req_services="key"
-    if [[ "$VIRT_DRIVER" == "ironic" ]]; then
-        req_services+=" nova glance neutron"
-    fi
-    for srv in $req_services; do
-        if ! is_service_enabled "$srv"; then
-            die $LINENO "$srv should be enabled for Ironic."
-        fi
-    done
-
-    if use_library_from_git "ironic-lib"; then
-        git_clone_by_name "ironic-lib"
-        setup_dev_lib "ironic-lib"
-    fi
-
-    git_clone $IRONIC_REPO $IRONIC_DIR $IRONIC_BRANCH
-    setup_develop $IRONIC_DIR
-
-    if [[ "$IRONIC_IPXE_ENABLED" == "True" ]] ; then
-        install_apache_wsgi
-    fi
-}
-
-# install_ironicclient() - Collect sources and prepare
-function install_ironicclient {
-    if use_library_from_git "python-ironicclient"; then
-        git_clone_by_name "python-ironicclient"
-        setup_dev_lib "python-ironicclient"
-        sudo install -D -m 0644 -o $STACK_USER {${GITDIR["python-ironicclient"]}/tools/,/etc/bash_completion.d/}ironic.bash_completion
-    else
-        # nothing actually "requires" ironicclient, so force instally from pypi
-        pip_install_gr python-ironicclient
-    fi
-}
-
-# _cleanup_ironic_apache_wsgi() - Remove wsgi files, disable and remove apache vhost file
-function _cleanup_ironic_apache_wsgi {
-    sudo rm -rf $IRONIC_HTTP_DIR
-    disable_apache_site ironic
-    sudo rm -f $(apache_site_config_for ironic)
-    restart_apache_server
-}
-
-# _config_ironic_apache_wsgi() - Set WSGI config files of Ironic
-function _config_ironic_apache_wsgi {
-    local ironic_apache_conf
-    ironic_apache_conf=$(apache_site_config_for ironic)
-    sudo cp $FILES/apache-ironic.template $ironic_apache_conf
-    sudo sed -e "
-        s|%PUBLICPORT%|$IRONIC_HTTP_PORT|g;
-        s|%HTTPROOT%|$IRONIC_HTTP_DIR|g;
-    " -i $ironic_apache_conf
-    enable_apache_site ironic
-}
-
-# cleanup_ironic() - Remove residual data files, anything left over from previous
-# runs that would need to clean up.
-function cleanup_ironic {
-    sudo rm -rf $IRONIC_AUTH_CACHE_DIR $IRONIC_CONF_DIR
-}
-
-# configure_ironic_dirs() - Create all directories required by Ironic and
-# associated services.
-function configure_ironic_dirs {
-    sudo install -d -o $STACK_USER $IRONIC_CONF_DIR $STACK_USER $IRONIC_DATA_DIR \
-        $IRONIC_STATE_PATH $IRONIC_TFTPBOOT_DIR $IRONIC_TFTPBOOT_DIR/pxelinux.cfg
-    sudo chown -R $STACK_USER:$LIBVIRT_GROUP $IRONIC_TFTPBOOT_DIR
-
-    if [[ "$IRONIC_IPXE_ENABLED" == "True" ]] ; then
-        sudo install -d -o $STACK_USER -g $LIBVIRT_GROUP $IRONIC_HTTP_DIR
-    fi
-
-    if [ ! -f $IRONIC_PXE_BOOT_IMAGE ]; then
-        die $LINENO "PXE boot file $IRONIC_PXE_BOOT_IMAGE not found."
-    fi
-
-    # Copy PXE binary
-    if [[ "$IRONIC_IPXE_ENABLED" == "True" ]] ; then
-        cp $IRONIC_PXE_BOOT_IMAGE $IRONIC_TFTPBOOT_DIR
-    else
-        # Syslinux >= 5.00 pxelinux.0 binary is not "stand-alone" anymore,
-        # it depends on some c32 modules to work correctly.
-        # More info: http://www.syslinux.org/wiki/index.php/Library_modules
-        cp -aR $(dirname $IRONIC_PXE_BOOT_IMAGE)/*.{c32,0} $IRONIC_TFTPBOOT_DIR
-    fi
-}
-
-# configure_ironic() - Set config files, create data dirs, etc
-function configure_ironic {
-    configure_ironic_dirs
-
-    # Copy over ironic configuration file and configure common parameters.
-    cp $IRONIC_DIR/etc/ironic/ironic.conf.sample $IRONIC_CONF_FILE
-    iniset $IRONIC_CONF_FILE DEFAULT debug True
-    inicomment $IRONIC_CONF_FILE DEFAULT log_file
-    iniset $IRONIC_CONF_FILE database connection `database_connection_url ironic`
-    iniset $IRONIC_CONF_FILE DEFAULT state_path $IRONIC_STATE_PATH
-    iniset $IRONIC_CONF_FILE DEFAULT use_syslog $SYSLOG
-    # Configure Ironic conductor, if it was enabled.
-    if is_service_enabled ir-cond; then
-        configure_ironic_conductor
-    fi
-
-    # Configure Ironic API, if it was enabled.
-    if is_service_enabled ir-api; then
-        configure_ironic_api
-    fi
-
-    # Format logging
-    if [ "$LOG_COLOR" == "True" ] && [ "$SYSLOG" == "False" ]; then
-        setup_colorized_logging $IRONIC_CONF_FILE DEFAULT tenant user
-    fi
-
-    if [[ "$IRONIC_IPXE_ENABLED" == "True" ]]; then
-        _config_ironic_apache_wsgi
-    fi
-}
-
-# configure_ironic_api() - Is used by configure_ironic(). Performs
-# API specific configuration.
-function configure_ironic_api {
-    iniset $IRONIC_CONF_FILE DEFAULT auth_strategy keystone
-    iniset $IRONIC_CONF_FILE oslo_policy policy_file $IRONIC_POLICY_JSON
-
-    # TODO(Yuki Nishiwaki): This is a temporary work-around until Ironic is fixed(bug#1422632).
-    # These codes need to be changed to use the function of configure_auth_token_middleware
-    # after Ironic conforms to the new auth plugin.
-    iniset $IRONIC_CONF_FILE keystone_authtoken identity_uri $KEYSTONE_AUTH_URI
-    iniset $IRONIC_CONF_FILE keystone_authtoken auth_uri $KEYSTONE_SERVICE_URI/v2.0
-    iniset $IRONIC_CONF_FILE keystone_authtoken admin_user ironic
-    iniset $IRONIC_CONF_FILE keystone_authtoken admin_password $SERVICE_PASSWORD
-    iniset $IRONIC_CONF_FILE keystone_authtoken admin_tenant_name $SERVICE_TENANT_NAME
-    iniset $IRONIC_CONF_FILE keystone_authtoken cafile $SSL_BUNDLE_FILE
-    iniset $IRONIC_CONF_FILE keystone_authtoken signing_dir $IRONIC_AUTH_CACHE_DIR/api
-
-    iniset_rpc_backend ironic $IRONIC_CONF_FILE
-    iniset $IRONIC_CONF_FILE api port $IRONIC_SERVICE_PORT
-
-    cp -p $IRONIC_DIR/etc/ironic/policy.json $IRONIC_POLICY_JSON
-}
-
-# configure_ironic_conductor() - Is used by configure_ironic().
-# Sets conductor specific settings.
-function configure_ironic_conductor {
-    cp $IRONIC_DIR/etc/ironic/rootwrap.conf $IRONIC_ROOTWRAP_CONF
-    cp -r $IRONIC_DIR/etc/ironic/rootwrap.d $IRONIC_CONF_DIR
-    local ironic_rootwrap
-    ironic_rootwrap=$(get_rootwrap_location ironic)
-    local rootwrap_isudoer_cmd="$ironic_rootwrap $IRONIC_CONF_DIR/rootwrap.conf *"
-
-    # Set up the rootwrap sudoers for ironic
-    local tempfile
-    tempfile=`mktemp`
-    echo "$STACK_USER ALL=(root) NOPASSWD: $rootwrap_isudoer_cmd" >$tempfile
-    chmod 0440 $tempfile
-    sudo chown root:root $tempfile
-    sudo mv $tempfile /etc/sudoers.d/ironic-rootwrap
-
-    iniset $IRONIC_CONF_FILE DEFAULT rootwrap_config $IRONIC_ROOTWRAP_CONF
-    iniset $IRONIC_CONF_FILE DEFAULT enabled_drivers $IRONIC_ENABLED_DRIVERS
-    iniset $IRONIC_CONF_FILE conductor api_url $IRONIC_SERVICE_PROTOCOL://$HOST_IP:$IRONIC_SERVICE_PORT
-    if [[ -n "$IRONIC_CALLBACK_TIMEOUT" ]]; then
-        iniset $IRONIC_CONF_FILE conductor deploy_callback_timeout $IRONIC_CALLBACK_TIMEOUT
-    fi
-    iniset $IRONIC_CONF_FILE pxe tftp_server $IRONIC_TFTPSERVER_IP
-    iniset $IRONIC_CONF_FILE pxe tftp_root $IRONIC_TFTPBOOT_DIR
-    iniset $IRONIC_CONF_FILE pxe tftp_master_path $IRONIC_TFTPBOOT_DIR/master_images
-
-    local pxe_params=""
-    if [[ "$IRONIC_VM_LOG_CONSOLE" == "True" ]] ; then
-        pxe_params+="nofb nomodeset vga=normal console=ttyS0"
-        if is_deployed_with_ipa_ramdisk; then
-            pxe_params+=" systemd.journald.forward_to_console=yes"
-        fi
-    fi
-    # When booting with less than 1GB, we need to switch from default tmpfs
-    # to ramfs for ramdisks to decompress successfully.
-    if (is_ironic_hardware && [[ "$IRONIC_HW_NODE_RAM" -lt 1024 ]]) ||
-        (! is_ironic_hardware && [[ "$IRONIC_VM_SPECS_RAM" -lt 1024 ]]); then
-        pxe_params+=" rootfstype=ramfs"
-    fi
-    if [[ -n "$pxe_params" ]]; then
-        iniset $IRONIC_CONF_FILE pxe pxe_append_params "$pxe_params"
-    fi
-
-    if is_deployed_by_agent; then
-        if [[ "$SWIFT_ENABLE_TEMPURLS" == "True" ]] ; then
-            iniset $IRONIC_CONF_FILE glance swift_temp_url_key $SWIFT_TEMPURL_KEY
-        else
-            die $LINENO "SWIFT_ENABLE_TEMPURLS must be True to use agent_ssh driver in Ironic."
-        fi
-        iniset $IRONIC_CONF_FILE glance swift_endpoint_url http://${HOST_IP}:${SWIFT_DEFAULT_BIND_PORT:-8080}
-        iniset $IRONIC_CONF_FILE glance swift_api_version v1
-        local tenant_id
-        tenant_id=$(get_or_create_project $SERVICE_TENANT_NAME default)
-        iniset $IRONIC_CONF_FILE glance swift_account AUTH_${tenant_id}
-        iniset $IRONIC_CONF_FILE glance swift_container glance
-        iniset $IRONIC_CONF_FILE glance swift_temp_url_duration 3600
-        iniset $IRONIC_CONF_FILE agent heartbeat_timeout 30
-        iniset $IRONIC_CONF_FILE agent agent_erase_devices_priority 0
-    fi
-
-    if [[ "$IRONIC_IPXE_ENABLED" == "True" ]] ; then
-        local pxebin
-        pxebin=`basename $IRONIC_PXE_BOOT_IMAGE`
-        iniset $IRONIC_CONF_FILE pxe ipxe_enabled True
-        iniset $IRONIC_CONF_FILE pxe pxe_config_template '\$pybasedir/drivers/modules/ipxe_config.template'
-        iniset $IRONIC_CONF_FILE pxe pxe_bootfile_name $pxebin
-        iniset $IRONIC_CONF_FILE pxe http_root $IRONIC_HTTP_DIR
-        iniset $IRONIC_CONF_FILE pxe http_url "http://$IRONIC_HTTP_SERVER:$IRONIC_HTTP_PORT"
-    fi
-}
-
-# create_ironic_cache_dir() - Part of the init_ironic() process
-function create_ironic_cache_dir {
-    # Create cache dir
-    sudo mkdir -p $IRONIC_AUTH_CACHE_DIR/api
-    sudo chown $STACK_USER $IRONIC_AUTH_CACHE_DIR/api
-    rm -f $IRONIC_AUTH_CACHE_DIR/api/*
-    sudo mkdir -p $IRONIC_AUTH_CACHE_DIR/registry
-    sudo chown $STACK_USER $IRONIC_AUTH_CACHE_DIR/registry
-    rm -f $IRONIC_AUTH_CACHE_DIR/registry/*
-}
-
-# create_ironic_accounts() - Set up common required ironic accounts
-
-# Tenant               User       Roles
-# ------------------------------------------------------------------
-# service              ironic     admin        # if enabled
-function create_ironic_accounts {
-
-    # Ironic
-    if [[ "$ENABLED_SERVICES" =~ "ir-api" ]]; then
-        # Get ironic user if exists
-
-        # NOTE(Shrews): This user MUST have admin level privileges!
-        create_service_user "ironic" "admin"
-
-        if [[ "$KEYSTONE_CATALOG_BACKEND" = 'sql' ]]; then
-
-            get_or_create_service "ironic" "baremetal" "Ironic baremetal provisioning service"
-            get_or_create_endpoint "baremetal" \
-                "$REGION_NAME" \
-                "$IRONIC_SERVICE_PROTOCOL://$IRONIC_HOSTPORT" \
-                "$IRONIC_SERVICE_PROTOCOL://$IRONIC_HOSTPORT" \
-                "$IRONIC_SERVICE_PROTOCOL://$IRONIC_HOSTPORT"
-        fi
-    fi
-}
-
-
-# init_ironic() - Initialize databases, etc.
-function init_ironic {
-    # Save private network as cleaning network
-    local cleaning_network_uuid
-    cleaning_network_uuid=$(neutron net-list | grep private | get_field 1)
-    iniset $IRONIC_CONF_FILE neutron cleaning_network_uuid ${cleaning_network_uuid}
-
-    # (Re)create  ironic database
-    recreate_database ironic
-
-    # Migrate ironic database
-    $IRONIC_BIN_DIR/ironic-dbsync --config-file=$IRONIC_CONF_FILE
-
-    create_ironic_cache_dir
-}
-
-# _ironic_bm_vm_names() - Generates list of names for baremetal VMs.
-function _ironic_bm_vm_names {
-    local idx
-    local num_vms
-    num_vms=$(($IRONIC_VM_COUNT - 1))
-    for idx in $(seq 0 $num_vms); do
-        echo "baremetal${IRONIC_VM_NETWORK_BRIDGE}_${idx}"
-    done
-}
-
-# start_ironic() - Start running processes, including screen
-function start_ironic {
-    # Start Ironic API server, if enabled.
-    if is_service_enabled ir-api; then
-        start_ironic_api
-    fi
-
-    # Start Ironic conductor, if enabled.
-    if is_service_enabled ir-cond; then
-        start_ironic_conductor
-    fi
-
-    # Start Apache if iPXE is enabled
-    if [[ "$IRONIC_IPXE_ENABLED" == "True" ]] ; then
-        restart_apache_server
-    fi
-}
-
-# start_ironic_api() - Used by start_ironic().
-# Starts Ironic API server.
-function start_ironic_api {
-    run_process ir-api "$IRONIC_BIN_DIR/ironic-api --config-file=$IRONIC_CONF_FILE"
-    echo "Waiting for ir-api ($IRONIC_HOSTPORT) to start..."
-    if ! timeout $SERVICE_TIMEOUT sh -c "while ! wget --no-proxy -q -O- $IRONIC_SERVICE_PROTOCOL://$IRONIC_HOSTPORT; do sleep 1; done"; then
-        die $LINENO "ir-api did not start"
-    fi
-}
-
-# start_ironic_conductor() - Used by start_ironic().
-# Starts Ironic conductor.
-function start_ironic_conductor {
-    run_process ir-cond "$IRONIC_BIN_DIR/ironic-conductor --config-file=$IRONIC_CONF_FILE"
-    # TODO(romcheg): Find a way to check whether the conductor has started.
-}
-
-# stop_ironic() - Stop running processes
-function stop_ironic {
-    stop_process ir-api
-    stop_process ir-cond
-
-    # Cleanup the WSGI files
-    if [[ "$IRONIC_IPXE_ENABLED" == "True" ]] ; then
-        _cleanup_ironic_apache_wsgi
-    fi
-}
-
-function create_ovs_taps {
-    local ironic_net_id
-    ironic_net_id=$(neutron net-list | grep private | get_field 1)
-
-    # Work around: No netns exists on host until a Neutron port is created.  We
-    # need to create one in Neutron to know what netns to tap into prior to the
-    # first node booting.
-    local port_id
-    port_id=$(neutron port-create private | grep " id " | get_field 2)
-
-    # intentional sleep to make sure the tag has been set to port
-    sleep 10
-
-    local tapdev
-    tapdev=$(sudo ip netns exec qdhcp-${ironic_net_id} ip link list | grep " tap" | cut -d':' -f2 | cut -d'@' -f1 | cut -b2-)
-    local tag_id
-    tag_id=$(sudo ovs-vsctl show |grep ${tapdev} -A1 -m1 | grep tag | cut -d':' -f2 | cut -b2-)
-
-    # make sure veth pair is not existing, otherwise delete its links
-    sudo ip link show ovs-tap1 && sudo ip link delete ovs-tap1
-    sudo ip link show brbm-tap1 && sudo ip link delete brbm-tap1
-    # create veth pair for future interconnection between br-int and brbm
-    sudo ip link add brbm-tap1 type veth peer name ovs-tap1
-    sudo ip link set dev brbm-tap1 up
-    sudo ip link set dev ovs-tap1 up
-
-    sudo ovs-vsctl -- --if-exists del-port ovs-tap1 -- add-port br-int ovs-tap1 tag=$tag_id
-    sudo ovs-vsctl -- --if-exists del-port brbm-tap1 -- add-port $IRONIC_VM_NETWORK_BRIDGE brbm-tap1
-
-    # Remove the port needed only for workaround.
-    neutron port-delete $port_id
-
-    # Finally, share the fixed tenant network across all tenants.  This allows the host
-    # to serve TFTP to a single network namespace via the tap device created above.
-    neutron net-update $ironic_net_id --shared true
-}
-
-function create_bridge_and_vms {
-    # Call libvirt setup scripts in a new shell to ensure any new group membership
-    sudo su $STACK_USER -c "$IRONIC_SCRIPTS_DIR/setup-network"
-    if [[ "$IRONIC_VM_LOG_CONSOLE" == "True" ]] ; then
-        local log_arg="$IRONIC_VM_LOG_DIR"
-    else
-        local log_arg=""
-    fi
-    local vm_name
-    for vm_name in $(_ironic_bm_vm_names); do
-        sudo su $STACK_USER -c "$IRONIC_SCRIPTS_DIR/create-node $vm_name \
-            $IRONIC_VM_SPECS_CPU $IRONIC_VM_SPECS_RAM $IRONIC_VM_SPECS_DISK \
-            amd64 $IRONIC_VM_NETWORK_BRIDGE $IRONIC_VM_EMULATOR \
-            $log_arg" >> $IRONIC_VM_MACS_CSV_FILE
-    done
-    create_ovs_taps
-}
-
-function wait_for_nova_resources {
-    # After nodes have been enrolled, we need to wait for both ironic and
-    # nova's periodic tasks to populate the resource tracker with available
-    # nodes and resources. Wait up to 2 minutes for a given resource before
-    # timing out.
-    local resource=$1
-    local expected_count=$2
-    local i
-    echo_summary "Waiting 2 minutes for Nova resource tracker to pick up $resource >= $expected_count"
-    for i in $(seq 1 120); do
-        if [ $(nova hypervisor-stats | grep " $resource " | get_field 2) -ge $expected_count ]; then
-            return 0
-        fi
-        sleep 1
-    done
-    die $LINENO "Timed out waiting for Nova hypervisor-stats $resource >= $expected_count"
-}
-
-function enroll_nodes {
-    local chassis_id
-    chassis_id=$(ironic chassis-create -d "ironic test chassis" | grep " uuid " | get_field 2)
-
-    if ! is_ironic_hardware; then
-        local ironic_node_cpu=$IRONIC_VM_SPECS_CPU
-        local ironic_node_ram=$IRONIC_VM_SPECS_RAM
-        local ironic_node_disk=$IRONIC_VM_SPECS_DISK
-        local ironic_ephemeral_disk=$IRONIC_VM_EPHEMERAL_DISK
-        local ironic_hwinfo_file=$IRONIC_VM_MACS_CSV_FILE
-        local node_options="\
-            -i deploy_kernel=$IRONIC_DEPLOY_KERNEL_ID \
-            -i deploy_ramdisk=$IRONIC_DEPLOY_RAMDISK_ID \
-            -i ssh_virt_type=$IRONIC_SSH_VIRT_TYPE \
-            -i ssh_address=$IRONIC_VM_SSH_ADDRESS \
-            -i ssh_port=$IRONIC_VM_SSH_PORT \
-            -i ssh_username=$IRONIC_SSH_USERNAME \
-            -i ssh_key_filename=$IRONIC_KEY_FILE"
-    else
-        local ironic_node_cpu=$IRONIC_HW_NODE_CPU
-        local ironic_node_ram=$IRONIC_HW_NODE_RAM
-        local ironic_node_disk=$IRONIC_HW_NODE_DISK
-        local ironic_ephemeral_disk=$IRONIC_HW_EPHEMERAL_DISK
-        if [[ -z "${IRONIC_DEPLOY_DRIVER##*_ipmitool}" ]]; then
-            local ironic_hwinfo_file=$IRONIC_IPMIINFO_FILE
-        fi
-    fi
-
-    local total_nodes=0
-    local total_cpus=0
-    while read hardware_info; do
-        if ! is_ironic_hardware; then
-            local mac_address=$hardware_info
-        elif [[ -z "${IRONIC_DEPLOY_DRIVER##*_ipmitool}" ]]; then
-            local ipmi_address
-            ipmi_address=$(echo $hardware_info |awk  '{print $1}')
-            local mac_address
-            mac_address=$(echo $hardware_info |awk '{print $2}')
-            local ironic_ipmi_username
-            ironic_ipmi_username=$(echo $hardware_info |awk '{print $3}')
-            local ironic_ipmi_passwd
-            ironic_ipmi_passwd=$(echo $hardware_info |awk '{print $4}')
-            # Currently we require all hardware platform have same CPU/RAM/DISK info
-            # in future, this can be enhanced to support different type, and then
-            # we create the bare metal flavor with minimum value
-            local node_options="-i ipmi_address=$ipmi_address -i ipmi_password=$ironic_ipmi_passwd\
-                -i ipmi_username=$ironic_ipmi_username"
-            node_options+=" -i deploy_kernel=$IRONIC_DEPLOY_KERNEL_ID"
-            node_options+=" -i deploy_ramdisk=$IRONIC_DEPLOY_RAMDISK_ID"
-        fi
-
-        # First node created will be used for testing in ironic w/o glance
-        # scenario, so we need to know its UUID.
-        local standalone_node_uuid=""
-        if [ $total_nodes -eq 0 ]; then
-            standalone_node_uuid="--uuid $IRONIC_NODE_UUID"
-        fi
-
-        local node_id
-        node_id=$(ironic node-create $standalone_node_uuid\
-            --chassis_uuid $chassis_id \
-            --driver $IRONIC_DEPLOY_DRIVER \
-            --name node-$total_nodes \
-            -p cpus=$ironic_node_cpu\
-            -p memory_mb=$ironic_node_ram\
-            -p local_gb=$ironic_node_disk\
-            -p cpu_arch=x86_64 \
-            $node_options \
-            | grep " uuid " | get_field 2)
-
-        ironic port-create --address $mac_address --node $node_id
-
-        total_nodes=$((total_nodes+1))
-        total_cpus=$((total_cpus+$ironic_node_cpu))
-    done < $ironic_hwinfo_file
-
-    # create the nova flavor
-    # NOTE(adam_g): Attempting to use an autogenerated UUID for flavor id here uncovered
-    # bug (LP: #1333852) in Trove.  This can be changed to use an auto flavor id when the
-    # bug is fixed in Juno.
-    local adjusted_disk
-    adjusted_disk=$(($ironic_node_disk - $ironic_ephemeral_disk))
-    nova flavor-create --ephemeral $ironic_ephemeral_disk baremetal 551 $ironic_node_ram $adjusted_disk $ironic_node_cpu
-
-    nova flavor-key baremetal set "cpu_arch"="x86_64"
-
-    if [ "$VIRT_DRIVER" == "ironic" ]; then
-        wait_for_nova_resources "count" $total_nodes
-        wait_for_nova_resources "vcpus" $total_cpus
-    fi
-}
-
-function configure_iptables {
-    # enable tftp natting for allowing connections to HOST_IP's tftp server
-    sudo modprobe nf_conntrack_tftp
-    sudo modprobe nf_nat_tftp
-    # explicitly allow DHCP - packets are occassionally being dropped here
-    sudo iptables -I INPUT -p udp --dport 67:68 --sport 67:68 -j ACCEPT || true
-    # nodes boot from TFTP and callback to the API server listening on $HOST_IP
-    sudo iptables -I INPUT -d $HOST_IP -p udp --dport 69 -j ACCEPT || true
-    sudo iptables -I INPUT -d $HOST_IP -p tcp --dport $IRONIC_SERVICE_PORT -j ACCEPT || true
-    if is_deployed_by_agent; then
-        # agent ramdisk gets instance image from swift
-        sudo iptables -I INPUT -d $HOST_IP -p tcp --dport ${SWIFT_DEFAULT_BIND_PORT:-8080} -j ACCEPT || true
-    fi
-
-    if [[ "$IRONIC_IPXE_ENABLED" == "True" ]] ; then
-        sudo iptables -I INPUT -d $HOST_IP -p tcp --dport $IRONIC_HTTP_PORT -j ACCEPT || true
-    fi
-}
-
-function configure_tftpd {
-    # stop tftpd and setup serving via xinetd
-    stop_service tftpd-hpa || true
-    [ -f /etc/init/tftpd-hpa.conf ] && echo "manual" | sudo tee /etc/init/tftpd-hpa.override
-    sudo cp $IRONIC_TEMPLATES_DIR/tftpd-xinetd.template /etc/xinetd.d/tftp
-    sudo sed -e "s|%TFTPBOOT_DIR%|$IRONIC_TFTPBOOT_DIR|g" -i /etc/xinetd.d/tftp
-
-    # setup tftp file mapping to satisfy requests at the root (booting) and
-    # /tftpboot/ sub-dir (as per deploy-ironic elements)
-    echo "r ^([^/]) $IRONIC_TFTPBOOT_DIR/\1" >$IRONIC_TFTPBOOT_DIR/map-file
-    echo "r ^(/tftpboot/) $IRONIC_TFTPBOOT_DIR/\2" >>$IRONIC_TFTPBOOT_DIR/map-file
-
-    chmod -R 0755 $IRONIC_TFTPBOOT_DIR
-    restart_service xinetd
-}
-
-function configure_ironic_ssh_keypair {
-    if [[ ! -d $HOME/.ssh ]]; then
-        mkdir -p $HOME/.ssh
-        chmod 700 $HOME/.ssh
-    fi
-    if [[ ! -e $IRONIC_KEY_FILE ]]; then
-        if [[ ! -d $(dirname $IRONIC_KEY_FILE) ]]; then
-            mkdir -p $(dirname $IRONIC_KEY_FILE)
-        fi
-        echo -e 'n\n' | ssh-keygen -q -t rsa -P '' -f $IRONIC_KEY_FILE
-    fi
-    cat $IRONIC_KEY_FILE.pub | tee -a $IRONIC_AUTHORIZED_KEYS_FILE
-}
-
-function ironic_ssh_check {
-    local key_file=$1
-    local floating_ip=$2
-    local port=$3
-    local default_instance_user=$4
-    local active_timeout=$5
-    if ! timeout $active_timeout sh -c "while ! ssh -p $port -o StrictHostKeyChecking=no -i $key_file ${default_instance_user}@$floating_ip echo success; do sleep 1; done"; then
-        die $LINENO "server didn't become ssh-able!"
-    fi
-}
-
-function configure_ironic_auxiliary {
-    configure_ironic_ssh_keypair
-    ironic_ssh_check $IRONIC_KEY_FILE $IRONIC_VM_SSH_ADDRESS $IRONIC_VM_SSH_PORT $IRONIC_SSH_USERNAME $IRONIC_SSH_TIMEOUT
-}
-
-function build_ipa_coreos_ramdisk {
-    echo "Building ironic-python-agent deploy ramdisk"
-    local kernel_path=$1
-    local ramdisk_path=$2
-    git_clone $IRONIC_PYTHON_AGENT_REPO $IRONIC_PYTHON_AGENT_DIR $IRONIC_PYTHON_AGENT_BRANCH
-    cd $IRONIC_PYTHON_AGENT_DIR
-    imagebuild/coreos/build_coreos_image.sh
-    cp imagebuild/coreos/UPLOAD/coreos_production_pxe_image-oem.cpio.gz $ramdisk_path
-    cp imagebuild/coreos/UPLOAD/coreos_production_pxe.vmlinuz $kernel_path
-    sudo rm -rf UPLOAD
-    cd -
-}
-
-# build deploy kernel+ramdisk, then upload them to glance
-# this function sets ``IRONIC_DEPLOY_KERNEL_ID``, ``IRONIC_DEPLOY_RAMDISK_ID``
-function upload_baremetal_ironic_deploy {
-    declare -g IRONIC_DEPLOY_KERNEL_ID IRONIC_DEPLOY_RAMDISK_ID
-    echo_summary "Creating and uploading baremetal images for ironic"
-
-    # install diskimage-builder
-    if [[ $(type -P ramdisk-image-create) == "" ]]; then
-        pip_install_gr "diskimage-builder"
-    fi
-
-    if [ -z "$IRONIC_DEPLOY_KERNEL" -o -z "$IRONIC_DEPLOY_RAMDISK" ]; then
-        local IRONIC_DEPLOY_KERNEL_PATH=$TOP_DIR/files/ir-deploy-$IRONIC_DEPLOY_DRIVER.kernel
-        local IRONIC_DEPLOY_RAMDISK_PATH=$TOP_DIR/files/ir-deploy-$IRONIC_DEPLOY_DRIVER.initramfs
-    else
-        local IRONIC_DEPLOY_KERNEL_PATH=$IRONIC_DEPLOY_KERNEL
-        local IRONIC_DEPLOY_RAMDISK_PATH=$IRONIC_DEPLOY_RAMDISK
-    fi
-
-    if [ ! -e "$IRONIC_DEPLOY_RAMDISK_PATH" -o ! -e "$IRONIC_DEPLOY_KERNEL_PATH" ]; then
-        # files don't exist, need to build them
-        if [ "$IRONIC_BUILD_DEPLOY_RAMDISK" = "True" ]; then
-            # we can build them only if we're not offline
-            if [ "$OFFLINE" != "True" ]; then
-                if is_deployed_with_ipa_ramdisk; then
-                    build_ipa_coreos_ramdisk $IRONIC_DEPLOY_KERNEL_PATH $IRONIC_DEPLOY_RAMDISK_PATH
-                else
-                    ramdisk-image-create $IRONIC_DEPLOY_FLAVOR \
-                        -o $TOP_DIR/files/ir-deploy-$IRONIC_DEPLOY_DRIVER
-                fi
-            else
-                die $LINENO "Deploy kernel+ramdisk files don't exist and cannot be build in OFFLINE mode"
-            fi
-        else
-            if is_deployed_with_ipa_ramdisk; then
-                # download the agent image tarball
-                wget "$IRONIC_AGENT_KERNEL_URL" -O $IRONIC_DEPLOY_KERNEL_PATH
-                wget "$IRONIC_AGENT_RAMDISK_URL" -O $IRONIC_DEPLOY_RAMDISK_PATH
-            else
-                die $LINENO "Deploy kernel+ramdisk files don't exist and their building was disabled explicitly by IRONIC_BUILD_DEPLOY_RAMDISK"
-            fi
-        fi
-    fi
-
-    local token
-    token=$(openstack token issue -c id -f value)
-    die_if_not_set $LINENO token "Keystone fail to get token"
-
-    # load them into glance
-    IRONIC_DEPLOY_KERNEL_ID=$(openstack \
-        --os-token $token \
-        --os-url $GLANCE_SERVICE_PROTOCOL://$GLANCE_HOSTPORT \
-        image create \
-        $(basename $IRONIC_DEPLOY_KERNEL_PATH) \
-        --public --disk-format=aki \
-        --container-format=aki \
-        < $IRONIC_DEPLOY_KERNEL_PATH  | grep ' id ' | get_field 2)
-    IRONIC_DEPLOY_RAMDISK_ID=$(openstack \
-        --os-token $token \
-        --os-url $GLANCE_SERVICE_PROTOCOL://$GLANCE_HOSTPORT \
-        image create \
-        $(basename $IRONIC_DEPLOY_RAMDISK_PATH) \
-        --public --disk-format=ari \
-        --container-format=ari \
-        < $IRONIC_DEPLOY_RAMDISK_PATH  | grep ' id ' | get_field 2)
-}
-
-function prepare_baremetal_basic_ops {
-    if ! is_ironic_hardware; then
-        configure_ironic_auxiliary
-    fi
-    upload_baremetal_ironic_deploy
-    if ! is_ironic_hardware; then
-        create_bridge_and_vms
-    fi
-    enroll_nodes
-    configure_tftpd
-    configure_iptables
-}
-
-function cleanup_baremetal_basic_ops {
-    rm -f $IRONIC_VM_MACS_CSV_FILE
-    if [ -f $IRONIC_KEY_FILE ]; then
-        local key
-        key=$(cat $IRONIC_KEY_FILE.pub)
-        # remove public key from authorized_keys
-        grep -v "$key" $IRONIC_AUTHORIZED_KEYS_FILE > temp && mv temp $IRONIC_AUTHORIZED_KEYS_FILE
-        chmod 0600 $IRONIC_AUTHORIZED_KEYS_FILE
-    fi
-    sudo rm -rf $IRONIC_DATA_DIR $IRONIC_STATE_PATH
-
-    local vm_name
-    for vm_name in $(_ironic_bm_vm_names); do
-        sudo su $STACK_USER -c "$IRONIC_SCRIPTS_DIR/cleanup-node $vm_name $IRONIC_VM_NETWORK_BRIDGE"
-    done
-
-    sudo rm -rf /etc/xinetd.d/tftp /etc/init/tftpd-hpa.override
-    restart_service xinetd
-    sudo iptables -D INPUT -d $HOST_IP -p udp --dport 69 -j ACCEPT || true
-    sudo iptables -D INPUT -d $HOST_IP -p tcp --dport $IRONIC_SERVICE_PORT -j ACCEPT || true
-    if is_deployed_by_agent; then
-        # agent ramdisk gets instance image from swift
-        sudo iptables -D INPUT -d $HOST_IP -p tcp --dport ${SWIFT_DEFAULT_BIND_PORT:-8080} -j ACCEPT || true
-    fi
-    sudo rmmod nf_conntrack_tftp || true
-    sudo rmmod nf_nat_tftp || true
-}
-
-# Restore xtrace + pipefail
-$XTRACE
-$PIPEFAIL
-
-# Tell emacs to use shell-script-mode
-## Local variables:
-## mode: shell-script
-## End:
diff --git a/lib/keystone b/lib/keystone
index c484795..b4b7df9 100644
--- a/lib/keystone
+++ b/lib/keystone
@@ -28,7 +28,7 @@
 # - _cleanup_keystone_apache_wsgi
 
 # Save trace setting
-XTRACE=$(set +o | grep xtrace)
+_XTRACE_KEYSTONE=$(set +o | grep xtrace)
 set +o xtrace
 
 # Defaults
@@ -52,10 +52,6 @@
 KEYSTONE_CONF=$KEYSTONE_CONF_DIR/keystone.conf
 KEYSTONE_PASTE_INI=${KEYSTONE_PASTE_INI:-$KEYSTONE_CONF_DIR/keystone-paste.ini}
 
-# Set up additional extensions, such as oauth1, federation
-# Example of KEYSTONE_EXTENSIONS=oauth1,federation
-KEYSTONE_EXTENSIONS=${KEYSTONE_EXTENSIONS:-}
-
 # Toggle for deploying Keystone under HTTPD + mod_wsgi
 KEYSTONE_USE_MOD_WSGI=${KEYSTONE_USE_MOD_WSGI:-${ENABLE_HTTPD_MOD_WSGI_SERVICES}}
 
@@ -192,8 +188,6 @@
         inidelete $KEYSTONE_PASTE_INI composite:admin \\/v2.0
     fi
 
-    configure_keystone_extensions
-
     # Rewrite stock ``keystone.conf``
 
     if is_service_enabled ldap; then
@@ -218,8 +212,6 @@
 
     iniset_rpc_backend keystone $KEYSTONE_CONF
 
-    iniset $KEYSTONE_CONF eventlet_server admin_bind_host "$KEYSTONE_ADMIN_BIND_HOST"
-
     # Register SSL certificates if provided
     if is_ssl_enabled_service key; then
         ensure_certificates KEYSTONE
@@ -299,35 +291,17 @@
         iniset $KEYSTONE_CONF DEFAULT logging_debug_format_suffix "%(funcName)s %(pathname)s:%(lineno)d"
         iniset $KEYSTONE_CONF DEFAULT logging_exception_prefix "%(process)d TRACE %(name)s %(instance)s"
         _config_keystone_apache_wsgi
+    else
+        iniset $KEYSTONE_CONF eventlet_server admin_bind_host "$KEYSTONE_ADMIN_BIND_HOST"
+        iniset $KEYSTONE_CONF eventlet_server admin_workers "$API_WORKERS"
+        # Public workers will use the server default, typically number of CPU.
     fi
 
     iniset $KEYSTONE_CONF DEFAULT max_token_size 16384
 
-    iniset $KEYSTONE_CONF eventlet_server admin_workers "$API_WORKERS"
-    # Public workers will use the server default, typically number of CPU.
-
     iniset $KEYSTONE_CONF fernet_tokens key_repository "$KEYSTONE_CONF_DIR/fernet-keys/"
 }
 
-function configure_keystone_extensions {
-    # Add keystone extension into keystone v3 application pipeline
-    local extension_value
-    local api_v3
-    local extension
-    local api_v3_extension
-    for extension_value in ${KEYSTONE_EXTENSIONS//,/ }; do
-        if [[ -z "${extension_value}" ]]; then
-            continue
-        fi
-        api_v3=$(iniget $KEYSTONE_PASTE_INI pipeline:api_v3 pipeline)
-        extension=$(echo $api_v3 | sed -ne "/${extension_value}/ p;" )
-        if [[ -z $extension ]]; then
-            api_v3_extension=$(echo $api_v3 | sed -ne "s/service_v3/${extension_value}_extension service_v3/p;" )
-            iniset $KEYSTONE_PASTE_INI pipeline:api_v3 pipeline "$api_v3_extension"
-        fi
-    done
-}
-
 # create_keystone_accounts() - Sets up common required keystone accounts
 
 # Tenant               User       Roles
@@ -472,14 +446,6 @@
     # Initialize keystone database
     $KEYSTONE_BIN_DIR/keystone-manage db_sync
 
-    local extension_value
-    for extension_value in ${KEYSTONE_EXTENSIONS//,/ }; do
-        if [[ -z "${extension_value}" ]]; then
-            continue
-        fi
-        $KEYSTONE_BIN_DIR/keystone-manage db_sync --extension "${extension_value}"
-    done
-
     if [[ "$KEYSTONE_TOKEN_FORMAT" == "pki" || "$KEYSTONE_TOKEN_FORMAT" == "pkiz" ]]; then
         # Set up certificates
         rm -rf $KEYSTONE_CONF_DIR/ssl
@@ -595,7 +561,7 @@
 
 
 # Restore xtrace
-$XTRACE
+$_XTRACE_KEYSTONE
 
 # Tell emacs to use shell-script-mode
 ## Local variables:
diff --git a/lib/ldap b/lib/ldap
index 0414fea..65056ae 100644
--- a/lib/ldap
+++ b/lib/ldap
@@ -8,7 +8,7 @@
 # - install_ldap()
 
 # Save trace setting
-XTRACE=$(set +o | grep xtrace)
+_XTRACE_LDAP=$(set +o | grep xtrace)
 set +o xtrace
 
 
@@ -166,7 +166,7 @@
 }
 
 # Restore xtrace
-$XTRACE
+$_XTRACE_LDAP
 
 # Tell emacs to use shell-script-mode
 ## Local variables:
diff --git a/lib/lvm b/lib/lvm
index 468a99a..ae6023a 100644
--- a/lib/lvm
+++ b/lib/lvm
@@ -16,7 +16,7 @@
 
 
 # Save trace setting
-MY_XTRACE=$(set +o | grep xtrace)
+_XTRACE_LVM=$(set +o | grep xtrace)
 set +o xtrace
 
 
@@ -182,7 +182,7 @@
 }
 
 # Restore xtrace
-$MY_XTRACE
+$_XTRACE_LVM
 
 # mode: shell-script
 # End:
diff --git a/lib/neutron-legacy b/lib/neutron-legacy
index 7b6f1c6..ad10dbe 100644
--- a/lib/neutron-legacy
+++ b/lib/neutron-legacy
@@ -112,6 +112,12 @@
 NEUTRON_CONF=$NEUTRON_CONF_DIR/neutron.conf
 export NEUTRON_TEST_CONFIG_FILE=${NEUTRON_TEST_CONFIG_FILE:-"$NEUTRON_CONF_DIR/debug.ini"}
 
+# Default provider for load balancer service
+DEFAULT_LB_PROVIDER=LOADBALANCER:Haproxy:neutron_lbaas.services.loadbalancer.drivers.haproxy.plugin_driver.HaproxyOnHostPluginDriver:default
+
+# Default provider for VPN service
+DEFAULT_VPN_PROVIDER=VPN:openswan:neutron_vpnaas.services.vpn.service_drivers.ipsec.IPsecVPNDriver:default
+
 # Agent binaries.  Note, binary paths for other agents are set in per-service
 # scripts in lib/neutron_plugins/services/
 AGENT_DHCP_BINARY="$NEUTRON_BIN_DIR/neutron-dhcp-agent"
@@ -165,6 +171,9 @@
 
 ## Provider Network Information
 PROVIDER_SUBNET_NAME=${PROVIDER_SUBNET_NAME:-"provider_net"}
+IPV6_PROVIDER_SUBNET_NAME=${IPV6_PROVIDER_SUBNET_NAME:-"provider_net_v6"}
+IPV6_PROVIDER_FIXED_RANGE=${IPV6_PROVIDER_FIXED_RANGE:-}
+IPV6_PROVIDER_NETWORK_GATEWAY=${IPV6_PROVIDER_NETWORK_GATEWAY:-}
 
 # Define the public bridge that will transmit traffic from VMs to the
 # physical network - used by both the OVS and Linux Bridge drivers.
@@ -256,7 +265,7 @@
 
 # If using GRE tunnels for tenant networks, specify the range of
 # tunnel IDs from which tenant networks are allocated. Can be
-# overriden in ``localrc`` in necesssary.
+# overridden in ``localrc`` in necessary.
 TENANT_TUNNEL_RANGES=${TENANT_TUNNEL_RANGES:-1:1000}
 
 # To use VLANs for tenant networks, set to True in localrc. VLANs
@@ -357,7 +366,7 @@
 fi
 
 # Save trace setting
-XTRACE=$(set +o | grep xtrace)
+_XTRACE_NEUTRON=$(set +o | grep xtrace)
 set +o xtrace
 
 
@@ -536,7 +545,7 @@
 
     if is_provider_network; then
         die_if_not_set $LINENO PHYSICAL_NETWORK "You must specify the PHYSICAL_NETWORK"
-        die_if_not_set $LINENO PROVIDER_NETWORK_TYPE "You must specifiy the PROVIDER_NETWORK_TYPE"
+        die_if_not_set $LINENO PROVIDER_NETWORK_TYPE "You must specify the PROVIDER_NETWORK_TYPE"
         NET_ID=$(neutron net-create $PHYSICAL_NETWORK --tenant_id $TENANT_ID --provider:network_type $PROVIDER_NETWORK_TYPE --provider:physical_network "$PHYSICAL_NETWORK" ${SEGMENTATION_ID:+--provider:segmentation_id $SEGMENTATION_ID} --shared | grep ' id ' | get_field 2)
         die_if_not_set $LINENO NET_ID "Failure creating NET_ID for $PHYSICAL_NETWORK $TENANT_ID"
 
@@ -545,14 +554,16 @@
             die_if_not_set $LINENO SUBNET_ID "Failure creating SUBNET_ID for $PROVIDER_SUBNET_NAME $TENANT_ID"
         fi
 
-        if [[ "$IP_VERSION" =~ .*6 ]]; then
-            SUBNET_V6_ID=$(neutron subnet-create --tenant_id $TENANT_ID --ip_version 6 --ipv6-address-mode $IPV6_ADDRESS_MODE --gateway $V6_NETWORK_GATEWAY --name $PROVIDER_SUBNET_NAME_V6 --subnetpool_id None $NET_ID $FIXED_RANGE_V6 | grep 'id' | get_field 2)
-            die_if_not_set $LINENO SUBNET_V6_ID "Failure creating SUBNET_V6_ID for $PROVIDER_SUBNET_NAME_V6 $TENANT_ID"
+        if [[ "$IP_VERSION" =~ .*6 ]] && [[ -n "$IPV6_PROVIDER_FIXED_RANGE" ]] && [[ -n "$IPV6_PROVIDER_NETWORK_GATEWAY" ]]; then
+            SUBNET_V6_ID=$(neutron subnet-create --tenant_id $TENANT_ID --ip_version 6 --ipv6-address-mode $IPV6_ADDRESS_MODE --gateway $IPV6_PROVIDER_NETWORK_GATEWAY --name $IPV6_PROVIDER_SUBNET_NAME --subnetpool None $NET_ID $IPV6_PROVIDER_FIXED_RANGE | grep 'id' | get_field 2)
+            die_if_not_set $LINENO SUBNET_V6_ID "Failure creating SUBNET_V6_ID for $IPV6_PROVIDER_SUBNET_NAME $TENANT_ID"
         fi
 
-        sudo ip link set $OVS_PHYSICAL_BRIDGE up
-        sudo ip link set br-int up
-        sudo ip link set $PUBLIC_INTERFACE up
+        if [[ $Q_AGENT == "openvswitch" ]]; then
+            sudo ip link set $OVS_PHYSICAL_BRIDGE up
+            sudo ip link set br-int up
+            sudo ip link set $PUBLIC_INTERFACE up
+        fi
     else
         NET_ID=$(neutron net-create --tenant-id $TENANT_ID "$PRIVATE_NETWORK_NAME" | grep ' id ' | get_field 2)
         die_if_not_set $LINENO NET_ID "Failure creating NET_ID for $PRIVATE_NETWORK_NAME $TENANT_ID"
@@ -698,7 +709,7 @@
 function start_neutron_l2_agent {
     run_process q-agt "$AGENT_BINARY --config-file $NEUTRON_CONF --config-file /$Q_PLUGIN_CONF_FILE"
 
-    if is_provider_network; then
+    if is_provider_network && [[ $Q_AGENT == "openvswitch" ]]; then
         sudo ovs-vsctl --no-wait -- --may-exist add-port $OVS_PHYSICAL_BRIDGE $PUBLIC_INTERFACE
         sudo ip link set $OVS_PHYSICAL_BRIDGE up
         sudo ip link set br-int up
@@ -797,17 +808,12 @@
 
         local IP_ADD=""
         local IP_DEL=""
+        local IP_UP=""
         local DEFAULT_ROUTE_GW
         DEFAULT_ROUTE_GW=$(ip -f $af r | awk "/default.+$from_intf/ { print \$3; exit }")
         local ADD_OVS_PORT=""
 
-        if [[ $af == "inet" ]]; then
-            IP_BRD=$(ip -f $af a s dev $from_intf | grep inet | awk '{ print $2, $3, $4; exit }')
-        fi
-
-        if [[ $af == "inet6" ]]; then
-            IP_BRD=$(ip -f $af a s dev $from_intf | grep 'scope global' | sed '/temporary/d' | awk '{ print $2, $3, $4; exit }')
-        fi
+        IP_BRD=$(ip -f $af a s dev $from_intf scope global primary | grep inet | awk '{ print $2, $3, $4; exit }')
 
         if [ "$DEFAULT_ROUTE_GW" != "" ]; then
             ADD_DEFAULT_ROUTE="sudo ip -f $af r replace default via $DEFAULT_ROUTE_GW dev $to_intf"
@@ -820,9 +826,10 @@
         if [[ "$IP_BRD" != "" ]]; then
             IP_DEL="sudo ip addr del $IP_BRD dev $from_intf"
             IP_ADD="sudo ip addr add $IP_BRD dev $to_intf"
+            IP_UP="sudo ip link set $to_intf up"
         fi
 
-        $IP_DEL; $IP_ADD; $ADD_OVS_PORT; $ADD_DEFAULT_ROUTE
+        $IP_DEL; $IP_ADD; $IP_UP; $ADD_OVS_PORT; $ADD_DEFAULT_ROUTE
     fi
 }
 
@@ -834,6 +841,10 @@
         _move_neutron_addresses_route "$OVS_PHYSICAL_BRIDGE" "$PUBLIC_INTERFACE" False "inet"
 
         if [[ $(ip -f inet6 a s dev "$OVS_PHYSICAL_BRIDGE" | grep -c 'global') != 0 ]]; then
+            # ip(8) wants the prefix length when deleting
+            local v6_gateway
+            v6_gateway=$(ip -6 a s dev $OVS_PHYSICAL_BRIDGE | grep $IPV6_PUBLIC_NETWORK_GATEWAY | awk '{ print $2 }')
+            sudo ip -6 addr del $v6_gateway dev $OVS_PHYSICAL_BRIDGE
             _move_neutron_addresses_route "$OVS_PHYSICAL_BRIDGE" "$PUBLIC_INTERFACE" False "inet6"
         fi
 
@@ -872,7 +883,10 @@
 function _configure_neutron_common {
     _create_neutron_conf_dir
 
-    cp $NEUTRON_DIR/etc/neutron.conf $NEUTRON_CONF
+    # Uses oslo config generator to generate core sample configuration files
+    (cd $NEUTRON_DIR && exec ./tools/generate_config_file_samples.sh)
+
+    cp $NEUTRON_DIR/etc/neutron.conf.sample $NEUTRON_CONF
 
     Q_POLICY_FILE=$NEUTRON_CONF_DIR/policy.json
     cp $NEUTRON_DIR/etc/policy.json $Q_POLICY_FILE
@@ -897,7 +911,9 @@
     Q_PLUGIN_CONF_FILE=$Q_PLUGIN_CONF_PATH/$Q_PLUGIN_CONF_FILENAME
     # NOTE(hichihara): Some neutron vendor plugins were already decomposed and
     # there is no config file in Neutron tree. They should prepare the file in each plugin.
-    if [ -f $NEUTRON_DIR/$Q_PLUGIN_CONF_FILE ]; then
+    if [ -f "$NEUTRON_DIR/$Q_PLUGIN_CONF_FILE.sample" ]; then
+        cp "$NEUTRON_DIR/$Q_PLUGIN_CONF_FILE.sample" /$Q_PLUGIN_CONF_FILE
+    elif [ -f $NEUTRON_DIR/$Q_PLUGIN_CONF_FILE ]; then
         cp $NEUTRON_DIR/$Q_PLUGIN_CONF_FILE /$Q_PLUGIN_CONF_FILE
     fi
 
@@ -905,6 +921,8 @@
     iniset $NEUTRON_CONF DEFAULT state_path $DATA_DIR/neutron
     iniset $NEUTRON_CONF DEFAULT use_syslog $SYSLOG
     iniset $NEUTRON_CONF DEFAULT bind_host $Q_LISTEN_ADDRESS
+    iniset $NEUTRON_CONF oslo_concurrency lock_path $DATA_DIR/neutron/lock
+
     # If addition config files are set, make sure their path name is set as well
     if [[ ${#Q_PLUGIN_EXTRA_CONF_FILES[@]} > 0 && $Q_PLUGIN_EXTRA_CONF_PATH == '' ]]; then
         die $LINENO "Neutron additional plugin config not set.. exiting"
@@ -961,7 +979,7 @@
         return
     fi
 
-    cp $NEUTRON_DIR/etc/l3_agent.ini $NEUTRON_TEST_CONFIG_FILE
+    cp $NEUTRON_DIR/etc/l3_agent.ini.sample $NEUTRON_TEST_CONFIG_FILE
 
     iniset $NEUTRON_TEST_CONFIG_FILE DEFAULT verbose False
     iniset $NEUTRON_TEST_CONFIG_FILE DEFAULT debug False
@@ -977,7 +995,7 @@
 
 function _configure_neutron_dhcp_agent {
 
-    cp $NEUTRON_DIR/etc/dhcp_agent.ini $Q_DHCP_CONF_FILE
+    cp $NEUTRON_DIR/etc/dhcp_agent.ini.sample $Q_DHCP_CONF_FILE
 
     iniset $Q_DHCP_CONF_FILE DEFAULT verbose True
     iniset $Q_DHCP_CONF_FILE DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL
@@ -1009,7 +1027,7 @@
         neutron_vpn_configure_agent
     fi
 
-    cp $NEUTRON_DIR/etc/l3_agent.ini $Q_L3_CONF_FILE
+    cp $NEUTRON_DIR/etc/l3_agent.ini.sample $Q_L3_CONF_FILE
 
     iniset $Q_L3_CONF_FILE DEFAULT verbose True
     iniset $Q_L3_CONF_FILE DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL
@@ -1030,7 +1048,7 @@
 }
 
 function _configure_neutron_metadata_agent {
-    cp $NEUTRON_DIR/etc/metadata_agent.ini $Q_META_CONF_FILE
+    cp $NEUTRON_DIR/etc/metadata_agent.ini.sample $Q_META_CONF_FILE
 
     iniset $Q_META_CONF_FILE DEFAULT verbose True
     iniset $Q_META_CONF_FILE DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL
@@ -1051,8 +1069,12 @@
 }
 
 function _configure_neutron_lbaas {
-    if [ -f $NEUTRON_LBAAS_DIR/etc/neutron_lbaas.conf ]; then
-        cp $NEUTRON_LBAAS_DIR/etc/neutron_lbaas.conf $NEUTRON_CONF_DIR
+    # Uses oslo config generator to generate LBaaS sample configuration files
+    (cd $NEUTRON_LBAAS_DIR && exec ./tools/generate_config_file_samples.sh)
+
+    if [ -f $NEUTRON_LBAAS_DIR/etc/neutron_lbaas.conf.sample ]; then
+        cp $NEUTRON_LBAAS_DIR/etc/neutron_lbaas.conf.sample $NEUTRON_CONF_DIR/neutron_lbaas.conf
+        iniset $NEUTRON_CONF_DIR/neutron_lbaas.conf service_providers service_provider $DEFAULT_LB_PROVIDER
     fi
     neutron_agent_lbaas_configure_common
     neutron_agent_lbaas_configure_agent
@@ -1072,8 +1094,11 @@
 }
 
 function _configure_neutron_vpn {
-    if [ -f $NEUTRON_VPNAAS_DIR/etc/neutron_vpnaas.conf ]; then
-        cp $NEUTRON_VPNAAS_DIR/etc/neutron_vpnaas.conf $NEUTRON_CONF_DIR
+    # Uses oslo config generator to generate VPNaaS sample configuration files
+    (cd $NEUTRON_VPNAAS_DIR && exec ./tools/generate_config_file_samples.sh)
+    if [ -f $NEUTRON_VPNAAS_DIR/etc/neutron_vpnaas.conf.sample ]; then
+        cp $NEUTRON_VPNAAS_DIR/etc/neutron_vpnaas.conf.sample $NEUTRON_CONF_DIR/neutron_vpnaas.conf
+        iniset $NEUTRON_CONF_DIR/neutron_vpnaas.conf service_providers service_provider $DEFAULT_VPN_PROVIDER
     fi
     neutron_vpn_install_agent_packages
     neutron_vpn_configure_common
@@ -1122,7 +1147,7 @@
     iniset $NEUTRON_CONF DEFAULT auth_strategy $Q_AUTH_STRATEGY
     _neutron_setup_keystone $NEUTRON_CONF keystone_authtoken
 
-    # Configuration for neutron notifations to nova.
+    # Configuration for neutron notifications to nova.
     iniset $NEUTRON_CONF DEFAULT notify_nova_on_port_status_changes $Q_NOTIFY_NOVA_PORT_STATUS_CHANGES
     iniset $NEUTRON_CONF DEFAULT notify_nova_on_port_data_changes $Q_NOTIFY_NOVA_PORT_DATA_CHANGES
 
@@ -1321,7 +1346,7 @@
                 sudo ip addr add $ext_gw_ip/$cidr_len dev $ext_gw_interface
                 sudo ip link set $ext_gw_interface up
             fi
-            ROUTER_GW_IP=`neutron port-list -c fixed_ips -c device_owner | grep router_gateway | awk -F '"' -v subnet_id=$PUB_SUBNET_ID '$4 == subnet_id { print $8; }'`
+            ROUTER_GW_IP=`neutron port-list -c fixed_ips -c device_owner | grep router_gateway | awk -F'ip_address'  '{ print $2 }' | cut -f3 -d\" | tr '\n' ' '`
             die_if_not_set $LINENO ROUTER_GW_IP "Failure retrieving ROUTER_GW_IP"
             sudo ip route replace  $FIXED_RANGE via $ROUTER_GW_IP
         fi
@@ -1352,7 +1377,7 @@
         sudo sysctl -w net.ipv6.conf.all.forwarding=1
         # Configure and enable public bridge
         # Override global IPV6_ROUTER_GW_IP with the true value from neutron
-        IPV6_ROUTER_GW_IP=`neutron port-list -c fixed_ips | grep $ipv6_pub_subnet_id | awk -F '"' -v subnet_id=$ipv6_pub_subnet_id '$4 == subnet_id { print $8; }'`
+        IPV6_ROUTER_GW_IP=`neutron port-list -c fixed_ips | grep $ipv6_pub_subnet_id | awk -F'ip_address' '{ print $2 }' | cut -f3 -d\" | tr '\n' ' '`
         die_if_not_set $LINENO IPV6_ROUTER_GW_IP "Failure retrieving IPV6_ROUTER_GW_IP"
 
         if is_neutron_ovs_base_plugin; then
@@ -1494,7 +1519,7 @@
 
 
 # Restore xtrace
-$XTRACE
+$_XTRACE_NEUTRON
 
 # Tell emacs to use shell-script-mode
 ## Local variables:
diff --git a/lib/neutron_plugins/bigswitch_floodlight b/lib/neutron_plugins/bigswitch_floodlight
index f52105e..586ded7 100644
--- a/lib/neutron_plugins/bigswitch_floodlight
+++ b/lib/neutron_plugins/bigswitch_floodlight
@@ -4,7 +4,7 @@
 # ------------------------------------
 
 # Save trace setting
-BS_XTRACE=$(set +o | grep xtrace)
+_XTRACE_NEUTRON_BIGSWITCH=$(set +o | grep xtrace)
 set +o xtrace
 
 source $TOP_DIR/lib/neutron_plugins/ovs_base
@@ -75,4 +75,4 @@
 }
 
 # Restore xtrace
-$BS_XTRACE
+$_XTRACE_NEUTRON_BIGSWITCH
diff --git a/lib/neutron_plugins/brocade b/lib/neutron_plugins/brocade
index 953360e..6ba0a66 100644
--- a/lib/neutron_plugins/brocade
+++ b/lib/neutron_plugins/brocade
@@ -4,7 +4,7 @@
 # ----------------------
 
 # Save trace setting
-BRCD_XTRACE=$(set +o | grep xtrace)
+_XTRACE_NEUTRON_BROCADE=$(set +o | grep xtrace)
 set +o xtrace
 
 function is_neutron_ovs_base_plugin {
@@ -81,4 +81,4 @@
 }
 
 # Restore xtrace
-$BRCD_XTRACE
+$_XTRACE_NEUTRON_BROCADE
diff --git a/lib/neutron_plugins/cisco b/lib/neutron_plugins/cisco
index 7d0cf1a..fc2cb8a 100644
--- a/lib/neutron_plugins/cisco
+++ b/lib/neutron_plugins/cisco
@@ -4,7 +4,7 @@
 # ---------------------------
 
 # Save trace setting
-CISCO_XTRACE=$(set +o | grep xtrace)
+_XTRACE_NEUTRON_CISCO=$(set +o | grep xtrace)
 set +o xtrace
 
 # Scecify the VSM parameters
@@ -154,4 +154,4 @@
 }
 
 # Restore xtrace
-$CISCO_XTRACE
+$_XTRACE_NEUTRON_CISCO
diff --git a/lib/neutron_plugins/embrane b/lib/neutron_plugins/embrane
index 2028496..385dab8 100644
--- a/lib/neutron_plugins/embrane
+++ b/lib/neutron_plugins/embrane
@@ -4,7 +4,7 @@
 # ---------------------------
 
 # Save trace setting
-EMBR_XTRACE=$(set +o | grep xtrace)
+_XTRACE_NEUTRON_EMBR=$(set +o | grep xtrace)
 set +o xtrace
 
 source $TOP_DIR/lib/neutron_plugins/openvswitch
@@ -39,4 +39,5 @@
 }
 
 # Restore xtrace
-$EMBR_XTRACE
+$_XTRACE_NEUTRON_EMBR
+
diff --git a/lib/neutron_plugins/linuxbridge_agent b/lib/neutron_plugins/linuxbridge_agent
index f28bcfe..096722b 100644
--- a/lib/neutron_plugins/linuxbridge_agent
+++ b/lib/neutron_plugins/linuxbridge_agent
@@ -4,7 +4,7 @@
 # -----------------------------
 
 # Save trace setting
-PLUGIN_XTRACE=$(set +o | grep xtrace)
+_XTRACE_NEUTRON_LB=$(set +o | grep xtrace)
 set +o xtrace
 
 function neutron_lb_cleanup {
@@ -93,4 +93,4 @@
 }
 
 # Restore xtrace
-$PLUGIN_XTRACE
+$_XTRACE_NEUTRON_LB
diff --git a/lib/neutron_plugins/ml2 b/lib/neutron_plugins/ml2
index ace5335..30e1b03 100644
--- a/lib/neutron_plugins/ml2
+++ b/lib/neutron_plugins/ml2
@@ -4,7 +4,7 @@
 # ------------------------------
 
 # Save trace setting
-ML2_XTRACE=$(set +o | grep xtrace)
+_XTRACE_NEUTRON_ML2=$(set +o | grep xtrace)
 set +o xtrace
 
 # Enable this to simply and quickly enable tunneling with ML2.
@@ -137,4 +137,4 @@
 }
 
 # Restore xtrace
-$ML2_XTRACE
+$_XTRACE_NEUTRON_ML2
diff --git a/lib/neutron_plugins/nuage b/lib/neutron_plugins/nuage
index 9e5307b..61e634e 100644
--- a/lib/neutron_plugins/nuage
+++ b/lib/neutron_plugins/nuage
@@ -4,7 +4,7 @@
 # ----------------------
 
 # Save trace setting
-NU_XTRACE=$(set +o | grep xtrace)
+_XTRACE_NEUTRON_NU=$(set +o | grep xtrace)
 set +o xtrace
 
 function neutron_plugin_create_nova_conf {
@@ -66,4 +66,4 @@
 }
 
 # Restore xtrace
-$NU_XTRACE
+$_XTRACE_NEUTRON_NU
diff --git a/lib/neutron_plugins/openvswitch b/lib/neutron_plugins/openvswitch
index 891ab49..130eaac 100644
--- a/lib/neutron_plugins/openvswitch
+++ b/lib/neutron_plugins/openvswitch
@@ -7,7 +7,7 @@
 # which has been removed in Juno.
 
 # Save trace setting
-OVS_XTRACE=$(set +o | grep xtrace)
+_XTRACE_NEUTRON_OVS=$(set +o | grep xtrace)
 set +o xtrace
 
 source $TOP_DIR/lib/neutron_plugins/openvswitch_agent
@@ -56,4 +56,5 @@
 }
 
 # Restore xtrace
-$OVS_XTRACE
+$_XTRACE_NEUTRON_OVS
+
diff --git a/lib/neutron_plugins/openvswitch_agent b/lib/neutron_plugins/openvswitch_agent
index 6a33393..b1acacd 100644
--- a/lib/neutron_plugins/openvswitch_agent
+++ b/lib/neutron_plugins/openvswitch_agent
@@ -4,7 +4,7 @@
 # -----------------------------
 
 # Save trace setting
-OVSA_XTRACE=$(set +o | grep xtrace)
+_XTRACE_NEUTRON_OVSL2=$(set +o | grep xtrace)
 set +o xtrace
 
 source $TOP_DIR/lib/neutron_plugins/ovs_base
@@ -132,4 +132,4 @@
 }
 
 # Restore xtrace
-$OVSA_XTRACE
+$_XTRACE_NEUTRON_OVSL2
diff --git a/lib/neutron_plugins/ovs_base b/lib/neutron_plugins/ovs_base
index d3fd198..91aff33 100644
--- a/lib/neutron_plugins/ovs_base
+++ b/lib/neutron_plugins/ovs_base
@@ -4,7 +4,7 @@
 # -------------------------------------
 
 # Save trace setting
-OVSB_XTRACE=$(set +o | grep xtrace)
+_XTRACE_NEUTRON_OVS_BASE=$(set +o | grep xtrace)
 set +o xtrace
 
 OVS_BRIDGE=${OVS_BRIDGE:-br-int}
@@ -114,4 +114,4 @@
 }
 
 # Restore xtrace
-$OVSB_XTRACE
+$_XTRACE_NEUTRON_OVS_BASE
diff --git a/lib/neutron_plugins/services/firewall b/lib/neutron_plugins/services/firewall
index 3496da8..2b7f32d 100644
--- a/lib/neutron_plugins/services/firewall
+++ b/lib/neutron_plugins/services/firewall
@@ -4,7 +4,7 @@
 # ---------------------------
 
 # Save trace setting
-FW_XTRACE=$(set +o | grep xtrace)
+_XTRACE_NEUTRON_FIREWALL=$(set +o | grep xtrace)
 set +o xtrace
 
 FWAAS_PLUGIN=neutron_fwaas.services.firewall.fwaas_plugin.FirewallPlugin
@@ -14,8 +14,11 @@
 }
 
 function neutron_fwaas_configure_driver {
+    # Uses oslo config generator to generate FWaaS sample configuration files
+    (cd $NEUTRON_FWAAS_DIR && exec ./tools/generate_config_file_samples.sh)
+
     FWAAS_DRIVER_CONF_FILENAME=/etc/neutron/fwaas_driver.ini
-    cp $NEUTRON_FWAAS_DIR/etc/fwaas_driver.ini $FWAAS_DRIVER_CONF_FILENAME
+    cp $NEUTRON_FWAAS_DIR/etc/fwaas_driver.ini.sample $FWAAS_DRIVER_CONF_FILENAME
 
     iniset_multiline $FWAAS_DRIVER_CONF_FILENAME fwaas enabled True
     iniset_multiline $FWAAS_DRIVER_CONF_FILENAME fwaas driver "neutron_fwaas.services.firewall.drivers.linux.iptables_fwaas.IptablesFwaasDriver"
@@ -26,4 +29,4 @@
 }
 
 # Restore xtrace
-$FW_XTRACE
+$_XTRACE_NEUTRON_FIREWALL
diff --git a/lib/neutron_plugins/services/loadbalancer b/lib/neutron_plugins/services/loadbalancer
index 7865f6f..30e9480 100644
--- a/lib/neutron_plugins/services/loadbalancer
+++ b/lib/neutron_plugins/services/loadbalancer
@@ -4,7 +4,7 @@
 # ---------------------------
 
 # Save trace setting
-LB_XTRACE=$(set +o | grep xtrace)
+_XTRACE_NEUTRON_LB=$(set +o | grep xtrace)
 set +o xtrace
 
 
@@ -28,7 +28,7 @@
 
     LBAAS_AGENT_CONF_FILENAME="$LBAAS_AGENT_CONF_PATH/lbaas_agent.ini"
 
-    cp $NEUTRON_LBAAS_DIR/etc/lbaas_agent.ini $LBAAS_AGENT_CONF_FILENAME
+    cp $NEUTRON_LBAAS_DIR/etc/lbaas_agent.ini.sample $LBAAS_AGENT_CONF_FILENAME
 
     # ovs_use_veth needs to be set before the plugin configuration
     # occurs to allow plugins to override the setting.
@@ -48,4 +48,4 @@
 }
 
 # Restore xtrace
-$LB_XTRACE
+$_XTRACE_NEUTRON_LB
diff --git a/lib/neutron_plugins/services/metering b/lib/neutron_plugins/services/metering
index c75ab19..5b32468 100644
--- a/lib/neutron_plugins/services/metering
+++ b/lib/neutron_plugins/services/metering
@@ -4,7 +4,7 @@
 # ---------------------------
 
 # Save trace setting
-METER_XTRACE=$(set +o | grep xtrace)
+_XTRACE_NETURON_METER=$(set +o | grep xtrace)
 set +o xtrace
 
 
@@ -21,7 +21,7 @@
 
     METERING_AGENT_CONF_FILENAME="$METERING_AGENT_CONF_PATH/metering_agent.ini"
 
-    cp $NEUTRON_DIR/etc/metering_agent.ini $METERING_AGENT_CONF_FILENAME
+    cp $NEUTRON_DIR/etc/metering_agent.ini.sample $METERING_AGENT_CONF_FILENAME
 }
 
 function neutron_metering_stop {
@@ -29,4 +29,5 @@
 }
 
 # Restore xtrace
-$METER_XTRACE
+$_XTRACE_NETURON_METER
+
diff --git a/lib/neutron_plugins/services/vpn b/lib/neutron_plugins/services/vpn
index c0e7457..e790913 100644
--- a/lib/neutron_plugins/services/vpn
+++ b/lib/neutron_plugins/services/vpn
@@ -4,7 +4,7 @@
 # ---------------------------
 
 # Save trace setting
-VPN_XTRACE=$(set +o | grep xtrace)
+_XTRACE_NEUTRON_VPN=$(set +o | grep xtrace)
 set +o xtrace
 
 
@@ -29,7 +29,9 @@
 }
 
 function neutron_vpn_configure_agent {
-    cp $NEUTRON_VPNAAS_DIR/etc/vpn_agent.ini $Q_VPN_CONF_FILE
+    # Uses oslo config generator to generate LBaaS sample configuration files
+    (cd $NEUTRON_VPNAAS_DIR && exec ./tools/generate_config_file_samples.sh)
+    cp $NEUTRON_VPNAAS_DIR/etc/vpn_agent.ini.sample $Q_VPN_CONF_FILE
     if [[ "$IPSEC_PACKAGE" == "strongswan" ]]; then
         iniset_multiline $Q_VPN_CONF_FILE vpnagent vpn_device_driver neutron_vpnaas.services.vpn.device_drivers.strongswan_ipsec.StrongSwanDriver
         if is_fedora; then
@@ -53,4 +55,4 @@
 }
 
 # Restore xtrace
-$VPN_XTRACE
+$_XTRACE_NEUTRON_VPN
diff --git a/lib/neutron_thirdparty/bigswitch_floodlight b/lib/neutron_thirdparty/bigswitch_floodlight
index e3f4689..45a4f2e 100644
--- a/lib/neutron_thirdparty/bigswitch_floodlight
+++ b/lib/neutron_thirdparty/bigswitch_floodlight
@@ -4,7 +4,7 @@
 # ------------------------------------------
 
 # Save trace setting
-BS3_XTRACE=$(set +o | grep xtrace)
+_XTRACE_NEUTRON_BIGSWITCH=$(set +o | grep xtrace)
 set +o xtrace
 
 BS_FL_CONTROLLERS_PORT=${BS_FL_CONTROLLERS_PORT:-localhost:80}
@@ -51,4 +51,4 @@
 }
 
 # Restore xtrace
-$BS3_XTRACE
+$_XTRACE_NEUTRON_BIGSWITCH
diff --git a/lib/nova b/lib/nova
index ba05f53..6337f87 100644
--- a/lib/nova
+++ b/lib/nova
@@ -25,7 +25,7 @@
 # - cleanup_nova
 
 # Save trace setting
-XTRACE=$(set +o | grep xtrace)
+_XTRACE_LIB_NOVA=$(set +o | grep xtrace)
 set +o xtrace
 
 
@@ -668,6 +668,7 @@
         iniset $NOVA_CONF serial_console serialproxy_host "$NOVA_SERVICE_LISTEN_ADDRESS"
         iniset $NOVA_CONF serial_console enabled True
     fi
+    iniset $NOVA_CONF DEFAULT graceful_shutdown_timeout "$SERVICE_GRACEFUL_SHUTDOWN_TIMEOUT"
 }
 
 function init_nova_cells {
@@ -958,7 +959,7 @@
 
 
 # Restore xtrace
-$XTRACE
+$_XTRACE_LIB_NOVA
 
 # Tell emacs to use shell-script-mode
 ## Local variables:
diff --git a/lib/nova_plugins/functions-libvirt b/lib/nova_plugins/functions-libvirt
index 78c5978..dae55c6 100644
--- a/lib/nova_plugins/functions-libvirt
+++ b/lib/nova_plugins/functions-libvirt
@@ -8,7 +8,7 @@
 # ``STACK_USER`` has to be defined
 
 # Save trace setting
-LV_XTRACE=$(set +o | grep xtrace)
+_XTRACE_NOVA_FN_LIBVIRT=$(set +o | grep xtrace)
 set +o xtrace
 
 # Defaults
@@ -31,6 +31,11 @@
         fi
         install_package libvirt-bin libvirt-dev
         pip_install_gr libvirt-python
+        if [[ "$EBTABLES_RACE_FIX" == "True" ]]; then
+            # Work around for bug #1501558. We can remove this once we
+            # get to a version of Ubuntu that has new enough libvirt.
+            TOP_DIR=$TOP_DIR $TOP_DIR/tools/install_ebtables_workaround.sh
+        fi
         #pip_install_gr <there-si-no-guestfs-in-pypi>
     elif is_fedora || is_suse; then
         install_package kvm
@@ -129,7 +134,7 @@
 
 
 # Restore xtrace
-$LV_XTRACE
+$_XTRACE_NOVA_FN_LIBVIRT
 
 # Local variables:
 # mode: shell-script
diff --git a/lib/nova_plugins/hypervisor-fake b/lib/nova_plugins/hypervisor-fake
index 3180d91..2434dce 100644
--- a/lib/nova_plugins/hypervisor-fake
+++ b/lib/nova_plugins/hypervisor-fake
@@ -17,7 +17,7 @@
 # cleanup_nova_hypervisor - remove transient data and cache
 
 # Save trace setting
-MY_XTRACE=$(set +o | grep xtrace)
+_XTRACE_VIRTFAKE=$(set +o | grep xtrace)
 set +o xtrace
 
 
@@ -72,7 +72,7 @@
 
 
 # Restore xtrace
-$MY_XTRACE
+$_XTRACE_VIRTFAKE
 
 # Local variables:
 # mode: shell-script
diff --git a/lib/nova_plugins/hypervisor-ironic b/lib/nova_plugins/hypervisor-ironic
index b9e286d..c6ed85d 100644
--- a/lib/nova_plugins/hypervisor-ironic
+++ b/lib/nova_plugins/hypervisor-ironic
@@ -17,7 +17,7 @@
 # cleanup_nova_hypervisor - remove transient data and cache
 
 # Save trace setting
-MY_XTRACE=$(set +o | grep xtrace)
+_XTRACE_HYP_IRONIC=$(set +o | grep xtrace)
 set +o xtrace
 
 source $TOP_DIR/lib/nova_plugins/functions-libvirt
@@ -81,7 +81,7 @@
 
 
 # Restore xtrace
-$MY_XTRACE
+$_XTRACE_HYP_IRONIC
 
 # Local variables:
 # mode: shell-script
diff --git a/lib/nova_plugins/hypervisor-libvirt b/lib/nova_plugins/hypervisor-libvirt
index c54a716..8bbaa21 100644
--- a/lib/nova_plugins/hypervisor-libvirt
+++ b/lib/nova_plugins/hypervisor-libvirt
@@ -17,7 +17,7 @@
 # cleanup_nova_hypervisor - remove transient data and cache
 
 # Save trace setting
-MY_XTRACE=$(set +o | grep xtrace)
+_XTRACE_NOVA_LIBVIRT=$(set +o | grep xtrace)
 set +o xtrace
 
 source $TOP_DIR/lib/nova_plugins/functions-libvirt
@@ -105,7 +105,7 @@
 
 
 # Restore xtrace
-$MY_XTRACE
+$_XTRACE_NOVA_LIBVIRT
 
 # Local variables:
 # mode: shell-script
diff --git a/lib/nova_plugins/hypervisor-openvz b/lib/nova_plugins/hypervisor-openvz
index cce36b8..58ab5c1 100644
--- a/lib/nova_plugins/hypervisor-openvz
+++ b/lib/nova_plugins/hypervisor-openvz
@@ -17,7 +17,7 @@
 # cleanup_nova_hypervisor - remove transient data and cache
 
 # Save trace setting
-MY_XTRACE=$(set +o | grep xtrace)
+_XTRACE_OPENVZ=$(set +o | grep xtrace)
 set +o xtrace
 
 
@@ -62,7 +62,7 @@
 
 
 # Restore xtrace
-$MY_XTRACE
+$_XTRACE_OPENVZ
 
 # Local variables:
 # mode: shell-script
diff --git a/lib/nova_plugins/hypervisor-vsphere b/lib/nova_plugins/hypervisor-vsphere
index 698f836..7c08bc9 100644
--- a/lib/nova_plugins/hypervisor-vsphere
+++ b/lib/nova_plugins/hypervisor-vsphere
@@ -17,7 +17,7 @@
 # cleanup_nova_hypervisor - remove transient data and cache
 
 # Save trace setting
-MY_XTRACE=$(set +o | grep xtrace)
+_XTRACE_NOVA_VSPHERE=$(set +o | grep xtrace)
 set +o xtrace
 
 
@@ -64,7 +64,7 @@
 
 
 # Restore xtrace
-$MY_XTRACE
+$_XTRACE_NOVA_VSPHERE
 
 # Local variables:
 # mode: shell-script
diff --git a/lib/nova_plugins/hypervisor-xenserver b/lib/nova_plugins/hypervisor-xenserver
index e097990..3eb9149 100644
--- a/lib/nova_plugins/hypervisor-xenserver
+++ b/lib/nova_plugins/hypervisor-xenserver
@@ -17,7 +17,7 @@
 # cleanup_nova_hypervisor - remove transient data and cache
 
 # Save trace setting
-MY_XTRACE=$(set +o | grep xtrace)
+_XTRACE_XENSERVER=$(set +o | grep xtrace)
 set +o xtrace
 
 
@@ -111,7 +111,7 @@
 
 
 # Restore xtrace
-$MY_XTRACE
+$_XTRACE_XENSERVER
 
 # Local variables:
 # mode: shell-script
diff --git a/lib/oslo b/lib/oslo
index 56615fa..3d6fbb3 100644
--- a/lib/oslo
+++ b/lib/oslo
@@ -16,7 +16,7 @@
 # - install_oslo
 
 # Save trace setting
-XTRACE=$(set +o | grep xtrace)
+_XTRACE_LIB_OSLO=$(set +o | grep xtrace)
 set +o xtrace
 
 
@@ -95,7 +95,7 @@
 }
 
 # Restore xtrace
-$XTRACE
+$_XTRACE_LIB_OSLO
 
 # Tell emacs to use shell-script-mode
 ## Local variables:
diff --git a/lib/rpc_backend b/lib/rpc_backend
index 03eacd8..3864ade 100644
--- a/lib/rpc_backend
+++ b/lib/rpc_backend
@@ -21,7 +21,7 @@
 # of this file which is a standard interface.
 
 # Save trace setting
-XTRACE=$(set +o | grep xtrace)
+_XTRACE_RPC_BACKEND=$(set +o | grep xtrace)
 set +o xtrace
 
 # Functions
@@ -58,7 +58,7 @@
         # NOTE(bnemec): Retry initial rabbitmq configuration to deal with
         # the fact that sometimes it fails to start properly.
         # Reference: https://bugzilla.redhat.com/show_bug.cgi?id=1144100
-        # NOTE(tonyb): Extend the orginal retry logic to only restart rabbitmq
+        # NOTE(tonyb): Extend the original retry logic to only restart rabbitmq
         # every second time around the loop.
         # See: https://bugs.launchpad.net/devstack/+bug/1449056 for details on
         # why this is needed.  This can bee seen on vivid and Debian unstable
@@ -106,7 +106,7 @@
     fi
 }
 
-# iniset cofiguration
+# iniset configuration
 function iniset_rpc_backend {
     local package=$1
     local file=$2
@@ -141,7 +141,7 @@
 }
 
 # Restore xtrace
-$XTRACE
+$_XTRACE_RPC_BACKEND
 
 # Tell emacs to use shell-script-mode
 ## Local variables:
diff --git a/lib/stack b/lib/stack
index 47e8ce2..f09ddce 100644
--- a/lib/stack
+++ b/lib/stack
@@ -14,11 +14,12 @@
 # Functions
 # ---------
 
-# Generic service install handles venv creation if confgured for service
+# Generic service install handles venv creation if configured for service
 # stack_install_service service
 function stack_install_service {
     local service=$1
     if type install_${service} >/dev/null 2>&1; then
+        # FIXME(dhellmann): Needs to be python3-aware at some point.
         if [[ ${USE_VENV} = True && -n ${PROJECT_VENV[$service]:-} ]]; then
             rm -rf ${PROJECT_VENV[$service]}
             source $TOP_DIR/tools/build_venv.sh ${PROJECT_VENV[$service]} ${ADDITIONAL_VENV_PACKAGES//,/ }
diff --git a/lib/swift b/lib/swift
index ee0238d..b596142 100644
--- a/lib/swift
+++ b/lib/swift
@@ -24,7 +24,7 @@
 # - _cleanup_swift_apache_wsgi
 
 # Save trace setting
-XTRACE=$(set +o | grep xtrace)
+_XTRACE_LIB_SWIFT=$(set +o | grep xtrace)
 set +o xtrace
 
 
@@ -123,13 +123,13 @@
 # trace through the logs when looking for its use.
 SWIFT_LOG_TOKEN_LENGTH=${SWIFT_LOG_TOKEN_LENGTH:-12}
 
-# Set ``SWIFT_MAX_HEADER_SIZE`` to configure the maximun length of headers in
+# Set ``SWIFT_MAX_HEADER_SIZE`` to configure the maximum length of headers in
 # Swift API
 SWIFT_MAX_HEADER_SIZE=${SWIFT_MAX_HEADER_SIZE:-16384}
 
 # Set ``OBJECT_PORT_BASE``, ``CONTAINER_PORT_BASE``, ``ACCOUNT_PORT_BASE``
-# Port bases used in port number calclution for the service "nodes"
-# The specified port number will be used, the additinal ports calculated by
+# Port bases used in port number calculation for the service "nodes"
+# The specified port number will be used, the additional ports calculated by
 # base_port + node_num * 10
 OBJECT_PORT_BASE=${OBJECT_PORT_BASE:-6613}
 CONTAINER_PORT_BASE=${CONTAINER_PORT_BASE:-6611}
@@ -823,7 +823,7 @@
 }
 
 # Restore xtrace
-$XTRACE
+$_XTRACE_LIB_SWIFT
 
 # Tell emacs to use shell-script-mode
 ## Local variables:
diff --git a/lib/tempest b/lib/tempest
index 76fd6ca..ecc4865 100644
--- a/lib/tempest
+++ b/lib/tempest
@@ -38,7 +38,7 @@
 # - init_tempest
 
 # Save trace setting
-XTRACE=$(set +o | grep xtrace)
+_XTRACE_TEMPEST=$(set +o | grep xtrace)
 set +o xtrace
 
 
@@ -196,8 +196,8 @@
     if is_service_enabled nova; then
         # If ``DEFAULT_INSTANCE_TYPE`` is not declared, use the new behavior
         # Tempest creates its own instance types
+        available_flavors=$(nova flavor-list)
         if  [[ -z "$DEFAULT_INSTANCE_TYPE" ]]; then
-            available_flavors=$(nova flavor-list)
             if [[ ! ( $available_flavors =~ 'm1.nano' ) ]]; then
                 nova flavor-create m1.nano 42 64 0 1
             fi
@@ -210,15 +210,14 @@
         else
             # Check Nova for existing flavors, if ``DEFAULT_INSTANCE_TYPE`` is set use it.
             boto_instance_type=$DEFAULT_INSTANCE_TYPE
-            flavor_lines=`nova flavor-list`
             IFS=$'\r\n'
             flavors=""
-            for line in $flavor_lines; do
+            for line in $available_flavors; do
                 f=$(echo $line | awk "/ $DEFAULT_INSTANCE_TYPE / { print \$2 }")
                 flavors="$flavors $f"
             done
 
-            for line in $flavor_lines; do
+            for line in $available_flavors; do
                 flavors="$flavors `echo $line | grep -v "^\(|\s*ID\|+--\)" | cut -d' ' -f2`"
             done
 
@@ -330,6 +329,9 @@
     iniset $TEMPEST_CONFIG compute flavor_ref $flavor_ref
     iniset $TEMPEST_CONFIG compute flavor_ref_alt $flavor_ref_alt
     iniset $TEMPEST_CONFIG compute ssh_connect_method $ssh_connect_method
+    # set the equiv validation option here as well to ensure they are
+    # in sync. They shouldn't be separate options.
+    iniset $TEMPEST_CONFIG validation connect_method $ssh_connect_method
     if [[ ! $(is_service_enabled n-cell) && ! $(is_service_enabled neutron) ]]; then
         iniset $TEMPEST_CONFIG compute fixed_network_name $PRIVATE_NETWORK_NAME
     fi
@@ -358,6 +360,30 @@
         compute_api_extensions=$(remove_disabled_extensions $compute_api_extensions $DISABLE_COMPUTE_API_EXTENSIONS)
     fi
 
+    # Set the microversion range for compute tests.
+    # This is used to run the Nova microversions tests.
+    # Setting [None, latest] range of microversion which allow Tempest to run all microversions tests.
+    # NOTE- To avoid microversion tests failure on stable branch, we need to change "tempest_compute_max_microversion"
+    #       for stable branch on each release which should be changed from "latest" to max supported version of that release.
+    local tempest_compute_min_microversion=${TEMPEST_COMPUTE_MIN_MICROVERSION:-None}
+    local tempest_compute_max_microversion=${TEMPEST_COMPUTE_MAX_MICROVERSION:-"latest"}
+    # Reset microversions to None where v2.0 is running which does not support microversion.
+    # Both "None" means no microversion testing.
+    if [[ "$TEMPEST_COMPUTE_TYPE" == "compute_legacy" ]]; then
+        tempest_compute_min_microversion=None
+        tempest_compute_max_microversion=None
+    fi
+    if [ "$tempest_compute_min_microversion" == "None" ]; then
+        inicomment $TEMPEST_CONFIG compute-feature-enabled min_microversion
+    else
+        iniset $TEMPEST_CONFIG compute-feature-enabled min_microversion $tempest_compute_min_microversion
+    fi
+    if [ "$tempest_compute_max_microversion" == "None" ]; then
+        inicomment $TEMPEST_CONFIG compute-feature-enabled max_microversion
+    else
+        iniset $TEMPEST_CONFIG compute-feature-enabled max_microversion $tempest_compute_max_microversion
+    fi
+
     iniset $TEMPEST_CONFIG compute-feature-enabled resize True
     iniset $TEMPEST_CONFIG compute-feature-enabled live_migration ${LIVE_MIGRATION_AVAILABLE:-False}
     iniset $TEMPEST_CONFIG compute-feature-enabled change_password False
@@ -463,6 +489,8 @@
     iniset $TEMPEST_CONFIG volume-feature-enabled bootable True
     # TODO(jordanP): Remove the extend_with_snapshot flag when Juno is end of life.
     iniset $TEMPEST_CONFIG volume-feature-enabled extend_with_snapshot True
+    # TODO(obutenko): Remove the incremental_backup_force flag when Kilo and Juno is end of life.
+    iniset $TEMPEST_CONFIG volume-feature-enabled incremental_backup_force True
 
     local volume_api_extensions=${VOLUME_API_EXTENSIONS:-"all"}
     if [[ ! -z "$DISABLE_VOLUME_API_EXTENSIONS" ]]; then
@@ -508,7 +536,8 @@
     # Baremetal
     if [ "$VIRT_DRIVER" = "ironic" ] ; then
         iniset $TEMPEST_CONFIG baremetal driver_enabled True
-        iniset $TEMPEST_CONFIG baremetal unprovision_timeout 300
+        iniset $TEMPEST_CONFIG baremetal unprovision_timeout $BUILD_TIMEOUT
+        iniset $TEMPEST_CONFIG baremetal active_timeout $BUILD_TIMEOUT
         iniset $TEMPEST_CONFIG baremetal deploy_img_dir $FILES
         iniset $TEMPEST_CONFIG baremetal node_uuid $IRONIC_NODE_UUID
         iniset $TEMPEST_CONFIG compute-feature-enabled change_password False
@@ -649,7 +678,7 @@
 }
 
 # Restore xtrace
-$XTRACE
+$_XTRACE_TEMPEST
 
 # Tell emacs to use shell-script-mode
 ## Local variables:
diff --git a/lib/template b/lib/template
index 2703788..08d10bb 100644
--- a/lib/template
+++ b/lib/template
@@ -21,7 +21,7 @@
 # - cleanup_XXXX
 
 # Save trace setting
-XTRACE=$(set +o | grep xtrace)
+_XTRACE_TEMPLATE=$(set +o | grep xtrace)
 set +o xtrace
 
 
@@ -92,7 +92,7 @@
 }
 
 # Restore xtrace
-$XTRACE
+$_XTRACE_TEMPLATE
 
 # Tell emacs to use shell-script-mode
 ## Local variables:
diff --git a/samples/local.conf b/samples/local.conf
index b92097d..34c9e8b 100644
--- a/samples/local.conf
+++ b/samples/local.conf
@@ -63,7 +63,8 @@
 # Using milestone-proposed branches
 # ---------------------------------
 
-# Uncomment these to grab the milestone-proposed branches from the repos:
+# Uncomment these to grab the milestone-proposed branches from the
+# repos:
 #CINDER_BRANCH=milestone-proposed
 #GLANCE_BRANCH=milestone-proposed
 #HORIZON_BRANCH=milestone-proposed
@@ -74,6 +75,13 @@
 #NEUTRON_BRANCH=milestone-proposed
 #SWIFT_BRANCH=milestone-proposed
 
+# Using git versions of clients
+# -----------------------------
+# By default clients are installed from pip.  See LIBS_FROM_GIT in
+# stackrc for details on getting clients from specific branches or
+# revisions.  e.g.
+# LIBS_FROM_GIT="python-ironicclient"
+# IRONICCLIENT_BRANCH=refs/changes/44/2.../1
 
 # Swift
 # -----
diff --git a/stack.sh b/stack.sh
index 9b811b7..09ab474 100755
--- a/stack.sh
+++ b/stack.sh
@@ -75,6 +75,7 @@
 
 # Check if run in POSIX shell
 if [[ "${POSIXLY_CORRECT}" == "y" ]]; then
+    set +o xtrace
     echo "You are running POSIX compatibility mode, DevStack requires bash 4.2 or newer."
     exit 1
 fi
@@ -85,11 +86,11 @@
 # action to create a suitable user account.
 
 if [[ $EUID -eq 0 ]]; then
-    echo "You are running this script as root."
-    echo "Cut it out."
-    echo "Really."
-    echo "If you need an account to run DevStack, do this (as root, heh) to create a non-root account:"
-    echo "$TOP_DIR/tools/create-stack-user.sh"
+    set +o xtrace
+    echo "DevStack should be run as a user with sudo permissions, "
+    echo "not root."
+    echo "A \"stack\" user configured correctly can be created with:"
+    echo " $TOP_DIR/tools/create-stack-user.sh"
     exit 1
 fi
 
@@ -98,6 +99,7 @@
 # virtual env, and will fail in really odd ways if you do this. Make
 # this explicit as it has come up on the mailing list.
 if [[ -n "$VIRTUAL_ENV" ]]; then
+    set +o xtrace
     echo "You appear to be running under a python virtualenv."
     echo "DevStack does not support this, as we may break the"
     echo "virtualenv you are currently in by modifying "
@@ -111,6 +113,7 @@
 # on a lot of different environments, you sometimes run it on the
 # wrong box. This makes there be a way to prevent that.
 if [[ -e $HOME/.no-devstack ]]; then
+    set +o xtrace
     echo "You've marked this host as a no-devstack host, to save yourself from"
     echo "running devstack accidentally. If this is in error, please remove the"
     echo "~/.no-devstack file"
@@ -192,7 +195,7 @@
 
 # Warn users who aren't on an explicitly supported distro, but allow them to
 # override check and attempt installation with ``FORCE=yes ./stack``
-if [[ ! ${DISTRO} =~ (precise|trusty|vivid|wily|7.0|wheezy|sid|testing|jessie|f21|f22|rhel7) ]]; then
+if [[ ! ${DISTRO} =~ (trusty|vivid|wily|7.0|wheezy|sid|testing|jessie|f21|f22|f23|rhel7) ]]; then
     echo "WARNING: this script has not been tested on $DISTRO"
     if [[ "$FORCE" != "yes" ]]; then
         die $LINENO "If you wish to run this script anyway run with FORCE=yes"
@@ -573,7 +576,8 @@
 
 # Generic helper to configure passwords
 function read_password {
-    XTRACE=$(set +o | grep xtrace)
+    local xtrace
+    xtrace=$(set +o | grep xtrace)
     set +o xtrace
     var=$1; msg=$2
     pw=${!var}
@@ -616,7 +620,9 @@
         eval "$var=$pw"
         echo "$var=$pw" >> $localrc
     fi
-    $XTRACE
+
+    # restore previous xtrace value
+    $xtrace
 }
 
 
@@ -925,8 +931,8 @@
 restart_rpc_backend
 
 
-# Export Certicate Authority Bundle
-# ---------------------------------
+# Export Certificate Authority Bundle
+# -----------------------------------
 
 # If certificates were used and written to the SSL bundle file then these
 # should be exported so clients can validate their connections.
diff --git a/stackrc b/stackrc
index f400047..16621f1 100644
--- a/stackrc
+++ b/stackrc
@@ -72,18 +72,6 @@
     ENABLED_SERVICES+=,rabbit,tempest,mysql,dstat
 fi
 
-# SQLAlchemy supports multiple database drivers for each database server
-# type. For example, deployer may use MySQLdb, MySQLConnector, or oursql
-# to access MySQL database.
-#
-# When defined, the variable controls which database driver is used to
-# connect to database server. Otherwise using default driver defined for
-# each database type.
-#
-# You can find the list of currently supported drivers for each database
-# type at: http://docs.sqlalchemy.org/en/rel_0_9/core/engines.html
-# SQLALCHEMY_DATABASE_DRIVER="mysqldb"
-
 # Global toggle for enabling services under mod_wsgi. If this is set to
 # ``True`` all services that use HTTPD + mod_wsgi as the preferred method of
 # deployment, will be deployed under Apache. If this is set to ``False`` all
@@ -91,7 +79,7 @@
 ENABLE_HTTPD_MOD_WSGI_SERVICES=True
 
 # Set the default Nova APIs to enable
-NOVA_ENABLED_APIS=ec2,osapi_compute,metadata
+NOVA_ENABLED_APIS=osapi_compute,metadata
 
 # Set the root URL for Horizon
 HORIZON_APACHE_ROOT="/dashboard"
@@ -118,6 +106,17 @@
     source $RC_DIR/.localrc.password
 fi
 
+# Control whether Python 3 should be used.
+export USE_PYTHON3=${USE_PYTHON3:-False}
+
+# When Python 3 is supported by an application, adding the specific
+# version of Python 3 to this variable will install the app using that
+# version of the interpreter instead of 2.7.
+export PYTHON3_VERSION=${PYTHON3_VERSION:-3.4}
+
+# Just to be more explicit on the Python 2 version to use.
+export PYTHON2_VERSION=${PYTHON2_VERSION:-2.7}
+
 # allow local overrides of env variables, including repo config
 if [[ -f $RC_DIR/localrc ]]; then
     # Old-style user-supplied config
@@ -214,10 +213,6 @@
 HORIZON_REPO=${HORIZON_REPO:-${GIT_BASE}/openstack/horizon.git}
 HORIZON_BRANCH=${HORIZON_BRANCH:-master}
 
-# baremetal provisioning service
-IRONIC_REPO=${IRONIC_REPO:-${GIT_BASE}/openstack/ironic.git}
-IRONIC_BRANCH=${IRONIC_BRANCH:-master}
-
 # unified auth system (manages accounts/tokens)
 KEYSTONE_REPO=${KEYSTONE_REPO:-${GIT_BASE}/openstack/keystone.git}
 KEYSTONE_BRANCH=${KEYSTONE_BRANCH:-master}
@@ -268,6 +263,7 @@
 ##############
 #
 #  OpenStack Client Library Components
+#   Note default install is from pip, see LIBS_FROM_GIT
 #
 ##############
 
@@ -286,6 +282,8 @@
 # ironic client
 GITREPO["python-ironicclient"]=${IRONICCLIENT_REPO:-${GIT_BASE}/openstack/python-ironicclient.git}
 GITBRANCH["python-ironicclient"]=${IRONICCLIENT_BRANCH:-master}
+# ironic plugin is out of tree, but nova uses it. set GITDIR here.
+GITDIR["python-ironicclient"]=$DEST/python-ironicclient
 
 # the base authentication plugins that clients use to authenticate
 GITREPO["keystoneauth"]=${KEYSTONEAUTH_REPO:-${GIT_BASE}/openstack/keystoneauth.git}
@@ -317,6 +315,7 @@
 ###################
 #
 #  Oslo Libraries
+#   Note default install is from pip, see LIBS_FROM_GIT
 #
 ###################
 
@@ -471,6 +470,8 @@
 # ironic common lib
 GITREPO["ironic-lib"]=${IRONIC_LIB_REPO:-${GIT_BASE}/openstack/ironic-lib.git}
 GITBRANCH["ironic-lib"]=${IRONIC_LIB_BRANCH:-master}
+# this doesn't exist in a lib file, so set it here
+GITDIR["ironic-lib"]=$DEST/ironic-lib
 
 
 ##################
@@ -671,6 +672,9 @@
 # Service startup timeout
 SERVICE_TIMEOUT=${SERVICE_TIMEOUT:-60}
 
+# Service graceful shutdown timeout
+SERVICE_GRACEFUL_SHUTDOWN_TIMEOUT=${SERVICE_GRACEFUL_SHUTDOWN_TIMEOUT:-5}
+
 # Support alternative yum -- in future Fedora 'dnf' will become the
 # only supported installer, but for now 'yum' and 'dnf' are both
 # available in parallel with compatible CLIs.  Allow manual switching
@@ -766,6 +770,16 @@
 # Use native SSL for servers in ``SSL_ENABLED_SERVICES``
 USE_SSL=$(trueorfalse False USE_SSL)
 
+# ebtables is inherently racey. If you run it by two or more processes
+# simultaneously it will collide, badly, in the kernel and produce
+# failures or corruption of ebtables. The only way around it is for
+# all tools running ebtables to only ever do so with the --concurrent
+# flag. This requires libvirt >= 1.2.11.
+#
+# If you don't have this then the following work around will replace
+# ebtables with a wrapper script so that it is safe to run without
+# that flag.
+EBTABLES_RACE_FIX=$(trueorfalse False EBTABLES_RACE_FIX)
 
 # Following entries need to be last items in file
 
diff --git a/tests/run-process.sh b/tests/run-process.sh
index bdf1395..301b9a0 100755
--- a/tests/run-process.sh
+++ b/tests/run-process.sh
@@ -5,7 +5,7 @@
 #
 # Set USE_SCREEN True|False to change use of screen.
 #
-# This script emulates the basic exec envirnment in ``stack.sh`` to test
+# This script emulates the basic exec environment in ``stack.sh`` to test
 # the process spawn and kill operations.
 
 if [[ -z $1 ]]; then
diff --git a/tests/test_meta_config.sh b/tests/test_meta_config.sh
index f3e94af..327fb56 100755
--- a/tests/test_meta_config.sh
+++ b/tests/test_meta_config.sh
@@ -24,7 +24,7 @@
 }
 
 # mock function-common:die so that it does not
-# interupt our test script
+# interrupt our test script
 function die {
     exit -1
 }
diff --git a/tests/test_package_ordering.sh b/tests/test_package_ordering.sh
new file mode 100755
index 0000000..a568abf
--- /dev/null
+++ b/tests/test_package_ordering.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# basic test to ensure that package-install files remain sorted
+# alphabetically.
+
+TOP=$(cd $(dirname "$0")/.. && pwd)
+
+source $TOP/tests/unittest.sh
+
+PKG_FILES=$(find $TOP/files/debs $TOP/files/rpms $TOP/files/rpms-suse -type f)
+
+TMPDIR=$(mktemp -d)
+
+SORTED=${TMPDIR}/sorted
+UNSORTED=${TMPDIR}/unsorted
+
+for p in $PKG_FILES; do
+    grep -v '^#' $p > ${UNSORTED}
+    sort ${UNSORTED} > ${SORTED}
+
+    if [ -n "$(diff -c ${UNSORTED} ${SORTED})" ]; then
+        failed "$p is unsorted"
+        # output this, it's helpful to see what exactly is unsorted
+        diff -c ${UNSORTED} ${SORTED}
+    else
+        passed "$p is sorted"
+    fi
+done
+
+rm -rf ${TMPDIR}
+
+report_results
diff --git a/tests/unittest.sh b/tests/unittest.sh
index df7a8b4..26b5b8e 100644
--- a/tests/unittest.sh
+++ b/tests/unittest.sh
@@ -46,7 +46,7 @@
     ERROR=$((ERROR+1))
 }
 
-# assert string comparision of val1 equal val2, printing out msg
+# assert string comparison of val1 equal val2, printing out msg
 #  usage: assert_equal val1 val2 msg
 function assert_equal {
     local lineno
@@ -92,16 +92,17 @@
     fi
 }
 
-# print a summary of passing and failing tests, exiting
-# with an error if we have failed tests
+# Print a summary of passing and failing tests and exit
+# (with an error if we have failed tests)
 #  usage: report_results
 function report_results {
     echo "$PASS Tests PASSED"
-    if [[ $ERROR -gt 1 ]]; then
+    if [[ $ERROR -gt 0 ]]; then
         echo
         echo "The following $ERROR tests FAILED"
         echo -e "$FAILED_FUNCS"
         echo "---"
         exit 1
     fi
+    exit 0
 }
diff --git a/tools/install_ebtables_workaround.sh b/tools/install_ebtables_workaround.sh
new file mode 100755
index 0000000..45ced87
--- /dev/null
+++ b/tools/install_ebtables_workaround.sh
@@ -0,0 +1,31 @@
+#!/bin/bash -eu
+#
+# Copyright 2015 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+#
+# This replaces the ebtables on your system with a wrapper script that
+# does implicit locking. This is needed if libvirt < 1.2.11 on your platform.
+
+EBTABLES=/sbin/ebtables
+EBTABLESREAL=/sbin/ebtables.real
+FILES=$TOP_DIR/files
+
+if [[ -f "$EBTABLES" ]]; then
+    if file $EBTABLES | grep ELF; then
+        sudo mv $EBTABLES $EBTABLESREAL
+        sudo install -m 0755 $FILES/ebtables.workaround $EBTABLES
+        echo "Replaced ebtables with locking workaround"
+    fi
+fi
diff --git a/tools/install_pip.sh b/tools/install_pip.sh
index ab5efb2..f239c7b 100755
--- a/tools/install_pip.sh
+++ b/tools/install_pip.sh
@@ -8,6 +8,7 @@
 
 # Assumptions:
 # - update pip to $INSTALL_PIP_VERSION
+# - if USE_PYTHON3=True, PYTHON3_VERSION refers to a version already installed
 
 set -o errexit
 set -o xtrace
@@ -31,6 +32,8 @@
 echo "Distro: $DISTRO"
 
 function get_versions {
+    # FIXME(dhellmann): Deal with multiple python versions here? This
+    # is just used for reporting, so maybe not?
     PIP=$(which pip 2>/dev/null || which pip-python 2>/dev/null || true)
     if [[ -n $PIP ]]; then
         PIP_VERSION=$($PIP --version | awk '{ print $2}')
@@ -75,6 +78,9 @@
         touch $LOCAL_PIP.downloaded
     fi
     sudo -H -E python $LOCAL_PIP
+    if python3_enabled; then
+        sudo -H -E python${PYTHON3_VERSION} $LOCAL_PIP
+    fi
 }
 
 
@@ -114,6 +120,7 @@
 # python in f23 depends on the python-pip package
 if ! { is_fedora && [[ $DISTRO == "f23" ]]; }; then
     uninstall_package python-pip
+    uninstall_package python3-pip
 fi
 
 install_get_pip
@@ -122,6 +129,7 @@
     configure_pypi_alternative_url
 fi
 
+set -x
 pip_install -U setuptools
 
 get_versions
diff --git a/tools/install_prereqs.sh b/tools/install_prereqs.sh
index 38452cd..031f8a8 100755
--- a/tools/install_prereqs.sh
+++ b/tools/install_prereqs.sh
@@ -81,6 +81,9 @@
     fi
 fi
 
+if python3_enabled; then
+    install_python3
+fi
 
 # Mark end of run
 # ---------------
diff --git a/tools/ironic/scripts/cleanup-node b/tools/ironic/scripts/cleanup-node
deleted file mode 100755
index c4e4e70..0000000
--- a/tools/ironic/scripts/cleanup-node
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/usr/bin/env bash
-
-# **cleanup-nodes**
-
-# Cleans up baremetal poseur nodes and volumes created during ironic setup
-# Assumes calling user has proper libvirt group membership and access.
-
-set -exu
-
-LIBVIRT_STORAGE_POOL=${LIBVIRT_STORAGE_POOL:-"default"}
-LIBVIRT_CONNECT_URI=${LIBVIRT_CONNECT_URI:-"qemu:///system"}
-
-NAME=$1
-NETWORK_BRIDGE=$2
-
-export VIRSH_DEFAULT_CONNECT_URI=$LIBVIRT_CONNECT_URI
-
-VOL_NAME="$NAME.qcow2"
-virsh list | grep -q $NAME && virsh destroy $NAME
-virsh list --inactive | grep -q $NAME && virsh undefine $NAME
-
-if virsh pool-list | grep -q $LIBVIRT_STORAGE_POOL ; then
-  virsh vol-list $LIBVIRT_STORAGE_POOL | grep -q $VOL_NAME &&
-      virsh vol-delete $VOL_NAME --pool $LIBVIRT_STORAGE_POOL
-fi
diff --git a/tools/ironic/scripts/configure-vm b/tools/ironic/scripts/configure-vm
deleted file mode 100755
index 378fcb8..0000000
--- a/tools/ironic/scripts/configure-vm
+++ /dev/null
@@ -1,93 +0,0 @@
-#!/usr/bin/env python
-
-import argparse
-import os.path
-
-import libvirt
-
-templatedir = os.path.join(os.path.dirname(os.path.dirname(__file__)),
-                           'templates')
-
-
-CONSOLE_LOG = """
-    <serial type='file'>
-      <source path='%(console_log)s'/>
-      <target port='0'/>
-      <alias name='serial0'/>
-    </serial>
-    <serial type='pty'>
-      <source path='/dev/pts/49'/>
-      <target port='1'/>
-      <alias name='serial1'/>
-    </serial>
-    <console type='file'>
-      <source path='%(console_log)s'/>
-      <target type='serial' port='0'/>
-      <alias name='serial0'/>
-    </console>
-"""
-
-
-def main():
-    parser = argparse.ArgumentParser(
-        description="Configure a kvm virtual machine for the seed image.")
-    parser.add_argument('--name', default='seed',
-                        help='the name to give the machine in libvirt.')
-    parser.add_argument('--image',
-                        help='Use a custom image file (must be qcow2).')
-    parser.add_argument('--engine', default='qemu',
-                        help='The virtualization engine to use')
-    parser.add_argument('--arch', default='i686',
-                        help='The architecture to use')
-    parser.add_argument('--memory', default='2097152',
-                        help="Maximum memory for the VM in KB.")
-    parser.add_argument('--cpus', default='1',
-                        help="CPU count for the VM.")
-    parser.add_argument('--bootdev', default='hd',
-                        help="What boot device to use (hd/network).")
-    parser.add_argument('--network', default="brbm",
-                        help='The libvirt network name to use')
-    parser.add_argument('--libvirt-nic-driver', default='e1000',
-                        help='The libvirt network driver to use')
-    parser.add_argument('--console-log',
-                        help='File to log console')
-    parser.add_argument('--emulator', default=None,
-                        help='Path to emulator bin for vm template')
-    args = parser.parse_args()
-    with file(templatedir + '/vm.xml', 'rb') as f:
-        source_template = f.read()
-    params = {
-        'name': args.name,
-        'imagefile': args.image,
-        'engine': args.engine,
-        'arch': args.arch,
-        'memory': args.memory,
-        'cpus': args.cpus,
-        'bootdev': args.bootdev,
-        'network': args.network,
-        'nicdriver': args.libvirt_nic_driver,
-        'emulator': args.emulator,
-    }
-
-    if args.emulator:
-        params['emulator'] = args.emulator
-    else:
-        if os.path.exists("/usr/bin/kvm"):  # Debian
-            params['emulator'] = "/usr/bin/kvm"
-        elif os.path.exists("/usr/bin/qemu-kvm"):  # Redhat
-            params['emulator'] = "/usr/bin/qemu-kvm"
-
-    if args.console_log:
-        params['bios_serial'] = "<bios useserial='yes'/>"
-        params['console_log'] = CONSOLE_LOG % {'console_log': args.console_log}
-    else:
-        params['bios_serial'] = ''
-        params['console_log'] = ''
-    libvirt_template = source_template % params
-    conn = libvirt.open("qemu:///system")
-
-    a = conn.defineXML(libvirt_template)
-    print ("Created machine %s with UUID %s" % (args.name, a.UUIDString()))
-
-if __name__ == '__main__':
-    main()
diff --git a/tools/ironic/scripts/create-node b/tools/ironic/scripts/create-node
deleted file mode 100755
index b018acd..0000000
--- a/tools/ironic/scripts/create-node
+++ /dev/null
@@ -1,79 +0,0 @@
-#!/usr/bin/env bash
-
-# **create-nodes**
-
-# Creates baremetal poseur nodes for ironic testing purposes
-
-set -ex
-
-# Keep track of the DevStack directory
-TOP_DIR=$(cd $(dirname "$0")/.. && pwd)
-
-NAME=$1
-CPU=$2
-MEM=$(( 1024 * $3 ))
-# Extra G to allow fuzz for partition table : flavor size and registered size
-# need to be different to actual size.
-DISK=$(( $4 + 1))
-
-case $5 in
-    i386) ARCH='i686' ;;
-    amd64) ARCH='x86_64' ;;
-    *) echo "Unsupported arch $4!" ; exit 1 ;;
-esac
-
-BRIDGE=$6
-EMULATOR=$7
-LOGDIR=$8
-
-LIBVIRT_NIC_DRIVER=${LIBVIRT_NIC_DRIVER:-"e1000"}
-LIBVIRT_STORAGE_POOL=${LIBVIRT_STORAGE_POOL:-"default"}
-LIBVIRT_CONNECT_URI=${LIBVIRT_CONNECT_URI:-"qemu:///system"}
-
-export VIRSH_DEFAULT_CONNECT_URI=$LIBVIRT_CONNECT_URI
-
-if ! virsh pool-list --all | grep -q $LIBVIRT_STORAGE_POOL; then
-    virsh pool-define-as --name $LIBVIRT_STORAGE_POOL dir --target /var/lib/libvirt/images >&2
-    virsh pool-autostart $LIBVIRT_STORAGE_POOL >&2
-    virsh pool-start $LIBVIRT_STORAGE_POOL >&2
-fi
-
-pool_state=$(virsh pool-info $LIBVIRT_STORAGE_POOL | grep State | awk '{ print $2 }')
-if [ "$pool_state" != "running" ] ; then
-  [ ! -d /var/lib/libvirt/images ] && sudo mkdir /var/lib/libvirt/images
-  virsh pool-start $LIBVIRT_STORAGE_POOL >&2
-fi
-
-if [ -n "$LOGDIR" ] ; then
-  mkdir -p "$LOGDIR"
-fi
-
-PREALLOC=
-if [ -f /etc/debian_version ]; then
-    PREALLOC="--prealloc-metadata"
-fi
-
-if [ -n "$LOGDIR" ] ; then
-  VM_LOGGING="--console-log $LOGDIR/${NAME}_console.log"
-else
-  VM_LOGGING=""
-fi
-VOL_NAME="${NAME}.qcow2"
-
-if ! virsh list --all | grep -q $NAME; then
-  virsh vol-list --pool $LIBVIRT_STORAGE_POOL | grep -q $VOL_NAME &&
-      virsh vol-delete $VOL_NAME --pool $LIBVIRT_STORAGE_POOL >&2
-  virsh vol-create-as $LIBVIRT_STORAGE_POOL ${VOL_NAME} ${DISK}G --format qcow2 $PREALLOC >&2
-  volume_path=$(virsh vol-path --pool $LIBVIRT_STORAGE_POOL $VOL_NAME)
-  # Pre-touch the VM to set +C, as it can only be set on empty files.
-  sudo touch "$volume_path"
-  sudo chattr +C "$volume_path" || true
-  $TOP_DIR/scripts/configure-vm \
-    --bootdev network --name $NAME --image "$volume_path" \
-    --arch $ARCH --cpus $CPU --memory $MEM --libvirt-nic-driver $LIBVIRT_NIC_DRIVER \
-    --emulator $EMULATOR --network $BRIDGE $VM_LOGGING >&2
-
-fi
-
-# echo mac
-virsh dumpxml $NAME | grep "mac address" | head -1 | cut -d\' -f2
diff --git a/tools/ironic/scripts/setup-network b/tools/ironic/scripts/setup-network
deleted file mode 100755
index 83308ed..0000000
--- a/tools/ironic/scripts/setup-network
+++ /dev/null
@@ -1,28 +0,0 @@
-#!/usr/bin/env bash
-
-# **setup-network**
-
-# Setups openvswitch libvirt network suitable for
-# running baremetal poseur nodes for ironic testing purposes
-
-set -exu
-
-LIBVIRT_CONNECT_URI=${LIBVIRT_CONNECT_URI:-"qemu:///system"}
-
-# Keep track of the DevStack directory
-TOP_DIR=$(cd $(dirname "$0")/.. && pwd)
-BRIDGE_SUFFIX=${1:-''}
-BRIDGE_NAME=brbm$BRIDGE_SUFFIX
-
-export VIRSH_DEFAULT_CONNECT_URI="$LIBVIRT_CONNECT_URI"
-
-# Only add bridge if missing
-(sudo ovs-vsctl list-br | grep ${BRIDGE_NAME}$) || sudo ovs-vsctl add-br ${BRIDGE_NAME}
-
-# Remove bridge before replacing it.
-(virsh net-list | grep "${BRIDGE_NAME} ") && virsh net-destroy ${BRIDGE_NAME}
-(virsh net-list --inactive  | grep "${BRIDGE_NAME} ") && virsh net-undefine ${BRIDGE_NAME}
-
-virsh net-define <(sed s/brbm/$BRIDGE_NAME/ $TOP_DIR/templates/brbm.xml)
-virsh net-autostart ${BRIDGE_NAME}
-virsh net-start ${BRIDGE_NAME}
diff --git a/tools/ironic/templates/brbm.xml b/tools/ironic/templates/brbm.xml
deleted file mode 100644
index 0769d3f..0000000
--- a/tools/ironic/templates/brbm.xml
+++ /dev/null
@@ -1,6 +0,0 @@
-<network>
-  <name>brbm</name>
-  <forward mode='bridge'/>
-  <bridge name='brbm'/>
-  <virtualport type='openvswitch'/>
-</network>
diff --git a/tools/ironic/templates/tftpd-xinetd.template b/tools/ironic/templates/tftpd-xinetd.template
deleted file mode 100644
index 5f3d03f..0000000
--- a/tools/ironic/templates/tftpd-xinetd.template
+++ /dev/null
@@ -1,14 +0,0 @@
-service tftp
-{
-  protocol        = udp
-  port            = 69
-  socket_type     = dgram
-  wait            = yes
-  user            = root
-  server          = /usr/sbin/in.tftpd
-  server_args     = -v -v -v -v -v --map-file %TFTPBOOT_DIR%/map-file %TFTPBOOT_DIR%
-  disable         = no
-  # This is a workaround for Fedora, where TFTP will listen only on
-  # IPv6 endpoint, if IPv4 flag is not used.
-  flags           = IPv4
-}
diff --git a/tools/ironic/templates/vm.xml b/tools/ironic/templates/vm.xml
deleted file mode 100644
index ae7d685..0000000
--- a/tools/ironic/templates/vm.xml
+++ /dev/null
@@ -1,49 +0,0 @@
-<domain type='%(engine)s'>
-  <name>%(name)s</name>
-  <memory unit='KiB'>%(memory)s</memory>
-  <vcpu>%(cpus)s</vcpu>
-  <os>
-    <type arch='%(arch)s' machine='pc-1.0'>hvm</type>
-    <boot dev='%(bootdev)s'/>
-    <bootmenu enable='no'/>
-    %(bios_serial)s
-  </os>
-  <features>
-    <acpi/>
-    <apic/>
-    <pae/>
-  </features>
-  <clock offset='utc'/>
-  <on_poweroff>destroy</on_poweroff>
-  <on_reboot>restart</on_reboot>
-  <on_crash>restart</on_crash>
-  <devices>
-    <emulator>%(emulator)s</emulator>
-    <disk type='file' device='disk'>
-      <driver name='qemu' type='qcow2' cache='writeback'/>
-      <source file='%(imagefile)s'/>
-      <target dev='vda' bus='virtio'/>
-      <address type='pci' domain='0x0000' bus='0x00' slot='0x06' function='0x0'/>
-    </disk>
-    <controller type='ide' index='0'>
-      <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x1'/>
-    </controller>
-    <interface type='network'>
-      <source network='%(network)s'/>
-      <virtualport type='openvswitch'/>
-      <model type='%(nicdriver)s'/>
-      <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>
-    </interface>
-    <input type='mouse' bus='ps2'/>
-    <graphics type='vnc' port='-1' autoport='yes'/>
-    <video>
-      <model type='cirrus' vram='9216' heads='1'/>
-      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
-    </video>
-    %(console_log)s
-    <memballoon model='virtio'>
-      <address type='pci' domain='0x0000' bus='0x00' slot='0x07' function='0x0'/>
-    </memballoon>
-  </devices>
-</domain>
-
diff --git a/tox.ini b/tox.ini
index 9279455..f9d04f2 100644
--- a/tox.ini
+++ b/tox.ini
@@ -12,20 +12,20 @@
    {env:BASHATE_INSTALL_PATH:bashate==0.3.2}
 whitelist_externals = bash
 commands = bash -c "find {toxinidir}             \
-         -not \( -type d -name .?\* -prune \)    \ # prune all 'dot' dirs
-         -not \( -type d -name doc -prune \)     \ # skip documentation
-         -not \( -type d -name shocco -prune \)  \ # skip shocco
-         -type f                                 \ # only files
-         -not -name \*~                          \ # skip editors, readme, etc
+         -not \( -type d -name .?\* -prune \)    \
+         -not \( -type d -name doc -prune \)     \
+         -not \( -type d -name shocco -prune \)  \
+         -type f                                 \
+         -not -name \*~                          \
          -not -name \*.md                        \
          \(                                      \
           -name \*.sh -or                        \
           -name \*.orig -or                      \
-          -name \*rc -or                         \ # openrc files, etc
+          -name \*rc -or                         \
           -name functions\* -or                  \
-          -wholename \*/inc/\* -or               \ # /inc files and
-          -wholename \*/lib/\*                   \ # /lib files are shell, but
-         \)                                      \ #   have no extension
+          -wholename \*/inc/\* -or               \
+          -wholename \*/lib/\*                   \
+         \)                                      \
          -print0 | xargs -0 bashate -v -iE006 -eE005,E042"
 
 [testenv:docs]