Merge "Change the way to get conntrack-tools version"
diff --git a/clean.sh b/clean.sh
index e369eda..90b21eb 100755
--- a/clean.sh
+++ b/clean.sh
@@ -149,5 +149,10 @@
 
 # Clean up all *.pyc files
 if [[ -n "$DEST" ]] && [[ -d "$DEST" ]]; then
-    sudo find $DEST -name "*.pyc" -print0 | xargs -0 rm
+    find_version=`find --version | awk '{ print $NF; exit}'`
+    if vercmp "$find_version" "<" "4.2.3" ; then
+        sudo find $DEST -name "*.pyc" -print0 | xargs -0 rm
+    else
+        sudo find $DEST -name "*.pyc" -delete
+    fi
 fi
diff --git a/doc/source/plugin-registry.rst b/doc/source/plugin-registry.rst
index 4cfbcb1..2721eda 100644
--- a/doc/source/plugin-registry.rst
+++ b/doc/source/plugin-registry.rst
@@ -80,6 +80,7 @@
 manila-ui                              `git://git.openstack.org/openstack/manila-ui <https://git.openstack.org/cgit/openstack/manila-ui>`__
 masakari                               `git://git.openstack.org/openstack/masakari <https://git.openstack.org/cgit/openstack/masakari>`__
 meteos                                 `git://git.openstack.org/openstack/meteos <https://git.openstack.org/cgit/openstack/meteos>`__
+meteos-ui                              `git://git.openstack.org/openstack/meteos-ui <https://git.openstack.org/cgit/openstack/meteos-ui>`__
 mistral                                `git://git.openstack.org/openstack/mistral <https://git.openstack.org/cgit/openstack/mistral>`__
 mixmatch                               `git://git.openstack.org/openstack/mixmatch <https://git.openstack.org/cgit/openstack/mixmatch>`__
 mogan                                  `git://git.openstack.org/openstack/mogan <https://git.openstack.org/cgit/openstack/mogan>`__
diff --git a/functions b/functions
index 0be9794..89ee367 100644
--- a/functions
+++ b/functions
@@ -569,6 +569,19 @@
     esac
 }
 
+# This sets up defaults we like in devstack for logging for tracking
+# down issues, and makes sure everything is done the same between
+# projects.
+function setup_logging {
+    local conf_file=$1
+    local other_cond=${2:-"False"}
+    if [ "$LOG_COLOR" == "True" ] && [ "$SYSLOG" == "False" ] && [ "$other_cond" == "False" ]; then
+        setup_colorized_logging $conf_file
+    else
+        setup_standard_logging_identity $conf_file
+    fi
+}
+
 # This function sets log formatting options for colorizing log
 # output to stdout. It is meant to be called by lib modules.
 # The last two parameters are optional and can be used to specify
@@ -578,16 +591,21 @@
 # setup_colorized_logging something.conf SOMESECTION
 function setup_colorized_logging {
     local conf_file=$1
-    local conf_section=$2
-    local project_var=${3:-"project_name"}
-    local user_var=${4:-"user_name"}
+    local conf_section="DEFAULT"
+    local project_var="project_name"
+    local user_var="user_name"
     # Add color to logging output
-    iniset $conf_file $conf_section logging_context_format_string "%(asctime)s.%(msecs)03d %(color)s%(levelname)s %(name)s [%(request_id)s %("$user_var")s %("$project_var")s%(color)s] %(instance)s%(color)s%(message)s"
+    iniset $conf_file $conf_section logging_context_format_string "%(asctime)s.%(msecs)03d %(color)s%(levelname)s %(name)s [%(request_id)s %("$project_var")s %("$user_var")s%(color)s] %(instance)s%(color)s%(message)s"
     iniset $conf_file $conf_section logging_default_format_string "%(asctime)s.%(msecs)03d %(color)s%(levelname)s %(name)s [-%(color)s] %(instance)s%(color)s%(message)s"
     iniset $conf_file $conf_section logging_debug_format_suffix "from (pid=%(process)d) %(funcName)s %(pathname)s:%(lineno)d"
     iniset $conf_file $conf_section logging_exception_prefix "%(color)s%(asctime)s.%(msecs)03d TRACE %(name)s %(instance)s"
 }
 
+function setup_standard_logging_identity {
+    local conf_file=$1
+    iniset $conf_file DEFAULT logging_user_identity_format "%(project_name)s %(user_name)s"
+}
+
 # These functions are provided for basic fall-back functionality for
 # projects that include parts of DevStack (Grenade).  stack.sh will
 # override these with more specific versions for DevStack (with fancy
diff --git a/inc/python b/inc/python
index 04cde34..5afc07f 100644
--- a/inc/python
+++ b/inc/python
@@ -441,6 +441,13 @@
     fi
 }
 
+function install_devstack_tools {
+    # intentionally old to ensure devstack-gate has control
+    local dstools_version=${DSTOOLS_VERSION:-0.1.2}
+    install_python3
+    sudo pip3 install -U devstack-tools==${dstools_version}
+}
+
 # Restore xtrace
 $INC_PY_TRACE
 
diff --git a/lib/cinder b/lib/cinder
index 40f0f16..870ee0b 100644
--- a/lib/cinder
+++ b/lib/cinder
@@ -333,12 +333,7 @@
     iniset $CINDER_CONF DEFAULT volume_clear $CINDER_VOLUME_CLEAR
 
     # Format logging
-    if [ "$LOG_COLOR" == "True" ] && [ "$SYSLOG" == "False" ] && [ "$CINDER_USE_MOD_WSGI" == "False" ]; then
-        setup_colorized_logging $CINDER_CONF DEFAULT "project_id" "user_id"
-    else
-        # Set req-id, project-name and resource in log format
-        iniset $CINDER_CONF DEFAULT logging_context_format_string "%(asctime)s.%(msecs)03d %(levelname)s %(name)s [%(request_id)s %(project_name)s] %(resource)s%(message)s"
-    fi
+    setup_logging $CINDER_CONF $CINDER_USE_MOD_WSGI
 
     if [ "$CINDER_USE_MOD_WSGI" == "True" ]; then
         _cinder_config_apache_wsgi
diff --git a/lib/glance b/lib/glance
index 4ba1d20..26c4150 100644
--- a/lib/glance
+++ b/lib/glance
@@ -229,10 +229,8 @@
     fi
 
     # Format logging
-    if [ "$LOG_COLOR" == "True" ] && [ "$SYSLOG" == "False" ]; then
-        setup_colorized_logging $GLANCE_API_CONF DEFAULT tenant user
-        setup_colorized_logging $GLANCE_REGISTRY_CONF DEFAULT tenant user
-    fi
+    setup_logging $GLANCE_API_CONF
+    setup_logging $GLANCE_REGISTRY_CONF
 
     cp -p $GLANCE_DIR/etc/glance-registry-paste.ini $GLANCE_REGISTRY_PASTE_INI
 
@@ -273,7 +271,7 @@
     if is_service_enabled g-glare; then
         local dburl
         dburl=`database_connection_url glance`
-        setup_colorized_logging $GLANCE_GLARE_CONF DEFAULT tenant user
+        setup_logging $GLANCE_GLARE_CONF
         iniset $GLANCE_GLARE_CONF DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL
         iniset $GLANCE_GLARE_CONF DEFAULT bind_host $GLANCE_SERVICE_LISTEN_ADDRESS
         iniset $GLANCE_GLARE_CONF DEFAULT bind_port $GLANCE_GLARE_PORT
diff --git a/lib/keystone b/lib/keystone
index 34730b8..474af8b 100644
--- a/lib/keystone
+++ b/lib/keystone
@@ -284,7 +284,7 @@
 
     # Format logging
     if [ "$LOG_COLOR" == "True" ] && [ "$SYSLOG" == "False" ] && [ "$KEYSTONE_DEPLOY" != "mod_wsgi" ] ; then
-        setup_colorized_logging $KEYSTONE_CONF DEFAULT
+        setup_colorized_logging $KEYSTONE_CONF
     fi
 
     iniset $KEYSTONE_CONF DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL
diff --git a/lib/neutron-legacy b/lib/neutron-legacy
index 29c187e..5ec61f9 100644
--- a/lib/neutron-legacy
+++ b/lib/neutron-legacy
@@ -732,12 +732,7 @@
     fi
 
     # Format logging
-    if [ "$LOG_COLOR" == "True" ] && [ "$SYSLOG" == "False" ]; then
-        setup_colorized_logging $NEUTRON_CONF DEFAULT project_id
-    else
-        # Show user_name and project_name by default like in nova
-        iniset $NEUTRON_CONF DEFAULT logging_user_identity_format "%(user_name)s %(project_name)s"
-    fi
+    setup_logging $NEUTRON_CONF
 
     if is_service_enabled tls-proxy; then
         # Set the service port for a proxy to take the original
diff --git a/lib/nova b/lib/nova
index d57dec1..4d336f6 100644
--- a/lib/nova
+++ b/lib/nova
@@ -457,20 +457,15 @@
     iniset $NOVA_CONF DEFAULT scheduler_driver "$SCHEDULER"
     iniset $NOVA_CONF DEFAULT scheduler_default_filters "$FILTERS"
     iniset $NOVA_CONF DEFAULT default_floating_pool "$PUBLIC_NETWORK_NAME"
-    iniset $NOVA_CONF DEFAULT s3_host "$SERVICE_HOST"
-    iniset $NOVA_CONF DEFAULT s3_port "$S3_SERVICE_PORT"
     if [[ $SERVICE_IP_VERSION == 6 ]]; then
         iniset $NOVA_CONF DEFAULT my_ip "$HOST_IPV6"
         iniset $NOVA_CONF DEFAULT use_ipv6 "True"
     else
         iniset $NOVA_CONF DEFAULT my_ip "$HOST_IP"
     fi
-    iniset $NOVA_CONF database connection `database_connection_url nova`
-    iniset $NOVA_CONF api_database connection `database_connection_url nova_api`
     iniset $NOVA_CONF DEFAULT instance_name_template "${INSTANCE_NAME_PREFIX}%08x"
     iniset $NOVA_CONF DEFAULT osapi_compute_listen "$NOVA_SERVICE_LISTEN_ADDRESS"
     iniset $NOVA_CONF DEFAULT metadata_listen "$NOVA_SERVICE_LISTEN_ADDRESS"
-    iniset $NOVA_CONF DEFAULT s3_listen "$NOVA_SERVICE_LISTEN_ADDRESS"
 
     if is_fedora || is_suse; then
         # nova defaults to /usr/local/bin, but fedora and suse pip like to
@@ -478,6 +473,14 @@
         iniset $NOVA_CONF DEFAULT bindir "/usr/bin"
     fi
 
+    # only setup database connections if there are services that
+    # require them running on the host. The ensures that n-cpu doesn't
+    # leak a need to use the db in a multinode scenario.
+    if is_service_enabled n-api n-cond n-sched; then
+        iniset $NOVA_CONF database connection `database_connection_url nova`
+        iniset $NOVA_CONF api_database connection `database_connection_url nova_api`
+    fi
+
     if is_service_enabled n-api; then
         if is_service_enabled n-api-meta; then
             # If running n-api-meta as a separate service
@@ -519,12 +522,8 @@
         iniset $NOVA_CONF DEFAULT force_config_drive "$FORCE_CONFIG_DRIVE"
     fi
     # Format logging
-    if [ "$LOG_COLOR" == "True" ] && [ "$SYSLOG" == "False" ] && [ "$NOVA_USE_MOD_WSGI" == "False" ]  ; then
-        setup_colorized_logging $NOVA_CONF DEFAULT
-    else
-        # Show user_name and project_name instead of user_id and project_id
-        iniset $NOVA_CONF DEFAULT logging_user_identity_format "%(user_name)s %(project_name)s"
-    fi
+    setup_logging $NOVA_CONF $NOVA_USE_MOD_WSGI
+
     if [ "$NOVA_USE_MOD_WSGI" == "True" ]; then
         _config_nova_apache_wsgi
     fi
@@ -682,10 +681,10 @@
 
         # (Re)create nova databases
         recreate_database nova
-        recreate_database nova_api_cell0
+        recreate_database nova_cell0
 
         # Migrate nova database. If "nova-manage cell_v2 simple_cell_setup" has
-        # been run this migrates the "nova" and "nova_api_cell0" database.
+        # been run this migrates the "nova" and "nova_cell0" database.
         # Otherwise it just migrates the "nova" database.
         $NOVA_BIN_DIR/nova-manage --config-file $NOVA_CONF db sync
 
@@ -946,11 +945,12 @@
 
 # create_cell(): Group the available hosts into a cell
 function create_cell {
+    # NOTE(danms): map_cell0 always returns 1 right now; remove this when that is fixed
+    (nova-manage cell_v2 map_cell0 --database_connection `database_connection_url nova_cell0`|| true)
+
     if ! is_service_enabled n-cell; then
         nova-manage cell_v2 simple_cell_setup --transport-url $(get_transport_url)
     else
-        # NOTE(danms): map_cell0 always returns 1 right now; remove this when that is fixed
-        (nova-manage cell_v2 map_cell0 || true)
         nova-manage --config-file $NOVA_CELLS_CONF  --verbose cell_v2 map_cell_and_hosts \
                 --transport-url $(get_transport_url child_cell) --name 'cell1'
         nova-manage db sync
diff --git a/lib/tempest b/lib/tempest
index 5a07229..7cafadb 100644
--- a/lib/tempest
+++ b/lib/tempest
@@ -434,7 +434,11 @@
     iniset $TEMPEST_CONFIG validation network_for_ssh $TEMPEST_SSH_NETWORK_NAME
 
     # Volume
-    iniset $TEMPEST_CONFIG volume-feature-enabled manage_snapshot $(trueorfalse True TEMPEST_VOLUME_MANAGE_SNAPSHOT)
+    # Only turn on TEMPEST_VOLUME_MANAGE_SNAPSHOT by default for "lvm" backends
+    if [[ "$CINDER_ENABLED_BACKENDS" == *"lvm"* ]]; then
+        TEMPEST_VOLUME_MANAGE_SNAPSHOT=${TEMPEST_VOLUME_MANAGE_SNAPSHOT:-True}
+    fi
+    iniset $TEMPEST_CONFIG volume-feature-enabled manage_snapshot $(trueorfalse False TEMPEST_VOLUME_MANAGE_SNAPSHOT)
     # TODO(ynesenenko): Remove the volume_services flag when Liberty and Kilo will correct work with host info.
     iniset $TEMPEST_CONFIG volume-feature-enabled volume_services True
     # TODO(ameade): Remove the api_v3 flag when Mitaka and Liberty are end of life.
@@ -491,12 +495,6 @@
 
     # Baremetal
     if [ "$VIRT_DRIVER" = "ironic" ] ; then
-        iniset $TEMPEST_CONFIG baremetal driver_enabled True
-        iniset $TEMPEST_CONFIG baremetal unprovision_timeout $BUILD_TIMEOUT
-        iniset $TEMPEST_CONFIG baremetal active_timeout $BUILD_TIMEOUT
-        iniset $TEMPEST_CONFIG baremetal deploywait_timeout $BUILD_TIMEOUT
-        iniset $TEMPEST_CONFIG baremetal deploy_img_dir $FILES
-        iniset $TEMPEST_CONFIG baremetal node_uuid $IRONIC_NODE_UUID
         iniset $TEMPEST_CONFIG compute-feature-enabled change_password False
         iniset $TEMPEST_CONFIG compute-feature-enabled console_output False
         iniset $TEMPEST_CONFIG compute-feature-enabled interface_attach False
@@ -509,13 +507,19 @@
         iniset $TEMPEST_CONFIG compute-feature-enabled suspend False
     fi
 
-    # Libvirt-LXC
-    if [ "$VIRT_DRIVER" = "libvirt" ] && [ "$LIBVIRT_TYPE" = "lxc" ]; then
-        iniset $TEMPEST_CONFIG compute-feature-enabled rescue False
-        iniset $TEMPEST_CONFIG compute-feature-enabled resize False
-        iniset $TEMPEST_CONFIG compute-feature-enabled shelve False
-        iniset $TEMPEST_CONFIG compute-feature-enabled snapshot False
-        iniset $TEMPEST_CONFIG compute-feature-enabled suspend False
+    # Libvirt
+    if [ "$VIRT_DRIVER" = "libvirt" ]; then
+        # Libvirt-LXC
+        if [ "$LIBVIRT_TYPE" = "lxc" ]; then
+            iniset $TEMPEST_CONFIG compute-feature-enabled rescue False
+            iniset $TEMPEST_CONFIG compute-feature-enabled resize False
+            iniset $TEMPEST_CONFIG compute-feature-enabled shelve False
+            iniset $TEMPEST_CONFIG compute-feature-enabled snapshot False
+            iniset $TEMPEST_CONFIG compute-feature-enabled suspend False
+        elif ! is_service_enabled n-cell; then
+            # cells v1 does not support swapping volumes
+            iniset $TEMPEST_CONFIG compute-feature-enabled swap_volume True
+        fi
     fi
 
     # ``service_available``
diff --git a/stackrc b/stackrc
index 19f5b53..d8d0ee4 100644
--- a/stackrc
+++ b/stackrc
@@ -647,14 +647,9 @@
             IMAGE_URLS+="http://ca.downloads.xensource.com/OpenStack/cirros-0.3.4-x86_64-disk.vhd.tgz"
             IMAGE_URLS+=",http://download.cirros-cloud.net/${CIRROS_VERSION}/cirros-${CIRROS_VERSION}-x86_64-uec.tar.gz";;
         ironic)
-            # Ironic can do both partition and full disk images, depending on the driver
-            if [[ -z "${IRONIC_DEPLOY_DRIVER%%agent*}" ]]; then
-                DEFAULT_IMAGE_NAME=${DEFAULT_IMAGE_NAME:-cirros-${CIRROS_VERSION}-x86_64-disk}
-            else
-                DEFAULT_IMAGE_NAME=${DEFAULT_IMAGE_NAME:-cirros-${CIRROS_VERSION}-x86_64-uec}
-            fi
-            IMAGE_URLS+="http://download.cirros-cloud.net/${CIRROS_VERSION}/cirros-${CIRROS_VERSION}-x86_64-uec.tar.gz"
-            IMAGE_URLS+=",http://download.cirros-cloud.net/${CIRROS_VERSION}/cirros-${CIRROS_VERSION}-x86_64-disk.img";;
+            # NOTE(lucasagomes): The logic setting the default image
+            # now lives in the Ironic tree
+            ;;
         *) # Default to Cirros with kernel, ramdisk and disk image
             DEFAULT_IMAGE_NAME=${DEFAULT_IMAGE_NAME:-cirros-${CIRROS_VERSION}-${CIRROS_ARCH}-uec}
             IMAGE_URLS+="http://download.cirros-cloud.net/${CIRROS_VERSION}/cirros-${CIRROS_VERSION}-${CIRROS_ARCH}-uec.tar.gz";;
diff --git a/tools/discover_hosts.sh b/tools/discover_hosts.sh
new file mode 100755
index 0000000..4ec6a40
--- /dev/null
+++ b/tools/discover_hosts.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+# **discover_hosts.sh**
+
+# This is just a very simple script to run the
+# "nova-manage cell_v2 discover_hosts" command
+# which is needed to discover compute nodes and
+# register them with a parent cell in Nova.
+# This assumes that /etc/nova/nova.conf exists
+# and has the following entries filled in:
+#
+# [api_database]
+# connection = This is the URL to the nova_api database
+#
+# In other words this should be run on the primary
+# (API) node in a multi-node setup.
+
+if [[ -x $(which nova-manage) ]]; then
+    nova-manage cell_v2 discover_hosts --verbose
+fi