Merge "Fix distro detection for SUSE Linux Enterprise"
diff --git a/doc/source/guides/multinode-lab.rst b/doc/source/guides/multinode-lab.rst
index b4e2891..7978cd8 100644
--- a/doc/source/guides/multinode-lab.rst
+++ b/doc/source/guides/multinode-lab.rst
@@ -240,8 +240,8 @@
     sudo rm -rf /etc/libvirt/qemu/inst*
     sudo virsh list | grep inst | awk '{print $1}' | xargs -n1 virsh destroy
 
-Options pimp your stack
-=======================
+Going further
+=============
 
 Additional Users
 ----------------
@@ -302,10 +302,10 @@
 
 DevStack will automatically use an existing LVM volume group named
 ``stack-volumes`` to store cloud-created volumes. If ``stack-volumes``
-doesn't exist, DevStack will set up a 10Gb loop-mounted file to contain
-it. This obviously limits the number and size of volumes that can be
-created inside OpenStack. The size can be overridden by setting
-``VOLUME_BACKING_FILE_SIZE`` in ``local.conf``.
+doesn't exist, DevStack will set up a loop-mounted file to contain
+it.  If the default size is insufficient for the number and size of volumes
+required, it can be overridden by setting ``VOLUME_BACKING_FILE_SIZE`` in
+``local.conf`` (sizes given in ``truncate`` compatible format, e.g. ``24G``).
 
 ``stack-volumes`` can be pre-created on any physical volume supported by
 Linux's LVM. The name of the volume group can be changed by setting
diff --git a/doc/source/index.rst b/doc/source/index.rst
index 6c42a5b..9186f6d 100644
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -114,8 +114,8 @@
 You now have a working DevStack! Congrats!
 
 Your devstack will have installed ``keystone``, ``glance``, ``nova``,
-``cinder``, ``neutron``, and ``horizon``. Floating IPs will be
-available, guests have access to the external world.
+``placement``, ``cinder``, ``neutron``, and ``horizon``. Floating IPs
+will be available, guests have access to the external world.
 
 You can access horizon to experience the web interface to
 OpenStack, and manage vms, networks, volumes, and images from
diff --git a/doc/source/overview.rst b/doc/source/overview.rst
index 2479cd0..a609333 100644
--- a/doc/source/overview.rst
+++ b/doc/source/overview.rst
@@ -64,7 +64,8 @@
 
 The default services configured by DevStack are Identity (keystone),
 Object Storage (swift), Image Service (glance), Block Storage
-(cinder), Compute (nova), Networking (neutron), Dashboard (horizon)
+(cinder), Compute (nova), Placement (placement),
+Networking (neutron), Dashboard (horizon).
 
 Additional services not included directly in DevStack can be tied in to
 ``stack.sh`` using the :doc:`plugin mechanism <plugins>` to call
diff --git a/files/rpms-suse/n-cpu b/files/rpms-suse/n-cpu
index c11e9f0..9c724cb 100644
--- a/files/rpms-suse/n-cpu
+++ b/files/rpms-suse/n-cpu
@@ -1,8 +1,9 @@
+cdrkit-cdrtools-compat # dist:sle12
 cryptsetup
 dosfstools
 libosinfo
 lvm2
-mkisofs
+mkisofs # not:sle12
 open-iscsi
 sg3_utils
 # Stuff for diablo volumes
diff --git a/files/rpms-suse/nova b/files/rpms-suse/nova
index 4103a40..1d58121 100644
--- a/files/rpms-suse/nova
+++ b/files/rpms-suse/nova
@@ -1,3 +1,4 @@
+cdrkit-cdrtools-compat # dist:sle12
 conntrack-tools
 curl
 dnsmasq
@@ -11,7 +12,8 @@
 libvirt # NOPRIME
 libvirt-python # NOPRIME
 mariadb # NOPRIME
-mkisofs # required for config_drive
+# mkisofs is required for config_drive
+mkisofs # not:sle12
 parted
 polkit
 # qemu as fallback if kvm cannot be used
diff --git a/functions-common b/functions-common
index 28b12b2..bace9e0 100644
--- a/functions-common
+++ b/functions-common
@@ -1408,6 +1408,35 @@
         zypper --non-interactive install --auto-agree-with-licenses --no-recommends "$@"
 }
 
+# Run bindep and install packages it outputs
+#
+# Usage:
+#  install_bindep <path-to-bindep.txt> [profile,profile]
+#
+# Note unlike the bindep command itself, profile(s) specified should
+# be a single, comma-separated string, no spaces.
+function install_bindep {
+    local file=$1
+    local profiles=${2:-""}
+    local pkgs
+
+    if [[ ! -f $file ]]; then
+        die $LINENO "Can not find bindep file: $file"
+    fi
+
+    # converting here makes it much easier to work with passing
+    # arguments
+    profiles=${profiles/,/ /}
+
+    # Note bindep returns 1 when packages need to be installed, so we
+    # have to ignore it's return for "-e"
+    pkgs=$($DEST/bindep-venv/bin/bindep -b --file $file $profiles || true)
+
+    if [[ -n "${pkgs}" ]]; then
+        install_package ${pkgs}
+    fi
+}
+
 function write_user_unit_file {
     local service=$1
     local command="$2"
diff --git a/inc/python b/inc/python
index f1df101..ec7eb5b 100644
--- a/inc/python
+++ b/inc/python
@@ -428,7 +428,14 @@
 # another project.
 #
 # use this for non namespaced libraries
+#
+# setup_dev_lib [-bindep] <name>
 function setup_dev_lib {
+    local bindep
+    if [[ $1 == -bindep* ]]; then
+        bindep="${1}"
+        shift
+    fi
     local name=$1
     local dir=${GITDIR[$name]}
     if python3_enabled; then
@@ -438,10 +445,10 @@
         # of Python.
         echo "Installing $name again without Python 3 enabled"
         USE_PYTHON3=False
-        setup_develop $dir
+        setup_develop $bindep $dir
         USE_PYTHON3=True
     fi
-    setup_develop $dir
+    setup_develop $bindep $dir
 }
 
 # this should be used if you want to install globally, all libraries should
@@ -452,11 +459,17 @@
 # extras: comma-separated list of optional dependencies to install
 #         (e.g., ldap,memcache).
 #         See https://docs.openstack.org/pbr/latest/user/using.html#extra-requirements
+# bindep: Set "-bindep" as first argument to install bindep.txt packages
 # The command is like "pip install <project_dir>[<extras>]"
 function setup_install {
+    local bindep
+    if [[ $1 == -bindep* ]]; then
+        bindep="${1}"
+        shift
+    fi
     local project_dir=$1
     local extras=$2
-    _setup_package_with_constraints_edit $project_dir "" $extras
+    _setup_package_with_constraints_edit $bindep $project_dir "" $extras
 }
 
 # this should be used for projects which run services, like all services
@@ -468,9 +481,14 @@
 #         See https://docs.openstack.org/pbr/latest/user/using.html#extra-requirements
 # The command is like "pip install -e <project_dir>[<extras>]"
 function setup_develop {
+    local bindep
+    if [[ $1 == -bindep* ]]; then
+        bindep="${1}"
+        shift
+    fi
     local project_dir=$1
     local extras=$2
-    _setup_package_with_constraints_edit $project_dir -e $extras
+    _setup_package_with_constraints_edit $bindep $project_dir -e $extras
 }
 
 # ``pip install -e`` the package, which processes the dependencies
@@ -489,6 +507,11 @@
 #         See https://docs.openstack.org/pbr/latest/user/using.html#extra-requirements
 # The command is like "pip install <flags> <project_dir>[<extras>]"
 function _setup_package_with_constraints_edit {
+    local bindep
+    if [[ $1 == -bindep* ]]; then
+        bindep="${1}"
+        shift
+    fi
     local project_dir=$1
     local flags=$2
     local extras=$3
@@ -509,7 +532,7 @@
             "$flags file://$project_dir#egg=$name"
     fi
 
-    setup_package $project_dir "$flags" $extras
+    setup_package $bindep $project_dir "$flags" $extras
 
     # If this project is in LIBS_FROM_GIT, verify it was actually installed
     # correctly.  This helps catch errors caused by constraints mismatches.
@@ -521,17 +544,30 @@
 }
 
 # ``pip install -e`` the package, which processes the dependencies
-# using pip before running `setup.py develop`
+# using pip before running `setup.py develop`.  The command is like
+# "pip install <flags> <project_dir>[<extras>]"
 #
 # Uses globals ``STACK_USER``
-# setup_package project_dir [flags] [extras]
-# project_dir: directory of project repo (e.g., /opt/stack/keystone)
-# flags: pip CLI options/flags
-# extras: comma-separated list of optional dependencies to install
-#         (e.g., ldap,memcache).
-#         See https://docs.openstack.org/pbr/latest/user/using.html#extra-requirements
-# The command is like "pip install <flags> <project_dir>[<extras>]"
+#
+# Usage:
+#  setup_package [-bindep[=profile,profile]] <project_dir> <flags> [extras]
+#
+# -bindep     : Use bindep to install dependencies; select extra profiles
+#               as comma separated arguments after "="
+# project_dir : directory of project repo (e.g., /opt/stack/keystone)
+# flags       : pip CLI options/flags
+# extras      : comma-separated list of optional dependencies to install
+#               (e.g., ldap,memcache).
+#               See https://docs.openstack.org/pbr/latest/user/using.html#extra-requirements
 function setup_package {
+    local bindep=0
+    local bindep_flag=""
+    local bindep_profiles=""
+    if [[ $1 == -bindep* ]]; then
+        bindep=1
+        IFS="=" read bindep_flag bindep_profiles <<< ${1}
+        shift
+    fi
     local project_dir=$1
     local flags=$2
     local extras=$3
@@ -547,6 +583,11 @@
         extras="[$extras]"
     fi
 
+    # install any bindep packages
+    if [[ $bindep == 1 ]]; then
+        install_bindep $project_dir/bindep.txt $bindep_profiles
+    fi
+
     pip_install $flags "$project_dir$extras"
     # ensure that further actions can do things like setup.py sdist
     if [[ "$flags" == "-e" ]]; then
diff --git a/lib/cinder b/lib/cinder
index 76bf928..d69b21e 100644
--- a/lib/cinder
+++ b/lib/cinder
@@ -349,18 +349,12 @@
 
         # block-storage is the official service type
         get_or_create_service "cinder" "block-storage" "Cinder Volume Service"
-        get_or_create_service "cinder" "volume" "Cinder Volume Service"
         if [ "$CINDER_USE_MOD_WSGI" == "False" ]; then
             get_or_create_endpoint \
                 "block-storage" \
                 "$REGION_NAME" \
                 "$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT/v3/\$(project_id)s"
 
-            get_or_create_endpoint \
-                "volume" \
-                "$REGION_NAME" \
-                "$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT/v1/\$(project_id)s"
-
             get_or_create_service "cinderv2" "volumev2" "Cinder Volume Service V2"
             get_or_create_endpoint \
                 "volumev2" \
@@ -378,11 +372,6 @@
                 "$REGION_NAME" \
                 "$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST/volume/v3/\$(project_id)s"
 
-            get_or_create_endpoint \
-                "volume" \
-                "$REGION_NAME" \
-                "$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST/volume/v1/\$(project_id)s"
-
             get_or_create_service "cinderv2" "volumev2" "Cinder Volume Service V2"
             get_or_create_endpoint \
                 "volumev2" \
diff --git a/lib/databases/mysql b/lib/databases/mysql
index ac0c083..4d0f5f3 100644
--- a/lib/databases/mysql
+++ b/lib/databases/mysql
@@ -16,7 +16,13 @@
 register_database mysql
 
 MYSQL_SERVICE_NAME=mysql
-if is_suse || is_fedora && ! is_oraclelinux; then
+if is_fedora && ! is_oraclelinux; then
+    MYSQL_SERVICE_NAME=mariadb
+elif is_suse && systemctl list-unit-files | grep -q 'mariadb\.service'; then
+    # Older mariadb packages on SLES 12 provided mysql.service.  The
+    # newer ones on SLES 12 and 15 use mariadb.service; they also
+    # provide a mysql.service symlink for backwards-compatibility, but
+    # let's not rely on that.
     MYSQL_SERVICE_NAME=mariadb
 fi
 
diff --git a/lib/etcd3 b/lib/etcd3
index 0748ea0..4f3a7a4 100644
--- a/lib/etcd3
+++ b/lib/etcd3
@@ -29,7 +29,7 @@
 ETCD_BIN_DIR="$DEST/bin"
 # Option below will mount ETCD_DATA_DIR as ramdisk, which is useful to run
 # etcd-heavy services in the gate VM's, e.g. Kubernetes.
-ETCD_USE_RAMDISK=$(trueorfalse False ETCD_USE_RAMDISK)
+ETCD_USE_RAMDISK=$(trueorfalse True ETCD_USE_RAMDISK)
 ETCD_RAMDISK_MB=${ETCD_RAMDISK_MB:-512}
 
 if is_ubuntu ; then
diff --git a/lib/glance b/lib/glance
index 94f6a22..65487cb 100644
--- a/lib/glance
+++ b/lib/glance
@@ -236,8 +236,8 @@
         CINDER_SERVICE_HOST=${CINDER_SERVICE_HOST:-$SERVICE_HOST}
         CINDER_SERVICE_PORT=${CINDER_SERVICE_PORT:-8776}
 
-        iniset $GLANCE_API_CONF DEFAULT cinder_endpoint_template "https://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT/v1/%(project_id)s"
-        iniset $GLANCE_CACHE_CONF DEFAULT cinder_endpoint_template "https://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT/v1/%(project_id)s"
+        iniset $GLANCE_API_CONF DEFAULT cinder_endpoint_template "https://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT/v3/%(project_id)s"
+        iniset $GLANCE_CACHE_CONF DEFAULT cinder_endpoint_template "https://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT/v3/%(project_id)s"
     fi
 
     if [[ "$WSGI_MODE" == "uwsgi" ]]; then
@@ -345,7 +345,7 @@
     if [[ "$WSGI_MODE" == "uwsgi" ]]; then
         run_process g-api "$GLANCE_BIN_DIR/uwsgi --procname-prefix glance-api --ini $GLANCE_UWSGI_CONF"
     else
-        run_process g-api "$GLANCE_BIN_DIR/glance-api --config-file=$GLANCE_CONF_DIR/glance-api.conf"
+        run_process g-api "$GLANCE_BIN_DIR/glance-api --config-dir=$GLANCE_CONF_DIR"
     fi
 
     echo "Waiting for g-api ($GLANCE_SERVICE_HOST) to start..."
diff --git a/lib/nova b/lib/nova
index 95fa7d3..6c9b944 100644
--- a/lib/nova
+++ b/lib/nova
@@ -327,10 +327,8 @@
                 sudo chown -R $STACK_USER $NOVA_INSTANCES_PATH
             fi
         fi
-        if is_suse; then
-            # iscsid is not started by default
-            start_service iscsid
-        fi
+        # ensure that iscsid is started, even when disabled by default
+        start_service iscsid
     fi
 
     # Rebuild the config file from scratch
diff --git a/roles/fetch-devstack-log-dir/tasks/main.yaml b/roles/fetch-devstack-log-dir/tasks/main.yaml
index 5a198b2..276c4e0 100644
--- a/roles/fetch-devstack-log-dir/tasks/main.yaml
+++ b/roles/fetch-devstack-log-dir/tasks/main.yaml
@@ -1,5 +1,10 @@
+# as the user in the guest may not exist on the executor
+# we do not preserve the group or owner of the copied logs.
+
 - name: Collect devstack logs
   synchronize:
     dest: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}"
     mode: pull
     src: "{{ devstack_base_dir }}/logs"
+    group: no
+    owner: no
diff --git a/stack.sh b/stack.sh
index 54a4f98..022d5b9 100755
--- a/stack.sh
+++ b/stack.sh
@@ -801,6 +801,11 @@
 # Install required infra support libraries
 install_infra
 
+# Install bindep
+$VIRTUALENV_CMD $DEST/bindep-venv
+# TODO(ianw) : optionally install from zuul checkout?
+$DEST/bindep-venv/bin/pip install bindep
+
 # Extras Pre-install
 # ------------------
 # Phase: pre-install
diff --git a/stackrc b/stackrc
index d6f8606..53868f3 100644
--- a/stackrc
+++ b/stackrc
@@ -143,6 +143,13 @@
 _DEFAULT_PYTHON2_VERSION="$(_get_python_version python2)"
 export PYTHON2_VERSION=${PYTHON2_VERSION:-${_DEFAULT_PYTHON2_VERSION:-2.7}}
 
+# Create a virtualenv with this
+if [[ ${USE_PYTHON3} == True ]]; then
+    export VIRTUALENV_CMD="python3 -m venv"
+else
+    export VIRTUALENV_CMD="virtualenv "
+fi
+
 # allow local overrides of env variables, including repo config
 if [[ -f $RC_DIR/localrc ]]; then
     # Old-style user-supplied config