Merge "Fix readme for multi-node setup"
diff --git a/.gitignore b/.gitignore
index c6900c8..2778a65 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,9 +12,11 @@
 doc/build
 files/*.gz
 files/*.qcow2
+files/*.img
 files/images
 files/pip-*
 files/get-pip.py*
+files/ir-deploy*
 local.conf
 local.sh
 localrc
diff --git a/README.md b/README.md
index faf2d93..04f5fd9 100644
--- a/README.md
+++ b/README.md
@@ -264,10 +264,10 @@
 
 # Heat
 
-Heat is enabled by default (see `stackrc` file). To disable it explicitly
+Heat is disabled by default (see `stackrc` file). To enable it explicitly
 you'll need the following settings in your `localrc` section:
 
-    disable_service heat h-api h-api-cfn h-api-cw h-eng
+    enable_service heat h-api h-api-cfn h-api-cw h-eng
 
 Heat can also run in standalone mode, and be configured to orchestrate
 on an external OpenStack cloud. To launch only Heat in standalone mode
diff --git a/clean.sh b/clean.sh
index 035489c..7db519b 100755
--- a/clean.sh
+++ b/clean.sh
@@ -76,6 +76,7 @@
 # ==========
 
 # Phase: clean
+load_plugin_settings
 run_phase clean
 
 if [[ -d $TOP_DIR/extras.d ]]; then
@@ -114,9 +115,16 @@
 cleanup_rpc_backend
 cleanup_database
 
-# Clean out data, logs and status
-LOGDIR=$(dirname "$LOGFILE")
-sudo rm -rf $DATA_DIR $LOGDIR $DEST/status
+# Clean out data and status
+sudo rm -rf $DATA_DIR $DEST/status
+
+# Clean out the log file and log directories
+if [[ -n "$LOGFILE" ]] && [[ -f "$LOGFILE" ]]; then
+    sudo rm -f $LOGFILE
+fi
+if [[ -n "$LOGDIR" ]] && [[ -d "$LOGDIR" ]]; then
+    sudo rm -rf $LOGDIR
+fi
 if [[ -n "$SCREEN_LOGDIR" ]] && [[ -d "$SCREEN_LOGDIR" ]]; then
     sudo rm -rf $SCREEN_LOGDIR
 fi
diff --git a/doc/source/eucarc.rst b/doc/source/eucarc.rst
index 1284b88..c2ecbc6 100644
--- a/doc/source/eucarc.rst
+++ b/doc/source/eucarc.rst
@@ -13,7 +13,7 @@
 
     ::
 
-        EC2_URL=$(keystone catalog --service ec2 | awk '/ publicURL / { print $4 }')
+        EC2_URL=$(openstack catalog show ec2 | awk '/ publicURL: / { print $4 }')
 
 S3\_URL
     Set the S3 endpoint for euca2ools. The endpoint is extracted from
@@ -21,14 +21,14 @@
 
     ::
 
-        export S3_URL=$(keystone catalog --service s3 | awk '/ publicURL / { print $4 }')
+        export S3_URL=$(openstack catalog show s3 | awk '/ publicURL: / { print $4 }')
 
 EC2\_ACCESS\_KEY, EC2\_SECRET\_KEY
     Create EC2 credentials for the current tenant:user in Keystone.
 
     ::
 
-        CREDS=$(keystone ec2-credentials-create)
+        CREDS=$(openstack ec2 credentials create)
         export EC2_ACCESS_KEY=$(echo "$CREDS" | awk '/ access / { print $4 }')
         export EC2_SECRET_KEY=$(echo "$CREDS" | awk '/ secret / { print $4 }')
 
diff --git a/doc/source/guides/multinode-lab.rst b/doc/source/guides/multinode-lab.rst
index ff81c93..d963243 100644
--- a/doc/source/guides/multinode-lab.rst
+++ b/doc/source/guides/multinode-lab.rst
@@ -229,10 +229,10 @@
 ----------------
 
 DevStack creates two OpenStack users (``admin`` and ``demo``) and two
-tenants (also ``admin`` and ``demo``). ``admin`` is exactly what it
+projects (also ``admin`` and ``demo``). ``admin`` is exactly what it
 sounds like, a privileged administrative account that is a member of
-both the ``admin`` and ``demo`` tenants. ``demo`` is a normal user
-account that is only a member of the ``demo`` tenant. Creating
+both the ``admin`` and ``demo`` projects. ``demo`` is a normal user
+account that is only a member of the ``demo`` project. Creating
 additional OpenStack users can be done through the dashboard, sometimes
 it is easier to do them in bulk from a script, especially since they get
 blown away every time ``stack.sh`` runs. The following steps are ripe
@@ -243,21 +243,21 @@
     # Get admin creds
     . openrc admin admin
 
-    # List existing tenants
-    keystone tenant-list
+    # List existing projects
+    openstack project list
 
     # List existing users
-    keystone user-list
+    openstack user list
 
-    # Add a user and tenant
+    # Add a user and project
     NAME=bob
     PASSWORD=BigSecrete
-    TENANT=$NAME
-    keystone tenant-create --name=$NAME
-    keystone user-create --name=$NAME --pass=$PASSWORD
-    keystone user-role-add --user-id=<bob-user-id> --tenant-id=<bob-tenant-id> --role-id=<member-role-id>
-    # member-role-id comes from the existing member role created by stack.sh
-    # keystone role-list
+    PROJECT=$NAME
+    openstack project create $PROJECT
+    openstack user create $NAME --password=$PASSWORD --project $PROJECT
+    openstack role add Member --user $NAME --project $PROJECT
+    # The Member role is created by stack.sh
+    # openstack role list
 
 Swift
 -----
diff --git a/eucarc b/eucarc
index 343f4cc..1e672bd 100644
--- a/eucarc
+++ b/eucarc
@@ -19,7 +19,7 @@
 source $RC_DIR/openrc
 
 # Set the ec2 url so euca2ools works
-export EC2_URL=$(keystone catalog --service ec2 | awk '/ publicURL / { print $4 }')
+export EC2_URL=$(openstack catalog show ec2 | awk '/ publicURL: / { print $4 }')
 
 # Create EC2 credentials for the current user
 CREDS=$(openstack ec2 credentials create)
@@ -29,7 +29,7 @@
 # Euca2ools Certificate stuff for uploading bundles
 # See exercises/bundle.sh to see how to get certs using nova cli
 NOVA_KEY_DIR=${NOVA_KEY_DIR:-$RC_DIR}
-export S3_URL=$(keystone catalog --service s3 | awk '/ publicURL / { print $4 }')
+export S3_URL=$(openstack catalog show s3 | awk '/ publicURL: / { print $4 }')
 export EC2_USER_ID=42 # nova does not use user id, but bundling requires it
 export EC2_PRIVATE_KEY=${NOVA_KEY_DIR}/pk.pem
 export EC2_CERT=${NOVA_KEY_DIR}/cert.pem
diff --git a/exercises/client-args.sh b/exercises/client-args.sh
index 2f85d98..c33ef44 100755
--- a/exercises/client-args.sh
+++ b/exercises/client-args.sh
@@ -69,7 +69,7 @@
         STATUS_KEYSTONE="Skipped"
     else
         echo -e "\nTest Keystone"
-        if keystone $TENANT_ARG $ARGS catalog --service identity; then
+        if openstack $TENANT_ARG $ARGS catalog show identity; then
             STATUS_KEYSTONE="Succeeded"
         else
             STATUS_KEYSTONE="Failed"
diff --git a/functions b/functions
index 4dc20e7..2078db1 100644
--- a/functions
+++ b/functions
@@ -287,6 +287,10 @@
         img_property="--property hw_cdrom_bus=scsi"
     fi
 
+    if is_arch "aarch64"; then
+        img_property="--property hw_machine_type=virt --property hw_cdrom_bus=virtio --property os_command_line='console=ttyAMA0'"
+    fi
+
     if [ "$container_format" = "bare" ]; then
         if [ "$unpack" = "zcat" ]; then
             openstack --os-token $token --os-url $GLANCE_SERVICE_PROTOCOL://$GLANCE_HOSTPORT image create "$image_name" $img_property --public --container-format=$container_format --disk-format $disk_format < <(zcat --force "${image}")
diff --git a/functions-common b/functions-common
index f442211..f8543c1 100644
--- a/functions-common
+++ b/functions-common
@@ -1256,8 +1256,13 @@
 
     # sleep to allow bash to be ready to be send the command - we are
     # creating a new window in screen and then sends characters, so if
-    # bash isn't running by the time we send the command, nothing happens
-    sleep 3
+    # bash isn't running by the time we send the command, nothing
+    # happens.  This sleep was added originally to handle gate runs
+    # where we needed this to be at least 3 seconds to pass
+    # consistently on slow clouds. Now this is configurable so that we
+    # can determine a reasonable value for the local case which should
+    # be much smaller.
+    sleep ${SCREEN_SLEEP:-3}
 
     NL=`echo -ne '\015'`
     # This fun command does the following:
diff --git a/inc/python b/inc/python
index 39684b6..3d329b5 100644
--- a/inc/python
+++ b/inc/python
@@ -52,6 +52,18 @@
     fi
 }
 
+# Wrapper for ``pip install`` that only installs versions of libraries
+# from the global-requirements specification.
+#
+# Uses globals ``REQUIREMENTS_DIR``
+#
+# pip_install_gr packagename
+function pip_install_gr {
+    local name=$1
+    local clean_name=$(get_from_global_requirements $name)
+    pip_install $clean_name
+}
+
 # Wrapper for ``pip install`` to set cache and proxy environment variables
 # Uses globals ``OFFLINE``, ``PIP_VIRTUAL_ENV``,
 # ``PIP_UPGRADE``, ``TRACK_DEPENDS``, ``*_proxy``
@@ -125,7 +137,7 @@
 # get_from_global_requirements <package>
 function get_from_global_requirements {
     local package=$1
-    local required_pkg=$(grep -h ${package} $REQUIREMENTS_DIR/global-requirements.txt | cut -d\# -f1)
+    local required_pkg=$(grep -i -h ^${package} $REQUIREMENTS_DIR/global-requirements.txt | cut -d\# -f1)
     if [[ $required_pkg == ""  ]]; then
         die $LINENO "Can't find package $package in requirements"
     fi
diff --git a/lib/ceilometer b/lib/ceilometer
index 8135309..dba92ba 100644
--- a/lib/ceilometer
+++ b/lib/ceilometer
@@ -315,6 +315,9 @@
         git_clone_by_name "ceilometermiddleware"
         setup_dev_lib "ceilometermiddleware"
     else
+        # BUG: this should be a pip_install_gr except it was never
+        # included in global-requirements. Needs to be fixed by
+        # https://bugs.launchpad.net/ceilometer/+bug/1441655
         pip_install ceilometermiddleware
     fi
 }
diff --git a/lib/databases/mysql b/lib/databases/mysql
index 310817b..1b9a081 100644
--- a/lib/databases/mysql
+++ b/lib/databases/mysql
@@ -155,7 +155,7 @@
 
 function install_database_python_mysql {
     # Install Python client module
-    pip_install MySQL-python
+    pip_install_gr MySQL-python
     ADDITIONAL_VENV_PACKAGES+=",MySQL-python"
 }
 
diff --git a/lib/databases/postgresql b/lib/databases/postgresql
index a6bcf8c..e087a1e 100644
--- a/lib/databases/postgresql
+++ b/lib/databases/postgresql
@@ -104,7 +104,7 @@
 
 function install_database_python_postgresql {
     # Install Python client module
-    pip_install psycopg2
+    pip_install_gr psycopg2
     ADDITIONAL_VENV_PACKAGES+=",psycopg2"
 }
 
diff --git a/lib/horizon b/lib/horizon
index 63a9d0f..f953f5c 100644
--- a/lib/horizon
+++ b/lib/horizon
@@ -183,7 +183,7 @@
 # NOTE: It can be moved to common functions, but it is only used by compilation
 # of django_openstack_auth catalogs at the moment.
 function _prepare_message_catalog_compilation {
-    pip_install $(get_from_global_requirements Babel)
+    pip_install_gr Babel
 }
 
 
diff --git a/lib/ironic b/lib/ironic
index c8481ab..4ac0100 100644
--- a/lib/ironic
+++ b/lib/ironic
@@ -206,7 +206,7 @@
         sudo install -D -m 0644 -o $STACK_USER {${GITDIR["python-ironicclient"]}/tools/,/etc/bash_completion.d/}ironic.bash_completion
     else
         # nothing actually "requires" ironicclient, so force instally from pypi
-        pip_install python-ironicclient
+        pip_install_gr python-ironicclient
     fi
 }
 
@@ -729,7 +729,7 @@
 
     # install diskimage-builder
     if [[ $(type -P ramdisk-image-create) == "" ]]; then
-        pip_install diskimage_builder
+        pip_install_gr "diskimage-builder"
     fi
 
     if [ -z "$IRONIC_DEPLOY_KERNEL" -o -z "$IRONIC_DEPLOY_RAMDISK" ]; then
@@ -765,7 +765,7 @@
         fi
     fi
 
-    local token=$(keystone token-get | grep ' id ' | get_field 2)
+    local token=$(openstack token issue -c id -f value)
     die_if_not_set $LINENO token "Keystone fail to get token"
 
     # load them into glance
diff --git a/lib/keystone b/lib/keystone
index 7b41812..31659f4 100644
--- a/lib/keystone
+++ b/lib/keystone
@@ -524,7 +524,7 @@
         setup_dev_lib "keystonemiddleware"
     else
         # When not installing from repo, keystonemiddleware is still needed...
-        pip_install keystonemiddleware
+        pip_install_gr keystonemiddleware
     fi
 }
 
diff --git a/lib/ldap b/lib/ldap
index d69d3f8..d2dbc3b 100644
--- a/lib/ldap
+++ b/lib/ldap
@@ -142,7 +142,7 @@
         sudo ldapadd -Y EXTERNAL -H ldapi:/// -f /etc/openldap/schema/inetorgperson.ldif
     fi
 
-    pip_install ldappool
+    pip_install_gr ldappool
 
     rm -rf $tmp_ldap_dir
 }
diff --git a/lib/neutron_plugins/ovs_base b/lib/neutron_plugins/ovs_base
index 2997c6c..51999c6 100644
--- a/lib/neutron_plugins/ovs_base
+++ b/lib/neutron_plugins/ovs_base
@@ -93,11 +93,8 @@
         sudo ip link set $Q_PUBLIC_VETH_EX up
         sudo ip addr flush dev $Q_PUBLIC_VETH_EX
     else
-        # --no-wait causes a race condition if $PUBLIC_BRIDGE is not up when ip addr flush is called
         sudo ovs-vsctl -- --may-exist add-br $PUBLIC_BRIDGE
         sudo ovs-vsctl br-set-external-id $PUBLIC_BRIDGE bridge-id $PUBLIC_BRIDGE
-        # ensure no IP is configured on the public bridge
-        sudo ip addr flush dev $PUBLIC_BRIDGE
     fi
 }
 
diff --git a/lib/nova_plugins/functions-libvirt b/lib/nova_plugins/functions-libvirt
index d4a0768..04da5e2 100755
--- a/lib/nova_plugins/functions-libvirt
+++ b/lib/nova_plugins/functions-libvirt
@@ -31,12 +31,12 @@
             install_package python-guestfs
         fi
         install_package libvirt-bin libvirt-dev
-        pip_install libvirt-python
-        #pip_install <there-si-no-guestfs-in-pypi>
+        pip_install_gr libvirt-python
+        #pip_install_gr <there-si-no-guestfs-in-pypi>
     elif is_fedora || is_suse; then
         install_package kvm
         install_package libvirt libvirt-devel
-        pip_install libvirt-python
+        pip_install_gr libvirt-python
         install_package python-libguestfs
     fi
 
diff --git a/lib/nova_plugins/hypervisor-libvirt b/lib/nova_plugins/hypervisor-libvirt
index 4d1eb6c..a6a87f9 100644
--- a/lib/nova_plugins/hypervisor-libvirt
+++ b/lib/nova_plugins/hypervisor-libvirt
@@ -54,6 +54,12 @@
         iniset $NOVA_CONF DEFAULT vnc_enabled "false"
     fi
 
+    # arm64-specific configuration
+    if is_arch "aarch64"; then
+        # arm64 architecture currently does not support graphical consoles.
+        iniset $NOVA_CONF DEFAULT vnc_enabled "false"
+    fi
+
     ENABLE_FILE_INJECTION=$(trueorfalse False ENABLE_FILE_INJECTION)
     if [[ "$ENABLE_FILE_INJECTION" = "True" ]] ; then
         # When libguestfs is available for file injection, enable using
diff --git a/lib/nova_plugins/hypervisor-xenserver b/lib/nova_plugins/hypervisor-xenserver
index 4d0ec89..efce383 100644
--- a/lib/nova_plugins/hypervisor-xenserver
+++ b/lib/nova_plugins/hypervisor-xenserver
@@ -94,7 +94,7 @@
 
 # install_nova_hypervisor() - Install external components
 function install_nova_hypervisor {
-    pip_install xenapi
+    pip_install_gr xenapi
 }
 
 # start_nova_hypervisor - Start any required external services
diff --git a/lib/rpc_backend b/lib/rpc_backend
index cc083de..288987c 100644
--- a/lib/rpc_backend
+++ b/lib/rpc_backend
@@ -141,7 +141,7 @@
         # TODO(kgiusti) can remove once python qpid bindings are
         # available on all supported platforms _and_ pyngus is added
         # to the requirements.txt file in oslo.messaging
-        pip_install pyngus
+        pip_install_gr pyngus
     fi
 
     if is_service_enabled rabbit; then
@@ -273,6 +273,12 @@
         iniset $file oslo_messaging_rabbit rabbit_hosts $RABBIT_HOST
         iniset $file oslo_messaging_rabbit rabbit_password $RABBIT_PASSWORD
         iniset $file oslo_messaging_rabbit rabbit_userid $RABBIT_USERID
+        if [ -n "$RABBIT_HEARTBEAT_TIMEOUT_THRESHOLD" ]; then
+            iniset $file oslo_messaging_rabbit heartbeat_timeout_threshold $RABBIT_HEARTBEAT_TIMEOUT_THRESHOLD
+        fi
+        if [ -n "$RABBIT_HEARTBEAT_RATE" ]; then
+            iniset $file oslo_messaging_rabbit heartbeat_rate $RABBIT_HEARTBEAT_RATE
+        fi
     fi
 }
 
diff --git a/lib/tempest b/lib/tempest
index e8e9e0b..cd8fbd7 100644
--- a/lib/tempest
+++ b/lib/tempest
@@ -103,7 +103,7 @@
         setup_develop $TEMPEST_DIR
     else
         # install testr since its used to process tempest logs
-        pip_install $(get_from_global_requirements testrepository)
+        pip_install_gr testrepository
     fi
 
     local image_lines
diff --git a/lib/zaqar b/lib/zaqar
index 34f1915..8d51910 100644
--- a/lib/zaqar
+++ b/lib/zaqar
@@ -140,10 +140,10 @@
 function configure_redis {
     if is_ubuntu; then
         install_package redis-server
-        pip_install redis
+        pip_install_gr redis
     elif is_fedora; then
         install_package redis
-        pip_install redis
+        pip_install_gr redis
     else
         exit_distro_not_supported "redis installation"
     fi
diff --git a/pkg/elasticsearch.sh b/pkg/elasticsearch.sh
index f53c7f2..29dc22f 100755
--- a/pkg/elasticsearch.sh
+++ b/pkg/elasticsearch.sh
@@ -77,7 +77,7 @@
 }
 
 function install_elasticsearch {
-    pip_install elasticsearch
+    pip_install_gr elasticsearch
     if is_package_installed elasticsearch; then
         echo "Note: elasticsearch was already installed."
         return
diff --git a/stack.sh b/stack.sh
index adcaa21..3f4ae6c 100755
--- a/stack.sh
+++ b/stack.sh
@@ -714,6 +714,9 @@
 # Extras Pre-install
 # ------------------
 
+# Install required infra support libraries
+install_infra
+
 # Phase: pre-install
 run_phase stack pre-install
 
@@ -733,9 +736,6 @@
 
 echo_summary "Installing OpenStack project source"
 
-# Install required infra support libraries
-install_infra
-
 # Install Oslo libraries
 install_oslo
 
@@ -847,7 +847,7 @@
     git_clone_by_name "python-openstackclient"
     setup_dev_lib "python-openstackclient"
 else
-    pip_install 'python-openstackclient>=1.0.2'
+    pip_install_gr python-openstackclient
 fi
 
 if [[ $TRACK_DEPENDS = True ]]; then
@@ -1173,7 +1173,7 @@
 # See https://help.ubuntu.com/community/CloudInit for more on ``cloud-init``
 
 if is_service_enabled g-reg; then
-    TOKEN=$(keystone token-get | grep ' id ' | get_field 2)
+    TOKEN=$(openstack token issue -c id -f value)
     die_if_not_set $LINENO TOKEN "Keystone fail to get token"
 
     echo_summary "Uploading images"
@@ -1214,6 +1214,10 @@
 elif is_service_enabled $DATABASE_BACKENDS && is_service_enabled n-net; then
     NM_CONF=${NOVA_CONF}
     if is_service_enabled n-cell; then
+        # Both cells should have the same network uuid for server create
+        if [[ ! "$NETWORK_CREATE_ARGS" =~ "--uuid" ]]; then
+            NETWORK_CREATE_ARGS="$NETWORK_CREATE_ARGS --uuid $(uuidgen)"
+        fi
         # Create a small network in the API cell
         $NOVA_BIN_DIR/nova-manage --config-file $NM_CONF network create "$PRIVATE_NETWORK_NAME" $FIXED_RANGE 1 $FIXED_NETWORK_SIZE $NETWORK_CREATE_ARGS
         # Everything else should go in the child cell
diff --git a/tools/upload_image.sh b/tools/upload_image.sh
index 5d23f31..19c6b71 100755
--- a/tools/upload_image.sh
+++ b/tools/upload_image.sh
@@ -32,7 +32,7 @@
 fi
 
 # Get a token to authenticate to glance
-TOKEN=$(keystone token-get | grep ' id ' | get_field 2)
+TOKEN=$(openstack token issue -c id -f value)
 die_if_not_set $LINENO TOKEN "Keystone fail to get token"
 
 # Glance connection info.  Note the port must be specified.