Merge "Removed starting of nova-ec2-api service"
diff --git a/doc/source/plugin-registry.rst b/doc/source/plugin-registry.rst
index 7682def..8396d2f 100644
--- a/doc/source/plugin-registry.rst
+++ b/doc/source/plugin-registry.rst
@@ -28,6 +28,8 @@
 +------------------+---------------------------------------------+--------------------+
 |congress          |git://git.openstack.org/openstack/congress   | governance         |
 +------------------+---------------------------------------------+--------------------+
+|cue               |git://git.openstack.org/openstack/cue        | message-broker     |
++------------------+---------------------------------------------+--------------------+
 |gnocchi           |git://git.openstack.org/openstack/gnocchi    | metric             |
 +------------------+---------------------------------------------+--------------------+
 |ironic            |git://git.openstack.org/openstack/ironic     | baremetal          |
diff --git a/functions-common b/functions-common
index c3eef46..4a9db34 100644
--- a/functions-common
+++ b/functions-common
@@ -866,6 +866,32 @@
     echo $user_role_id
 }
 
+# Gets or adds user role to domain
+# Usage: get_or_add_user_domain_role <role> <user> <domain>
+function get_or_add_user_domain_role {
+    local user_role_id
+    # Gets user role id
+    user_role_id=$(openstack role list \
+        --user $2 \
+        --column "ID" \
+        --domain $3 \
+        --column "Name" \
+        | grep " $1 " | get_field 1)
+    if [[ -z "$user_role_id" ]]; then
+        # Adds role to user and get it
+        openstack role add $1 \
+            --user $2 \
+            --domain $3
+        user_role_id=$(openstack role list \
+            --user $2 \
+            --column "ID" \
+            --domain $3 \
+            --column "Name" \
+            | grep " $1 " | get_field 1)
+    fi
+    echo $user_role_id
+}
+
 # Gets or adds group role to project
 # Usage: get_or_add_group_project_role <role> <group> <project>
 function get_or_add_group_project_role {
@@ -1176,7 +1202,7 @@
     local package_dir=""
     for plugin in ${DEVSTACK_PLUGINS//,/ }; do
         local package_dir="$(_get_package_dir ${GITDIR[$plugin]}/devstack/files)"
-        files_to_parse+="$package_dir/$plugin"
+        files_to_parse+=" $package_dir/$plugin"
     done
     echo "$(_parse_package_files $files_to_parse)"
     $xtrace
diff --git a/lib/cinder b/lib/cinder
index 569f3ab..5bd940b 100644
--- a/lib/cinder
+++ b/lib/cinder
@@ -305,6 +305,8 @@
     if is_service_enabled tls-proxy; then
         # Set the service port for a proxy to take the original
         iniset $CINDER_CONF DEFAULT osapi_volume_listen_port $CINDER_SERVICE_PORT_INT
+
+        iniset $CINDER_CONF DEFAULT public_endpoint $CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT
     fi
 
     if [ "$SYSLOG" != "False" ]; then
diff --git a/lib/glance b/lib/glance
index 4f95975..19e7937 100644
--- a/lib/glance
+++ b/lib/glance
@@ -169,6 +169,9 @@
         iniset $GLANCE_API_CONF DEFAULT bind_port $GLANCE_SERVICE_PORT_INT
         iniset $GLANCE_API_CONF DEFAULT public_endpoint $GLANCE_SERVICE_PROTOCOL://$GLANCE_HOSTPORT
         iniset $GLANCE_REGISTRY_CONF DEFAULT bind_port $GLANCE_REGISTRY_PORT_INT
+
+        iniset $GLANCE_API_CONF keystone_authtoken identity_uri $KEYSTONE_AUTH_URI
+        iniset $GLANCE_REGISTRY_CONF keystone_authtoken identity_uri $KEYSTONE_AUTH_URI
     fi
 
     # Register SSL certificates if provided
diff --git a/lib/heat b/lib/heat
index 3666776..6301230 100644
--- a/lib/heat
+++ b/lib/heat
@@ -415,28 +415,13 @@
     fi
 
     if [[ "$HEAT_STACK_DOMAIN" == "True" ]]; then
-        # Note we have to pass token/endpoint here because the current endpoint and
-        # version negotiation in OSC means just --os-identity-api-version=3 won't work
-        D_ID=$(openstack --os-token $OS_TOKEN --os-url=$KEYSTONE_SERVICE_URI_V3 \
-            --os-identity-api-version=3 domain list | grep ' heat ' | get_field 1)
-
-        if [[ -z "$D_ID" ]]; then
-            D_ID=$(openstack --os-token $OS_TOKEN --os-url=$KEYSTONE_SERVICE_URI_V3 \
-                --os-identity-api-version=3 domain create heat \
-                --description "Owns users and projects created by heat" \
-                | grep ' id ' | get_field 2)
-            iniset $HEAT_CONF DEFAULT stack_user_domain_id ${D_ID}
-
-            openstack --os-token $OS_TOKEN --os-url=$KEYSTONE_SERVICE_URI_V3 \
-                --os-identity-api-version=3 user create --password $SERVICE_PASSWORD \
-                --domain $D_ID heat_domain_admin \
-                --description "Manages users and projects created by heat"
-            openstack --os-token $OS_TOKEN --os-url=$KEYSTONE_SERVICE_URI_V3 \
-                --os-identity-api-version=3 role add \
-                --user heat_domain_admin --domain ${D_ID} admin
-            iniset $HEAT_CONF DEFAULT stack_domain_admin heat_domain_admin
-            iniset $HEAT_CONF DEFAULT stack_domain_admin_password $SERVICE_PASSWORD
-        fi
+        # domain -> heat and user -> heat_domain_admin
+        domain_id=$(get_or_create_domain heat 'Owns users and projects created by heat')
+        iniset $HEAT_CONF DEFAULT stack_user_domain_id ${domain_id}
+        get_or_create_user heat_domain_admin $SERVICE_PASSWORD heat
+        get_or_add_user_domain_role admin heat_domain_admin heat
+        iniset $HEAT_CONF DEFAULT stack_domain_admin heat_domain_admin
+        iniset $HEAT_CONF DEFAULT stack_domain_admin_password $SERVICE_PASSWORD
     fi
 }
 
diff --git a/lib/keystone b/lib/keystone
index 336ad12..b4b7df9 100644
--- a/lib/keystone
+++ b/lib/keystone
@@ -225,6 +225,9 @@
         # Set the service ports for a proxy to take the originals
         iniset $KEYSTONE_CONF eventlet_server public_port $KEYSTONE_SERVICE_PORT_INT
         iniset $KEYSTONE_CONF eventlet_server admin_port $KEYSTONE_AUTH_PORT_INT
+
+        iniset $KEYSTONE_CONF DEFAULT public_endpoint $KEYSTONE_SERVICE_URI
+        iniset $KEYSTONE_CONF DEFAULT admin_endpoint $KEYSTONE_AUTH_URI
     fi
 
     iniset $KEYSTONE_CONF DEFAULT admin_token "$SERVICE_TOKEN"
diff --git a/lib/neutron-legacy b/lib/neutron-legacy
index 6af44e6..cc5b75e 100644
--- a/lib/neutron-legacy
+++ b/lib/neutron-legacy
@@ -171,6 +171,9 @@
 
 ## Provider Network Information
 PROVIDER_SUBNET_NAME=${PROVIDER_SUBNET_NAME:-"provider_net"}
+IPV6_PROVIDER_SUBNET_NAME=${IPV6_PROVIDER_SUBNET_NAME:-"provider_net_v6"}
+IPV6_PROVIDER_FIXED_RANGE=${IPV6_PROVIDER_FIXED_RANGE:-}
+IPV6_PROVIDER_NETWORK_GATEWAY=${IPV6_PROVIDER_NETWORK_GATEWAY:-}
 
 # Define the public bridge that will transmit traffic from VMs to the
 # physical network - used by both the OVS and Linux Bridge drivers.
@@ -551,9 +554,9 @@
             die_if_not_set $LINENO SUBNET_ID "Failure creating SUBNET_ID for $PROVIDER_SUBNET_NAME $TENANT_ID"
         fi
 
-        if [[ "$IP_VERSION" =~ .*6 ]]; then
-            SUBNET_V6_ID=$(neutron subnet-create --tenant_id $TENANT_ID --ip_version 6 --ipv6-address-mode $IPV6_ADDRESS_MODE --gateway $V6_NETWORK_GATEWAY --name $PROVIDER_SUBNET_NAME_V6 --subnetpool_id None $NET_ID $FIXED_RANGE_V6 | grep 'id' | get_field 2)
-            die_if_not_set $LINENO SUBNET_V6_ID "Failure creating SUBNET_V6_ID for $PROVIDER_SUBNET_NAME_V6 $TENANT_ID"
+        if [[ "$IP_VERSION" =~ .*6 ]] && [[ -n "$IPV6_PROVIDER_FIXED_RANGE" ]] && [[ -n "$IPV6_PROVIDER_NETWORK_GATEWAY" ]]; then
+            SUBNET_V6_ID=$(neutron subnet-create --tenant_id $TENANT_ID --ip_version 6 --ipv6-address-mode $IPV6_ADDRESS_MODE --gateway $IPV6_PROVIDER_NETWORK_GATEWAY --name $IPV6_PROVIDER_SUBNET_NAME --subnetpool None $NET_ID $IPV6_PROVIDER_FIXED_RANGE | grep 'id' | get_field 2)
+            die_if_not_set $LINENO SUBNET_V6_ID "Failure creating SUBNET_V6_ID for $IPV6_PROVIDER_SUBNET_NAME $TENANT_ID"
         fi
 
         if [[ $Q_AGENT == "openvswitch" ]]; then
@@ -780,6 +783,10 @@
     if is_service_enabled q-metering; then
         neutron_metering_stop
     fi
+
+    if [[ "$Q_USE_ROOTWRAP_DAEMON" == "True" ]]; then
+        sudo pkill -9 -f $NEUTRON_ROOTWRAP-daemon || :
+    fi
 }
 
 # stop_neutron() - Stop running processes (non-screen)
diff --git a/lib/tempest b/lib/tempest
index ecc4865..c510984 100644
--- a/lib/tempest
+++ b/lib/tempest
@@ -281,18 +281,12 @@
     # Identity
     iniset $TEMPEST_CONFIG identity uri "$KEYSTONE_SERVICE_PROTOCOL://$KEYSTONE_SERVICE_HOST:5000/v2.0/"
     iniset $TEMPEST_CONFIG identity uri_v3 "$KEYSTONE_SERVICE_URI_V3"
-    iniset $TEMPEST_CONFIG identity username $TEMPEST_USERNAME
-    iniset $TEMPEST_CONFIG identity password "$password"
-    iniset $TEMPEST_CONFIG identity tenant_name $TEMPEST_TENANT_NAME
-    iniset $TEMPEST_CONFIG identity alt_username $ALT_USERNAME
-    iniset $TEMPEST_CONFIG identity alt_password "$password"
-    iniset $TEMPEST_CONFIG identity alt_tenant_name $ALT_TENANT_NAME
     if [[ "$TEMPEST_HAS_ADMIN" == "True" ]]; then
-        iniset $TEMPEST_CONFIG identity admin_username $ADMIN_USERNAME
-        iniset $TEMPEST_CONFIG identity admin_password "$password"
-        iniset $TEMPEST_CONFIG identity admin_tenant_name $ADMIN_TENANT_NAME
-        iniset $TEMPEST_CONFIG identity admin_tenant_id $ADMIN_TENANT_ID
-        iniset $TEMPEST_CONFIG identity admin_domain_name $ADMIN_DOMAIN_NAME
+        iniset $TEMPEST_CONFIG auth admin_username $ADMIN_USERNAME
+        iniset $TEMPEST_CONFIG auth admin_password "$password"
+        iniset $TEMPEST_CONFIG auth admin_tenant_name $ADMIN_TENANT_NAME
+        iniset $TEMPEST_CONFIG auth admin_tenant_id $ADMIN_TENANT_ID
+        iniset $TEMPEST_CONFIG auth admin_domain_name $ADMIN_DOMAIN_NAME
     fi
     if [ "$ENABLE_IDENTITY_V2" == "False" ]; then
         # Only Identity v3 is available; then skip Identity API v2 tests
@@ -319,11 +313,7 @@
 
     # Compute
     iniset $TEMPEST_CONFIG compute ssh_user ${DEFAULT_INSTANCE_USER:-cirros} # DEPRECATED
-    iniset $TEMPEST_CONFIG compute network_for_ssh $PRIVATE_NETWORK_NAME
-    iniset $TEMPEST_CONFIG compute ip_version_for_ssh 4
-    iniset $TEMPEST_CONFIG compute ssh_timeout $BUILD_TIMEOUT
     iniset $TEMPEST_CONFIG compute image_ref $image_uuid
-    iniset $TEMPEST_CONFIG compute image_ssh_user ${DEFAULT_INSTANCE_USER:-cirros}
     iniset $TEMPEST_CONFIG compute image_ref_alt $image_uuid_alt
     iniset $TEMPEST_CONFIG compute image_alt_ssh_user ${ALT_INSTANCE_USER:-cirros}
     iniset $TEMPEST_CONFIG compute flavor_ref $flavor_ref
@@ -483,6 +473,10 @@
 
     # Validation
     iniset $TEMPEST_CONFIG validation run_validation ${TEMPEST_RUN_VALIDATION:-False}
+    iniset $TEMPEST_CONFIG validation ip_version_for_ssh 4
+    iniset $TEMPEST_CONFIG validation ssh_timeout $BUILD_TIMEOUT
+    iniset $TEMPEST_CONFIG validation image_ssh_user ${DEFAULT_INSTANCE_USER:-cirros}
+    iniset $TEMPEST_CONFIG validation network_for_ssh $PRIVATE_NETWORK_NAME
 
     # Volume
     # TODO(dkranz): Remove the bootable flag when Juno is end of life.
@@ -528,7 +522,6 @@
 
     # Dashboard
     iniset $TEMPEST_CONFIG dashboard dashboard_url "http://$SERVICE_HOST/"
-    iniset $TEMPEST_CONFIG dashboard login_url "http://$SERVICE_HOST/auth/login/"
 
     # CLI
     iniset $TEMPEST_CONFIG cli cli_dir $NOVA_BIN_DIR
@@ -595,13 +588,13 @@
         else
             tempest-account-generator -c $TEMPEST_CONFIG --os-username $ADMIN_USERNAME --os-password $ADMIN_PASSWORD --os-tenant-name $ADMIN_TENANT_NAME -r $TEMPEST_CONCURRENCY etc/accounts.yaml
         fi
-        iniset $TEMPEST_CONFIG auth allow_tenant_isolation False
+        iniset $TEMPEST_CONFIG auth use_dynamic_credentials False
         iniset $TEMPEST_CONFIG auth test_accounts_file "etc/accounts.yaml"
     elif [[ $TEMPEST_HAS_ADMIN == "False" ]]; then
-        iniset $TEMPEST_CONFIG auth allow_tenant_isolation ${TEMPEST_ALLOW_TENANT_ISOLATION:-False}
+        iniset $TEMPEST_CONFIG auth use_dynamic_credentials ${TEMPEST_ALLOW_TENANT_ISOLATION:-False}
 
     else
-        iniset $TEMPEST_CONFIG auth allow_tenant_isolation ${TEMPEST_ALLOW_TENANT_ISOLATION:-True}
+        iniset $TEMPEST_CONFIG auth use_dynamic_credentials ${TEMPEST_ALLOW_TENANT_ISOLATION:-True}
     fi
 
     # Restore IFS
diff --git a/stack.sh b/stack.sh
index 09ab474..bc67ce0 100755
--- a/stack.sh
+++ b/stack.sh
@@ -266,9 +266,7 @@
 # Some distros need to add repos beyond the defaults provided by the vendor
 # to pick up required packages.
 
-if is_fedora && [[ $DISTRO == "rhel7" ]]; then
-    # RHEL requires EPEL for many Open Stack dependencies
-
+function _install_epel_and_rdo {
     # NOTE: We always remove and install latest -- some environments
     # use snapshot images, and if EPEL version updates they break
     # unless we update them to latest version.
@@ -298,18 +296,27 @@
     sudo yum-config-manager --enable epel-bootstrap
     yum_install epel-release || \
         die $LINENO "Error installing EPEL repo, cannot continue"
-    # EPEL rpm has installed it's version
     sudo rm -f /etc/yum.repos.d/epel-bootstrap.repo
 
     # ... and also optional to be enabled
     sudo yum-config-manager --enable rhel-7-server-optional-rpms
 
+    # install the lastest RDO
     sudo yum install -y https://rdoproject.org/repos/rdo-release.rpm
 
     if is_oraclelinux; then
         sudo yum-config-manager --enable ol7_optional_latest ol7_addons ol7_MySQL56
     fi
+}
 
+# If you have all the repos installed above already setup (e.g. a CI
+# situation where they are on your image) you may choose to skip this
+# to speed things up
+SKIP_EPEL_INSTALL=$(trueorfalse False SKIP_EPEL_INSTALL)
+
+if is_fedora && [[ $DISTRO == "rhel7" ]] && \
+        [[ ${SKIP_EPEL_INSTALL} != True ]]; then
+    _install_epel_and_rdo
 fi
 
 
diff --git a/tools/dstat.sh b/tools/dstat.sh
index 6ba4515..3c0b3be 100755
--- a/tools/dstat.sh
+++ b/tools/dstat.sh
@@ -13,7 +13,7 @@
 LOGDIR=$1
 
 # Command line arguments for primary DStat process.
-DSTAT_OPTS="-tcmndrylpg --top-cpu-adv --top-io-adv"
+DSTAT_OPTS="-tcmndrylpg --top-cpu-adv --top-io-adv --swap"
 
 # Command-line arguments for secondary background DStat process.
 DSTAT_CSV_OPTS="-tcmndrylpg --output $LOGDIR/dstat-csv.log"
diff --git a/tools/update_clouds_yaml.py b/tools/update_clouds_yaml.py
index 3a364fe..eb7265f 100755
--- a/tools/update_clouds_yaml.py
+++ b/tools/update_clouds_yaml.py
@@ -36,6 +36,7 @@
         self._cloud_data = {
             'region_name': args.os_region_name,
             'identity_api_version': args.os_identity_api_version,
+            'volume_api_version': args.os_volume_api_version,
             'auth': {
                 'auth_url': args.os_auth_url,
                 'username': args.os_username,
@@ -82,6 +83,7 @@
     parser.add_argument('--os-cloud', required=True)
     parser.add_argument('--os-region-name', default='RegionOne')
     parser.add_argument('--os-identity-api-version', default='3')
+    parser.add_argument('--os-volume-api-version', default='2')
     parser.add_argument('--os-cacert')
     parser.add_argument('--os-auth-url', required=True)
     parser.add_argument('--os-username', required=True)