Merge "Rehome functions to enable Neutron's placement integration"
diff --git a/.zuul.yaml b/.zuul.yaml
index cbcb863..7a85266 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -630,9 +630,7 @@
voting: false
timeout: 9000
vars:
- # NOTE(yoctozepto): With concurrency equal 2, there is a random event
- # that this job will run out of memory at some point.
- tempest_concurrency: 1
+ configure_swap_size: 4096
# NOTE(yoctozepto): Debian Bullseye does not yet offer OVN. Switch to OVS
# for the time being.
devstack_localrc:
diff --git a/doc/source/configuration.rst b/doc/source/configuration.rst
index 6745614..b4fff4f 100644
--- a/doc/source/configuration.rst
+++ b/doc/source/configuration.rst
@@ -642,6 +642,12 @@
VOLUME_NAME_PREFIX="volume-"
VOLUME_BACKING_FILE_SIZE=24G
+When running highly concurrent tests, the default per-project quotas
+for volumes, backups, or snapshots may be too small. These can be
+adjusted by setting ``CINDER_QUOTA_VOLUMES``, ``CINDER_QUOTA_BACKUPS``,
+or ``CINDER_QUOTA_SNAPSHOTS`` to the desired value. (The default for
+each is 10.)
+
Keystone
~~~~~~~~
@@ -666,7 +672,6 @@
disable_service horizon
KEYSTONE_SERVICE_HOST=<KEYSTONE_IP_ADDRESS_FROM_REGION_ONE>
- KEYSTONE_AUTH_HOST=<KEYSTONE_IP_ADDRESS_FROM_REGION_ONE>
REGION_NAME=RegionTwo
KEYSTONE_REGION_NAME=RegionOne
diff --git a/files/apache-keystone.template b/files/apache-keystone.template
index 1284360..1a353e5 100644
--- a/files/apache-keystone.template
+++ b/files/apache-keystone.template
@@ -1,5 +1,4 @@
Listen %PUBLICPORT%
-Listen %ADMINPORT%
LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\" %D(us)" keystone_combined
<Directory %KEYSTONE_BIN%>
@@ -20,20 +19,6 @@
%SSLKEYFILE%
</VirtualHost>
-<VirtualHost *:%ADMINPORT%>
- WSGIDaemonProcess keystone-admin processes=3 threads=1 user=%USER% display-name=%{GROUP} %VIRTUALENV%
- WSGIProcessGroup keystone-admin
- WSGIScriptAlias / %KEYSTONE_BIN%/keystone-wsgi-admin
- WSGIApplicationGroup %{GLOBAL}
- WSGIPassAuthorization On
- ErrorLogFormat "%M"
- ErrorLog /var/log/%APACHE_NAME%/keystone.log
- CustomLog /var/log/%APACHE_NAME%/keystone_access.log keystone_combined
- %SSLENGINE%
- %SSLCERTFILE%
- %SSLKEYFILE%
-</VirtualHost>
-
%SSLLISTEN%<VirtualHost *:443>
%SSLLISTEN% %SSLENGINE%
%SSLLISTEN% %SSLCERTFILE%
@@ -49,13 +34,3 @@
WSGIApplicationGroup %{GLOBAL}
WSGIPassAuthorization On
</Location>
-
-Alias /identity_admin %KEYSTONE_BIN%/keystone-wsgi-admin
-<Location /identity_admin>
- SetHandler wsgi-script
- Options +ExecCGI
-
- WSGIProcessGroup keystone-admin
- WSGIApplicationGroup %{GLOBAL}
- WSGIPassAuthorization On
-</Location>
diff --git a/functions-common b/functions-common
index 11679e4..b1ca6ad 100644
--- a/functions-common
+++ b/functions-common
@@ -129,6 +129,28 @@
--os-password $ADMIN_PASSWORD \
--os-system-scope all
+ # system member
+ $PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
+ --file $CLOUDS_YAML \
+ --os-cloud devstack-system-member \
+ --os-region-name $REGION_NAME \
+ $CA_CERT_ARG \
+ --os-auth-url $KEYSTONE_SERVICE_URI \
+ --os-username system_member \
+ --os-password $ADMIN_PASSWORD \
+ --os-system-scope all
+
+ # system reader
+ $PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
+ --file $CLOUDS_YAML \
+ --os-cloud devstack-system-reader \
+ --os-region-name $REGION_NAME \
+ $CA_CERT_ARG \
+ --os-auth-url $KEYSTONE_SERVICE_URI \
+ --os-username system_reader \
+ --os-password $ADMIN_PASSWORD \
+ --os-system-scope all
+
cat >> $CLOUDS_YAML <<EOF
functional:
image_name: $DEFAULT_IMAGE_NAME
@@ -547,7 +569,7 @@
if [[ "$ERROR_ON_CLONE" = "True" ]]; then
echo "The $git_dest project was not found; if this is a gate job, add"
echo "the project to 'required-projects' in the job definition."
- die $LINENO "Cloning not allowed in this configuration"
+ die $LINENO "ERROR_ON_CLONE is set to True so cloning not allowed in this configuration"
fi
git_timed clone $git_clone_flags $git_remote $git_dest
fi
@@ -559,7 +581,7 @@
if [[ "$ERROR_ON_CLONE" = "True" ]]; then
echo "The $git_dest project was not found; if this is a gate job, add"
echo "the project to the \$PROJECTS variable in the job definition."
- die $LINENO "Cloning not allowed in this configuration"
+ die $LINENO "ERROR_ON_CLONE is set to True so cloning not allowed in this configuration"
fi
# '--branch' can also take tags
git_timed clone $git_clone_flags $git_remote $git_dest --branch $git_ref
@@ -936,6 +958,37 @@
echo $user_role_id
}
+# Gets or adds user role to system
+# Usage: get_or_add_user_system_role <role> <user> <system> [<user_domain>]
+function get_or_add_user_system_role {
+ local user_role_id
+ local domain_args
+
+ domain_args=$(_get_domain_args $4)
+
+ # Gets user role id
+ user_role_id=$(openstack role assignment list \
+ --role $1 \
+ --user $2 \
+ --system $3 \
+ $domain_args \
+ -f value -c Role)
+ if [[ -z "$user_role_id" ]]; then
+ # Adds role to user and get it
+ openstack role add $1 \
+ --user $2 \
+ --system $3 \
+ $domain_args
+ user_role_id=$(openstack role assignment list \
+ --role $1 \
+ --user $2 \
+ --system $3 \
+ $domain_args \
+ -f value -c Role)
+ fi
+ echo $user_role_id
+}
+
# Gets or adds group role to project
# Usage: get_or_add_group_project_role <role> <group> <project>
function get_or_add_group_project_role {
diff --git a/lib/cinder b/lib/cinder
index f3e2430..cefb609 100644
--- a/lib/cinder
+++ b/lib/cinder
@@ -286,6 +286,11 @@
iniset $CINDER_CONF DEFAULT allowed_direct_url_schemes $CINDER_ALLOWED_DIRECT_URL_SCHEMES
fi
+ # set default quotas
+ iniset $CINDER_CONF DEFAULT quota_volumes ${CINDER_QUOTA_VOLUMES:-10}
+ iniset $CINDER_CONF DEFAULT quota_backups ${CINDER_QUOTA_BACKUPS:-10}
+ iniset $CINDER_CONF DEFAULT quota_snapshots ${CINDER_QUOTA_SNAPSHOTS:-10}
+
# Avoid RPC timeouts in slow CI and test environments by doubling the
# default response timeout set by RPC clients. See bug #1873234 for more
# details and example failures.
diff --git a/lib/keystone b/lib/keystone
index 66e867c..065ca70 100644
--- a/lib/keystone
+++ b/lib/keystone
@@ -50,9 +50,7 @@
KEYSTONE_CONF_DIR=${KEYSTONE_CONF_DIR:-/etc/keystone}
KEYSTONE_CONF=$KEYSTONE_CONF_DIR/keystone.conf
KEYSTONE_PUBLIC_UWSGI_CONF=$KEYSTONE_CONF_DIR/keystone-uwsgi-public.ini
-KEYSTONE_ADMIN_UWSGI_CONF=$KEYSTONE_CONF_DIR/keystone-uwsgi-admin.ini
KEYSTONE_PUBLIC_UWSGI=$KEYSTONE_BIN_DIR/keystone-wsgi-public
-KEYSTONE_ADMIN_UWSGI=$KEYSTONE_BIN_DIR/keystone-wsgi-admin
# KEYSTONE_DEPLOY defines how keystone is deployed, allowed values:
# - mod_wsgi : Run keystone under Apache HTTPd mod_wsgi
@@ -81,21 +79,12 @@
KEYSTONE_TOKEN_FORMAT=${KEYSTONE_TOKEN_FORMAT:-fernet}
KEYSTONE_TOKEN_FORMAT=$(echo ${KEYSTONE_TOKEN_FORMAT} | tr '[:upper:]' '[:lower:]')
-# Set Keystone interface configuration
-KEYSTONE_AUTH_HOST=${KEYSTONE_AUTH_HOST:-$SERVICE_HOST}
-KEYSTONE_AUTH_PORT=${KEYSTONE_AUTH_PORT:-35357}
-KEYSTONE_AUTH_PORT_INT=${KEYSTONE_AUTH_PORT_INT:-35358}
-KEYSTONE_AUTH_PROTOCOL=${KEYSTONE_AUTH_PROTOCOL:-$SERVICE_PROTOCOL}
-
# Public facing bits
KEYSTONE_SERVICE_HOST=${KEYSTONE_SERVICE_HOST:-$SERVICE_HOST}
KEYSTONE_SERVICE_PORT=${KEYSTONE_SERVICE_PORT:-5000}
KEYSTONE_SERVICE_PORT_INT=${KEYSTONE_SERVICE_PORT_INT:-5001}
KEYSTONE_SERVICE_PROTOCOL=${KEYSTONE_SERVICE_PROTOCOL:-$SERVICE_PROTOCOL}
-# Bind hosts
-KEYSTONE_ADMIN_BIND_HOST=${KEYSTONE_ADMIN_BIND_HOST:-$KEYSTONE_SERVICE_HOST}
-
# Set the project for service accounts in Keystone
SERVICE_DOMAIN_NAME=${SERVICE_DOMAIN_NAME:-Default}
SERVICE_PROJECT_NAME=${SERVICE_PROJECT_NAME:-service}
@@ -106,7 +95,6 @@
# if we are running with SSL use https protocols
if is_service_enabled tls-proxy; then
- KEYSTONE_AUTH_PROTOCOL="https"
KEYSTONE_SERVICE_PROTOCOL="https"
fi
@@ -134,6 +122,9 @@
# Cache settings
KEYSTONE_ENABLE_CACHE=${KEYSTONE_ENABLE_CACHE:-True}
+# Whether to create a keystone admin endpoint for legacy applications
+KEYSTONE_ADMIN_ENDPOINT=$(trueorfalse False KEYSTONE_ADMIN_ENDPOINT)
+
# Functions
# ---------
@@ -154,11 +145,8 @@
sudo rm -f $(apache_site_config_for keystone)
else
stop_process "keystone"
- # TODO: remove admin at pike-2
remove_uwsgi_config "$KEYSTONE_PUBLIC_UWSGI_CONF" "$KEYSTONE_PUBLIC_UWSGI"
- remove_uwsgi_config "$KEYSTONE_ADMIN_UWSGI_CONF" "$KEYSTONE_ADMIN_UWSGI"
sudo rm -f $(apache_site_config_for keystone-wsgi-public)
- sudo rm -f $(apache_site_config_for keystone-wsgi-admin)
fi
}
@@ -171,12 +159,10 @@
local keystone_certfile=""
local keystone_keyfile=""
local keystone_service_port=$KEYSTONE_SERVICE_PORT
- local keystone_auth_port=$KEYSTONE_AUTH_PORT
local venv_path=""
if is_service_enabled tls-proxy; then
keystone_service_port=$KEYSTONE_SERVICE_PORT_INT
- keystone_auth_port=$KEYSTONE_AUTH_PORT_INT
fi
if [[ ${USE_VENV} = True ]]; then
venv_path="python-path=${PROJECT_VENV["keystone"]}/lib/$(python_version)/site-packages"
@@ -185,7 +171,6 @@
sudo cp $FILES/apache-keystone.template $keystone_apache_conf
sudo sed -e "
s|%PUBLICPORT%|$keystone_service_port|g;
- s|%ADMINPORT%|$keystone_auth_port|g;
s|%APACHE_NAME%|$APACHE_NAME|g;
s|%SSLLISTEN%|$keystone_ssl_listen|g;
s|%SSLENGINE%|$keystone_ssl|g;
@@ -223,12 +208,10 @@
iniset_rpc_backend keystone $KEYSTONE_CONF oslo_messaging_notifications
local service_port=$KEYSTONE_SERVICE_PORT
- local auth_port=$KEYSTONE_AUTH_PORT
if is_service_enabled tls-proxy; then
# Set the service ports for a proxy to take the originals
service_port=$KEYSTONE_SERVICE_PORT_INT
- auth_port=$KEYSTONE_AUTH_PORT_INT
fi
# Override the endpoints advertised by keystone (the public_endpoint and
@@ -238,7 +221,7 @@
# don't want the port (in the case of putting keystone on a path in
# apache).
iniset $KEYSTONE_CONF DEFAULT public_endpoint $KEYSTONE_SERVICE_URI
- iniset $KEYSTONE_CONF DEFAULT admin_endpoint $KEYSTONE_AUTH_URI
+ iniset $KEYSTONE_CONF DEFAULT admin_endpoint $KEYSTONE_SERVICE_URI
if [[ "$KEYSTONE_TOKEN_FORMAT" != "" ]]; then
iniset $KEYSTONE_CONF token provider $KEYSTONE_TOKEN_FORMAT
@@ -261,7 +244,6 @@
_config_keystone_apache_wsgi
else # uwsgi
write_uwsgi_config "$KEYSTONE_PUBLIC_UWSGI_CONF" "$KEYSTONE_PUBLIC_UWSGI" "/identity"
- write_uwsgi_config "$KEYSTONE_ADMIN_UWSGI_CONF" "$KEYSTONE_ADMIN_UWSGI" "/identity_admin"
fi
iniset $KEYSTONE_CONF DEFAULT max_token_size 16384
@@ -303,20 +285,28 @@
# admins admin admin admin
# nonadmins demo, alt_demo member, anotherrole demo, alt_demo
+# System User Roles
+# ------------------------------------------------------------------
+# all admin admin
+# all system_reader reader
+# all system_member member
+
# Migrated from keystone_data.sh
function create_keystone_accounts {
# The keystone bootstrapping process (performed via keystone-manage
- # bootstrap) creates an admin user, admin role, member role, and admin
+ # bootstrap) creates an admin user and an admin
# project. As a sanity check we exercise the CLI to retrieve the IDs for
# these values.
local admin_project
admin_project=$(openstack project show "admin" -f value -c id)
local admin_user
admin_user=$(openstack user show "admin" -f value -c id)
+ # These roles are also created during bootstrap but we don't need their IDs
local admin_role="admin"
local member_role="member"
+ local reader_role="reader"
async_run ks-domain-role get_or_add_user_domain_role $admin_role $admin_user default
@@ -367,6 +357,18 @@
async_run ks-alt-admin get_or_add_user_project_role $admin_role $admin_user $alt_demo_project
async_run ks-alt-another get_or_add_user_project_role $another_role $alt_demo_user $alt_demo_project
+ # Create two users, give one the member role on the system and the other
+ # the reader role on the system. These two users model system-member and
+ # system-reader personas. The admin user already has the admin role on the
+ # system and we can re-use this user as a system-admin.
+ system_member_user=$(get_or_create_user "system_member" \
+ "$ADMIN_PASSWORD" "default" "system_member@example.com")
+ async_run ks-system-member get_or_add_user_system_role $member_role $system_member_user "all"
+
+ system_reader_user=$(get_or_create_user "system_reader" \
+ "$ADMIN_PASSWORD" "default" "system_reader@example.com")
+ async_run ks-system-reader get_or_add_user_system_role $reader_role $system_reader_user "all"
+
# groups
local admin_group
admin_group=$(get_or_create_group "admins" \
@@ -383,6 +385,7 @@
async_wait ks-demo-{member,admin,another,invis}
async_wait ks-alt-{member,admin,another}
+ async_wait ks-system-{member,reader}
async_wait ks-group-{memberdemo,anotherdemo,memberalt,anotheralt,admin}
if is_service_enabled ldap; then
@@ -518,7 +521,7 @@
function start_keystone {
# Get right service port for testing
local service_port=$KEYSTONE_SERVICE_PORT
- local auth_protocol=$KEYSTONE_AUTH_PROTOCOL
+ local auth_protocol=$KEYSTONE_SERVICE_PROTOCOL
if is_service_enabled tls-proxy; then
service_port=$KEYSTONE_SERVICE_PORT_INT
auth_protocol="http"
@@ -546,7 +549,6 @@
# Start proxies if enabled
if is_service_enabled tls-proxy; then
start_tls_proxy keystone-service '*' $KEYSTONE_SERVICE_PORT $KEYSTONE_SERVICE_HOST $KEYSTONE_SERVICE_PORT_INT
- start_tls_proxy keystone-auth '*' $KEYSTONE_AUTH_PORT $KEYSTONE_AUTH_HOST $KEYSTONE_AUTH_PORT_INT
fi
# (re)start memcached to make sure we have a clean memcache.
@@ -569,9 +571,7 @@
# - ``ADMIN_PASSWORD``
# - ``IDENTITY_API_VERSION``
# - ``REGION_NAME``
-# - ``KEYSTONE_SERVICE_PROTOCOL``
-# - ``KEYSTONE_SERVICE_HOST``
-# - ``KEYSTONE_SERVICE_PORT``
+# - ``KEYSTONE_SERVICE_URI``
function bootstrap_keystone {
$KEYSTONE_BIN_DIR/keystone-manage bootstrap \
--bootstrap-username admin \
@@ -580,8 +580,16 @@
--bootstrap-role-name admin \
--bootstrap-service-name keystone \
--bootstrap-region-id "$REGION_NAME" \
- --bootstrap-admin-url "$KEYSTONE_AUTH_URI" \
--bootstrap-public-url "$KEYSTONE_SERVICE_URI"
+ if [ "$KEYSTONE_ADMIN_ENDPOINT" == "True" ]; then
+ openstack endpoint create --region "$REGION_NAME" \
+ --os-username admin \
+ --os-user-domain-id default \
+ --os-password "$ADMIN_PASSWORD" \
+ --os-project-name admin \
+ --os-project-domain-id default \
+ keystone admin "$KEYSTONE_SERVICE_URI"
+ fi
}
# create_ldap_domain() - Create domain file and initialize domain with a user
diff --git a/lib/neutron-legacy b/lib/neutron-legacy
index b41dfca..be29f99 100644
--- a/lib/neutron-legacy
+++ b/lib/neutron-legacy
@@ -556,11 +556,7 @@
function start_mutnauq_other_agents {
run_process q-dhcp "$AGENT_DHCP_BINARY --config-file $NEUTRON_CONF --config-file $Q_DHCP_CONF_FILE"
- if is_service_enabled neutron-vpnaas; then
- : # Started by plugin
- else
- run_process q-l3 "$AGENT_L3_BINARY $(determine_config_files neutron-l3-agent)"
- fi
+ run_process q-l3 "$AGENT_L3_BINARY $(determine_config_files neutron-l3-agent)"
run_process q-meta "$AGENT_META_BINARY --config-file $NEUTRON_CONF --config-file $Q_META_CONF_FILE"
run_process q-metering "$AGENT_METERING_BINARY --config-file $NEUTRON_CONF --config-file $METERING_AGENT_CONF_FILENAME"
diff --git a/lib/swift b/lib/swift
index 9885241..b376993 100644
--- a/lib/swift
+++ b/lib/swift
@@ -430,7 +430,7 @@
swift_pipeline+=" authtoken"
if is_service_enabled s3api;then
swift_pipeline+=" s3token"
- iniset ${SWIFT_CONFIG_PROXY_SERVER} filter:s3token auth_uri ${KEYSTONE_AUTH_URI_V3}
+ iniset ${SWIFT_CONFIG_PROXY_SERVER} filter:s3token auth_uri ${KEYSTONE_SERVICE_URI_V3}
iniset ${SWIFT_CONFIG_PROXY_SERVER} filter:s3token delay_auth_decision true
fi
swift_pipeline+=" keystoneauth"
@@ -521,7 +521,7 @@
local auth_vers
auth_vers=$(iniget ${testfile} func_test auth_version)
iniset ${testfile} func_test auth_host ${KEYSTONE_SERVICE_HOST}
- if [[ "$KEYSTONE_AUTH_PROTOCOL" == "https" ]]; then
+ if [[ "$KEYSTONE_SERVICE_PROTOCOL" == "https" ]]; then
iniset ${testfile} func_test auth_port 443
else
iniset ${testfile} func_test auth_port 80
diff --git a/stack.sh b/stack.sh
index 8a8e3ea..b5ad81b 100755
--- a/stack.sh
+++ b/stack.sh
@@ -876,7 +876,7 @@
install_keystonemiddleware
if is_service_enabled keystone; then
- if [ "$KEYSTONE_AUTH_HOST" == "$SERVICE_HOST" ]; then
+ if [ "$KEYSTONE_SERVICE_HOST" == "$SERVICE_HOST" ]; then
stack_install_service keystone
configure_keystone
fi
@@ -1090,10 +1090,13 @@
source $TOP_DIR/userrc_early
+# Write a clouds.yaml file
+write_clouds_yaml
+
if is_service_enabled keystone; then
echo_summary "Starting Keystone"
- if [ "$KEYSTONE_AUTH_HOST" == "$SERVICE_HOST" ]; then
+ if [ "$KEYSTONE_SERVICE_HOST" == "$SERVICE_HOST" ]; then
init_keystone
start_keystone
bootstrap_keystone
@@ -1118,9 +1121,6 @@
fi
-# Write a clouds.yaml file
-write_clouds_yaml
-
# Horizon
# -------
diff --git a/tools/fixup_stuff.sh b/tools/fixup_stuff.sh
index 71fba2e..fe5dafa 100755
--- a/tools/fixup_stuff.sh
+++ b/tools/fixup_stuff.sh
@@ -26,39 +26,6 @@
FILES=$TOP_DIR/files
fi
-# Keystone Port Reservation
-# -------------------------
-# Reserve and prevent ``KEYSTONE_AUTH_PORT`` and ``KEYSTONE_AUTH_PORT_INT`` from
-# being used as ephemeral ports by the system. The default(s) are 35357 and
-# 35358 which are in the Linux defined ephemeral port range (in disagreement
-# with the IANA ephemeral port range). This is a workaround for bug #1253482
-# where Keystone will try and bind to the port and the port will already be
-# in use as an ephemeral port by another process. This places an explicit
-# exception into the Kernel for the Keystone AUTH ports.
-function fixup_keystone {
- keystone_ports=${KEYSTONE_AUTH_PORT:-35357},${KEYSTONE_AUTH_PORT_INT:-35358}
-
- # Only do the reserved ports when available, on some system (like containers)
- # where it's not exposed we are almost pretty sure these ports would be
- # exclusive for our DevStack.
- if sysctl net.ipv4.ip_local_reserved_ports >/dev/null 2>&1; then
- # Get any currently reserved ports, strip off leading whitespace
- reserved_ports=$(sysctl net.ipv4.ip_local_reserved_ports | awk -F'=' '{print $2;}' | sed 's/^ //')
-
- if [[ -z "${reserved_ports}" ]]; then
- # If there are no currently reserved ports, reserve the keystone ports
- sudo sysctl -w net.ipv4.ip_local_reserved_ports=${keystone_ports}
- else
- # If there are currently reserved ports, keep those and also reserve the
- # Keystone specific ports. Duplicate reservations are merged into a single
- # reservation (or range) automatically by the kernel.
- sudo sysctl -w net.ipv4.ip_local_reserved_ports=${keystone_ports},${reserved_ports}
- fi
- else
- echo_summary "WARNING: unable to reserve keystone ports"
- fi
-}
-
# Python Packages
# ---------------
@@ -182,7 +149,6 @@
}
function fixup_all {
- fixup_keystone
fixup_ubuntu
fixup_fedora
fixup_suse