Merge "Fleetify nova conductor for N cells"
diff --git a/.gitignore b/.gitignore
index 7967e14..d2c127d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -24,6 +24,7 @@
files/get-pip.py*
files/ir-deploy*
files/ironic-inspector*
+files/etcd*
local.conf
local.sh
localrc
diff --git a/clean.sh b/clean.sh
index ef38fbf..9ffe3be 100755
--- a/clean.sh
+++ b/clean.sh
@@ -125,6 +125,13 @@
sudo rm -rf $SCREEN_LOGDIR
fi
+# Clean out the sytemd user unit files if systemd was used.
+if [[ "$USE_SYSTEMD" = "True" ]]; then
+ sudo find $SYSTEMD_DIR -type f -name '*devstack@*service' -delete
+ # Make systemd aware of the deletion.
+ $SYSTEMCTL daemon-reload
+fi
+
# Clean up venvs
DIRS_TO_CLEAN="$WHEELHOUSE ${PROJECT_VENV[@]} .config/openstack"
rm -rf $DIRS_TO_CLEAN
diff --git a/doc/source/configuration.rst b/doc/source/configuration.rst
index 66b8702..064bf51 100644
--- a/doc/source/configuration.rst
+++ b/doc/source/configuration.rst
@@ -195,6 +195,9 @@
Setting it here also makes it available for ``openrc`` to set ``OS_AUTH_URL``.
``HOST_IPV6`` is not set by default.
+For architecture specific configurations which differ from the x86 default
+here, see `arch-configuration`_.
+
Historical Notes
================
@@ -749,3 +752,60 @@
::
TERMINATE_TIMEOUT=30
+
+
+.. _arch-configuration:
+
+Architectures
+-------------
+
+The upstream CI runs exclusively on nodes with x86 architectures, but
+OpenStack supports even more architectures. Some of them need to configure
+Devstack in a certain way.
+
+KVM on s390x (IBM z Systems)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+KVM on s390x (IBM z Systems) is supported since the *Kilo* release. For
+an all-in-one setup, these minimal settings in the ``local.conf`` file
+are needed::
+
+ [[local|localrc]]
+ ADMIN_PASSWORD=secret
+ DATABASE_PASSWORD=$ADMIN_PASSWORD
+ RABBIT_PASSWORD=$ADMIN_PASSWORD
+ SERVICE_PASSWORD=$ADMIN_PASSWORD
+
+ DOWNLOAD_DEFAULT_IMAGES=False
+ IMAGE_URLS="https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-s390x-disk1.img"
+
+ enable_service n-sproxy
+ disable_service n-novnc
+ disable_service etcd3 # https://bugs.launchpad.net/devstack/+bug/1693192
+
+ [[post-config|$NOVA_CONF]]
+
+ [serial_console]
+ base_url=ws://$HOST_IP:6083/ # optional
+
+Reasoning:
+
+* The default image of Devstack is x86 only, so we deactivate the download
+ with ``DOWNLOAD_DEFAULT_IMAGES``. The referenced guest image
+ in the code above (``IMAGE_URLS``) serves as an example. The list of
+ possible s390x guest images is not limited to that.
+
+* This platform doesn't support a graphical console like VNC or SPICE.
+ The technical reason is the missing framebuffer on the platform. This
+ means we rely on the substitute feature *serial console* which needs the
+ proxy service ``n-sproxy``. We also disable VNC's proxy ``n-novnc`` for
+ that reason . The configuration in the ``post-config`` section is only
+ needed if you want to use the *serial console* outside of the all-in-one
+ setup.
+
+* The service ``etcd3`` needs to be disabled as long as bug report
+ https://bugs.launchpad.net/devstack/+bug/1693192 is not resolved.
+
+.. note:: To run *Tempest* against this *Devstack* all-in-one, you'll need
+ to use a guest image which is smaller than 1GB when uncompressed.
+ The example image from above is bigger than that!
diff --git a/doc/source/plugin-registry.rst b/doc/source/plugin-registry.rst
index 9bfedcf..c1c66b9 100644
--- a/doc/source/plugin-registry.rst
+++ b/doc/source/plugin-registry.rst
@@ -149,6 +149,7 @@
stackube `git://git.openstack.org/openstack/stackube <https://git.openstack.org/cgit/openstack/stackube>`__
tacker `git://git.openstack.org/openstack/tacker <https://git.openstack.org/cgit/openstack/tacker>`__
tap-as-a-service `git://git.openstack.org/openstack/tap-as-a-service <https://git.openstack.org/cgit/openstack/tap-as-a-service>`__
+tap-as-a-service-dashboard `git://git.openstack.org/openstack/tap-as-a-service-dashboard <https://git.openstack.org/cgit/openstack/tap-as-a-service-dashboard>`__
tricircle `git://git.openstack.org/openstack/tricircle <https://git.openstack.org/cgit/openstack/tricircle>`__
trio2o `git://git.openstack.org/openstack/trio2o <https://git.openstack.org/cgit/openstack/trio2o>`__
trove `git://git.openstack.org/openstack/trove <https://git.openstack.org/cgit/openstack/trove>`__
diff --git a/functions b/functions
index 3ca3717..e056c3f 100644
--- a/functions
+++ b/functions
@@ -732,6 +732,24 @@
sudo systemctl daemon-reload
}
+# Get a random port from the local port range
+#
+# This function returns an available port in the local port range. The search
+# order is not truly random, but should be considered a random value by the
+# user because it depends on the state of your local system.
+function get_random_port {
+ read lower_port upper_port < /proc/sys/net/ipv4/ip_local_port_range
+ while true; do
+ for (( port = upper_port ; port >= lower_port ; port-- )); do
+ sudo lsof -i ":$port" &> /dev/null
+ if [[ $? > 0 ]] ; then
+ break 2
+ fi
+ done
+ done
+ echo $port
+}
+
# Restore xtrace
$_XTRACE_FUNCTIONS
diff --git a/functions-common b/functions-common
index 30933ea..65db681 100644
--- a/functions-common
+++ b/functions-common
@@ -1508,8 +1508,13 @@
}
-# Helper function to build a basic unit file and run it under systemd.
-function _run_under_systemd {
+# Defines a systemd service which can be enabled and started later on.
+# arg1: The openstack service name ('n-cpu', 'c-sch', ...).
+# arg2: The command to start (e.g. path to service binary + config files).
+# arg3: The group which owns the process.
+# arg4: The user which owns the process.
+# Returns: The systemd service name which got defined.
+function _define_systemd_service {
local service=$1
local command="$2"
local cmd=$command
@@ -1524,9 +1529,7 @@
else
write_user_unit_file $systemd_service "$cmd" "$group" "$user"
fi
-
- $SYSTEMCTL enable $systemd_service
- $SYSTEMCTL start $systemd_service
+ echo $systemd_service
}
# Helper to remove the ``*.failure`` files under ``$SERVICE_DIR/$SCREEN_NAME``.
@@ -1567,11 +1570,19 @@
local user=$4
local name=$service
+ local systemd_service
time_start "run_process"
+ # Note we deliberately make all service files, even if the service
+ # isn't enabled, so it can be enabled by a dev manually on command
+ # line.
+ if [[ "$USE_SYSTEMD" = "True" ]]; then
+ systemd_service=$(_define_systemd_service "$name" "$command" "$group" "$user")
+ fi
if is_service_enabled $service; then
if [[ "$USE_SYSTEMD" = "True" ]]; then
- _run_under_systemd "$name" "$command" "$group" "$user"
+ $SYSTEMCTL enable $systemd_service
+ $SYSTEMCTL start $systemd_service
elif [[ "$USE_SCREEN" = "True" ]]; then
if [[ "$user" == "root" ]]; then
command="sudo $command"
@@ -2491,7 +2502,7 @@
if [[ -n "$start_time" ]]; then
die $LINENO "Trying to start the clock on $name, but it's already been started"
fi
- _TIME_START[$name]=$(date +%s)
+ _TIME_START[$name]=$(date +%s%3N)
}
# time_stop $name
@@ -2512,7 +2523,7 @@
if [[ -z "$start_time" ]]; then
die $LINENO "Trying to stop the clock on $name, but it was never started"
fi
- end_time=$(date +%s)
+ end_time=$(date +%s%3N)
elapsed_time=$(($end_time - $start_time))
total=${_TIME_TOTAL[$name]:-0}
# reset the clock so we can start it in the future
@@ -2520,6 +2531,49 @@
_TIME_TOTAL[$name]=$(($total + $elapsed_time))
}
+function oscwrap {
+ local out
+ local rc
+ local start
+ local end
+ # Cannot use timer_start and timer_stop as we run in subshells
+ # and those rely on modifying vars in the same process (which cannot
+ # happen from a subshell.
+ start=$(date +%s%3N)
+ out=$(command openstack "$@")
+ rc=$?
+ end=$(date +%s%3N)
+ echo $((end - start)) >> $OSCWRAP_TIMER_FILE
+
+ echo "$out"
+ return $rc
+}
+
+function install_oscwrap {
+ # File to accumulate our timing data
+ OSCWRAP_TIMER_FILE=$(mktemp)
+ # Bash by default doesn't expand aliases, allow it for the aliases
+ # we want to whitelist.
+ shopt -s expand_aliases
+ # Remove all aliases that might be expanded to preserve old unexpanded
+ # behavior
+ unalias -a
+ # Add only the alias we want for openstack
+ alias openstack=oscwrap
+}
+
+function cleanup_oscwrap {
+ local total=0
+ if python3_enabled ; then
+ local python=python3
+ else
+ local python=python
+ fi
+ total=$(cat $OSCWRAP_TIMER_FILE | $python -c "import sys; print(sum(int(l) for l in sys.stdin))")
+ _TIME_TOTAL["osc"]=$total
+ rm $OSCWRAP_TIMER_FILE
+}
+
# time_totals
# Print out total time summary
function time_totals {
@@ -2538,6 +2592,8 @@
fi
done
+ cleanup_oscwrap
+
xtrace=$(set +o | grep xtrace)
set +o xtrace
@@ -2549,6 +2605,8 @@
echo
for t in ${!_TIME_TOTAL[*]}; do
local v=${_TIME_TOTAL[$t]}
+ # because we're recording in milliseconds
+ v=$(($v / 1000))
printf "%-${len}s %3d\n" "$t" "$v"
done
echo "========================="
diff --git a/inc/python b/inc/python
index 4c443d6..718cbb2 100644
--- a/inc/python
+++ b/inc/python
@@ -320,6 +320,14 @@
fi
$xtrace
+
+ # Also install test requirements
+ local install_test_reqs=""
+ local test_req="${!#}/test-requirements.txt"
+ if [[ -e "$test_req" ]]; then
+ install_test_reqs="-r $test_req"
+ fi
+
# adding SETUPTOOLS_SYS_PATH_TECHNIQUE is a workaround to keep
# the same behaviour of setuptools before version 25.0.0.
# related issue: https://github.com/pypa/pip/issues/3874
@@ -329,24 +337,10 @@
no_proxy="${no_proxy:-}" \
PIP_FIND_LINKS=$PIP_FIND_LINKS \
SETUPTOOLS_SYS_PATH_TECHNIQUE=rewrite \
- $cmd_pip $upgrade \
+ $cmd_pip $upgrade $install_test_reqs \
$@
result=$?
- # Also install test requirements
- local test_req="${!#}/test-requirements.txt"
- if [[ $result == 0 ]] && [[ -e "$test_req" ]]; then
- echo "Installing test-requirements for $test_req"
- $sudo_pip \
- http_proxy=${http_proxy:-} \
- https_proxy=${https_proxy:-} \
- no_proxy=${no_proxy:-} \
- PIP_FIND_LINKS=$PIP_FIND_LINKS \
- $cmd_pip $upgrade \
- -r $test_req
- result=$?
- fi
-
time_stop "pip_install"
return $result
}
diff --git a/lib/apache b/lib/apache
index 43d5000..c1b6bf8 100644
--- a/lib/apache
+++ b/lib/apache
@@ -238,7 +238,13 @@
# create a home for the sockets; note don't use /tmp -- apache has
# a private view of it on some platforms.
local socket_dir='/var/run/uwsgi'
- sudo install -d -o $STACK_USER -m 755 $socket_dir
+
+ # /var/run will be empty on ubuntu after reboot, so we can use systemd-temptiles
+ # to automatically create $socket_dir.
+ sudo mkdir -p /etc/tmpfiles.d/
+ echo "d $socket_dir 0755 $STACK_USER root" | sudo tee /etc/tmpfiles.d/uwsgi.conf
+ sudo systemd-tmpfiles --create /etc/tmpfiles.d/uwsgi.conf
+
local socket="$socket_dir/${name}.socket"
# always cleanup given that we are using iniset here
diff --git a/lib/cinder b/lib/cinder
index 291eebe..2068812 100644
--- a/lib/cinder
+++ b/lib/cinder
@@ -407,8 +407,10 @@
# (Re)create cinder database
recreate_database cinder
+ time_start "dbsync"
# Migrate cinder database
$CINDER_BIN_DIR/cinder-manage --config-file $CINDER_CONF db sync
+ time_stop "dbsync"
fi
if is_service_enabled c-vol && [[ -n "$CINDER_ENABLED_BACKENDS" ]]; then
diff --git a/lib/databases/mysql b/lib/databases/mysql
index 7bbcace..a0cf7a4 100644
--- a/lib/databases/mysql
+++ b/lib/databases/mysql
@@ -71,6 +71,10 @@
elif is_fedora; then
mysql=mariadb
my_conf=/etc/my.cnf
+ local cracklib_conf=/etc/my.cnf.d/cracklib_password_check.cnf
+ if [ -f "$cracklib_conf" ]; then
+ inicomment -sudo "$cracklib_conf" "mariadb" "plugin-load-add"
+ fi
else
exit_distro_not_supported "mysql configuration"
fi
diff --git a/lib/etcd3 b/lib/etcd3
index f8b113b..ea58403 100644
--- a/lib/etcd3
+++ b/lib/etcd3
@@ -41,21 +41,14 @@
# start_etcd3() - Starts to run the etcd process
function start_etcd3 {
- # Don't install in sub nodes (multinode scenario)
- if [ "$SERVICE_HOST" != "$HOST_IP" ]; then
- return
- fi
-
- _install_etcd
-
local cmd="$ETCD_BIN_DIR/etcd"
cmd+=" --name $HOSTNAME --data-dir $ETCD_DATA_DIR"
cmd+=" --initial-cluster-state new --initial-cluster-token etcd-cluster-01"
cmd+=" --initial-cluster $HOSTNAME=http://$SERVICE_HOST:2380"
cmd+=" --initial-advertise-peer-urls http://$SERVICE_HOST:2380"
- cmd+=" --advertise-client-urls http://$SERVICE_HOST:$ETCD_PORT"
+ cmd+=" --advertise-client-urls http://${HOST_IP}:$ETCD_PORT"
cmd+=" --listen-peer-urls http://0.0.0.0:2380 "
- cmd+=" --listen-client-urls http://$SERVICE_HOST:$ETCD_PORT"
+ cmd+=" --listen-client-urls http://${HOST_IP}:$ETCD_PORT"
local unitfile="$SYSTEMD_DIR/$ETCD_SYSTEMD_SERVICE"
write_user_unit_file $ETCD_SYSTEMD_SERVICE "$cmd" "" "root"
@@ -96,7 +89,7 @@
sudo rm -rf $ETCD_DATA_DIR
}
-function _install_etcd {
+function install_etcd3 {
echo "Installing etcd"
# Make sure etcd3 downloads the correct architecture
@@ -115,10 +108,6 @@
ETCD_NAME=etcd-$ETCD_VERSION-linux-$ETCD_ARCH
- # Install the libraries needed. Note: tooz for example does not have a hard dependency on these libraries
- pip_install etcd3
- pip_install etcd3gw
-
# Create the necessary directories
sudo mkdir -p $ETCD_BIN_DIR
sudo mkdir -p $ETCD_DATA_DIR
diff --git a/lib/glance b/lib/glance
index 57b5f45..baf8c61 100644
--- a/lib/glance
+++ b/lib/glance
@@ -285,11 +285,13 @@
# (Re)create glance database
recreate_database glance
+ time_start "dbsync"
# Migrate glance database
$GLANCE_BIN_DIR/glance-manage --config-file $GLANCE_CONF_DIR/glance-api.conf db_sync
# Load metadata definitions
$GLANCE_BIN_DIR/glance-manage --config-file $GLANCE_CONF_DIR/glance-api.conf db_load_metadefs
+ time_stop "dbsync"
create_glance_cache_dir
}
diff --git a/lib/keystone b/lib/keystone
index eaed937..eb46526 100644
--- a/lib/keystone
+++ b/lib/keystone
@@ -124,9 +124,15 @@
# Security compliance
KEYSTONE_SECURITY_COMPLIANCE_ENABLED=${KEYSTONE_SECURITY_COMPLIANCE_ENABLED:-True}
KEYSTONE_LOCKOUT_FAILURE_ATTEMPTS=${KEYSTONE_LOCKOUT_FAILURE_ATTEMPTS:-2}
-KEYSTONE_LOCKOUT_DURATION=${KEYSTONE_LOCKOUT_DURATION:-5}
+KEYSTONE_LOCKOUT_DURATION=${KEYSTONE_LOCKOUT_DURATION:-10}
KEYSTONE_UNIQUE_LAST_PASSWORD_COUNT=${KEYSTONE_UNIQUE_LAST_PASSWORD_COUNT:-2}
+# Number of bcrypt hashing rounds, increasing number exponentially increases required
+# resources to generate password hash. This is very effective way to protect from
+# bruteforce attacks. 4 is minimal value that can be specified for bcrypt and
+# it works way faster than default 12. Minimal value is great for CI and development
+# however may not be suitable for real production.
+KEYSTONE_PASSWORD_HASH_ROUNDS=${KEYSTONE_PASSWORD_HASH_ROUNDS:-4}
# Functions
# ---------
@@ -225,6 +231,7 @@
fi
iniset $KEYSTONE_CONF identity driver "$KEYSTONE_IDENTITY_BACKEND"
+ iniset $KEYSTONE_CONF identity password_hash_rounds $KEYSTONE_PASSWORD_HASH_ROUNDS
iniset $KEYSTONE_CONF assignment driver "$KEYSTONE_ASSIGNMENT_BACKEND"
iniset $KEYSTONE_CONF role driver "$KEYSTONE_ROLE_BACKEND"
iniset $KEYSTONE_CONF resource driver "$KEYSTONE_RESOURCE_BACKEND"
@@ -458,8 +465,10 @@
recreate_database keystone
fi
+ time_start "dbsync"
# Initialize keystone database
$KEYSTONE_BIN_DIR/keystone-manage --config-file $KEYSTONE_CONF db_sync
+ time_stop "dbsync"
if [[ "$KEYSTONE_TOKEN_FORMAT" == "pki" || "$KEYSTONE_TOKEN_FORMAT" == "pkiz" ]]; then
# Set up certificates
diff --git a/lib/libraries b/lib/libraries
new file mode 100644
index 0000000..4ceb804
--- /dev/null
+++ b/lib/libraries
@@ -0,0 +1,143 @@
+#!/bin/bash
+#
+# lib/oslo
+#
+# Functions to install libraries from git
+#
+# We need this to handle the fact that projects would like to use
+# pre-released versions of oslo libraries.
+
+# Dependencies:
+#
+# - ``functions`` file
+
+# ``stack.sh`` calls the entry points in this order:
+#
+# - install_libraries
+
+# Save trace setting
+_XTRACE_LIB_LIBRARIES=$(set +o | grep xtrace)
+set +o xtrace
+
+
+# Defaults
+# --------
+GITDIR["automaton"]=$DEST/automaton
+GITDIR["castellan"]=$DEST/castellan
+GITDIR["cliff"]=$DEST/cliff
+GITDIR["cursive"]=$DEST/cursive
+GITDIR["debtcollector"]=$DEST/debtcollector
+GITDIR["futurist"]=$DEST/futurist
+GITDIR["os-client-config"]=$DEST/os-client-config
+GITDIR["osc-lib"]=$DEST/osc-lib
+GITDIR["oslo.cache"]=$DEST/oslo.cache
+GITDIR["oslo.concurrency"]=$DEST/oslo.concurrency
+GITDIR["oslo.config"]=$DEST/oslo.config
+GITDIR["oslo.context"]=$DEST/oslo.context
+GITDIR["oslo.db"]=$DEST/oslo.db
+GITDIR["oslo.i18n"]=$DEST/oslo.i18n
+GITDIR["oslo.log"]=$DEST/oslo.log
+GITDIR["oslo.messaging"]=$DEST/oslo.messaging
+GITDIR["oslo.middleware"]=$DEST/oslo.middleware
+GITDIR["oslo.policy"]=$DEST/oslo.policy
+GITDIR["oslo.privsep"]=$DEST/oslo.privsep
+GITDIR["oslo.reports"]=$DEST/oslo.reports
+GITDIR["oslo.rootwrap"]=$DEST/oslo.rootwrap
+GITDIR["oslo.serialization"]=$DEST/oslo.serialization
+GITDIR["oslo.service"]=$DEST/oslo.service
+GITDIR["oslo.utils"]=$DEST/oslo.utils
+GITDIR["oslo.versionedobjects"]=$DEST/oslo.versionedobjects
+GITDIR["oslo.vmware"]=$DEST/oslo.vmware
+GITDIR["osprofiler"]=$DEST/osprofiler
+GITDIR["pycadf"]=$DEST/pycadf
+GITDIR["python-openstacksdk"]=$DEST/python-openstacksdk
+GITDIR["stevedore"]=$DEST/stevedore
+GITDIR["taskflow"]=$DEST/taskflow
+GITDIR["tooz"]=$DEST/tooz
+
+# Non oslo libraries are welcomed below as well, this prevents
+# duplication of this code.
+GITDIR["os-brick"]=$DEST/os-brick
+GITDIR["os-traits"]=$DEST/os-traits
+
+# Support entry points installation of console scripts
+OSLO_BIN_DIR=$(get_python_exec_prefix)
+
+
+# Functions
+# ---------
+
+function _install_lib_from_source {
+ local name=$1
+ if use_library_from_git "$name"; then
+ git_clone_by_name "$name"
+ setup_dev_lib "$name"
+ fi
+}
+
+# install_oslo - install libraries that oslo needs
+function install_oslo {
+ install_libs
+}
+
+# install_libs() - Install additional libraries that we need and want
+# on all environments. Some will only install here if from source,
+# others will always install.
+function install_libs {
+ _install_lib_from_source "automaton"
+ _install_lib_from_source "castellan"
+ _install_lib_from_source "cliff"
+ _install_lib_from_source "cursive"
+ _install_lib_from_source "debtcollector"
+ _install_lib_from_source "futurist"
+ _install_lib_from_source "osc-lib"
+ _install_lib_from_source "os-client-config"
+ _install_lib_from_source "oslo.cache"
+ _install_lib_from_source "oslo.concurrency"
+ _install_lib_from_source "oslo.config"
+ _install_lib_from_source "oslo.context"
+ _install_lib_from_source "oslo.db"
+ _install_lib_from_source "oslo.i18n"
+ _install_lib_from_source "oslo.log"
+ _install_lib_from_source "oslo.messaging"
+ _install_lib_from_source "oslo.middleware"
+ _install_lib_from_source "oslo.policy"
+ _install_lib_from_source "oslo.privsep"
+ _install_lib_from_source "oslo.reports"
+ _install_lib_from_source "oslo.rootwrap"
+ _install_lib_from_source "oslo.serialization"
+ _install_lib_from_source "oslo.service"
+ _install_lib_from_source "oslo.utils"
+ _install_lib_from_source "oslo.versionedobjects"
+ _install_lib_from_source "oslo.vmware"
+ _install_lib_from_source "osprofiler"
+ _install_lib_from_source "pycadf"
+ _install_lib_from_source "python-openstacksdk"
+ _install_lib_from_source "stevedore"
+ _install_lib_from_source "taskflow"
+ _install_lib_from_source "tooz"
+ # installation of additional libraries
+ #
+ # os-traits for nova
+ _install_lib_from_source "os-brick"
+ _install_lib_from_source "os-traits"
+ #
+ # python client libraries we might need from git can go here
+ _install_lib_from_source "python-barbicanclient"
+
+
+ # etcd (because tooz does not have a hard dependency on these)
+ #
+ # NOTE(sdague): this is currently a work around because tooz
+ # doesn't pull in etcd3.
+ pip_install etcd3
+ pip_install etcd3gw
+}
+
+# Restore xtrace
+$_XTRACE_LIB_LIBRARIES
+
+# Tell emacs to use shell-script-mode
+## Local variables:
+## mode: shell-script
+## End:
diff --git a/lib/neutron b/lib/neutron
index 5c88a50..2a660ec 100644
--- a/lib/neutron
+++ b/lib/neutron
@@ -336,8 +336,10 @@
recreate_database neutron
+ time_start "dbsync"
# Run Neutron db migrations
$NEUTRON_BIN_DIR/neutron-db-manage upgrade heads
+ time_stop "dbsync"
create_neutron_cache_dir
}
diff --git a/lib/neutron-legacy b/lib/neutron-legacy
index 67cf110..784f3a8 100644
--- a/lib/neutron-legacy
+++ b/lib/neutron-legacy
@@ -417,8 +417,10 @@
# init_mutnauq() - Initialize databases, etc.
function init_mutnauq {
recreate_database $Q_DB_NAME
+ time_start "dbsync"
# Run Neutron db migrations
$NEUTRON_BIN_DIR/neutron-db-manage --config-file $NEUTRON_CONF --config-file /$Q_PLUGIN_CONF_FILE upgrade head
+ time_stop "dbsync"
}
# install_mutnauq() - Collect source and prepare
diff --git a/lib/nova b/lib/nova
index 8eac254..788588f 100644
--- a/lib/nova
+++ b/lib/nova
@@ -462,7 +462,7 @@
fi
iniset $NOVA_CONF wsgi api_paste_config "$NOVA_API_PASTE_INI"
iniset $NOVA_CONF DEFAULT rootwrap_config "$NOVA_CONF_DIR/rootwrap.conf"
- iniset $NOVA_CONF DEFAULT scheduler_driver "$SCHEDULER"
+ iniset $NOVA_CONF scheduler driver "$SCHEDULER"
iniset $NOVA_CONF filter_scheduler enabled_filters "$FILTERS"
iniset $NOVA_CONF DEFAULT default_floating_pool "$PUBLIC_NETWORK_NAME"
if [[ $SERVICE_IP_VERSION == 6 ]]; then
@@ -676,7 +676,9 @@
iniset $NOVA_CELLS_CONF DEFAULT enabled_apis metadata
fi
+ time_start "dbsync"
$NOVA_BIN_DIR/nova-manage --config-file $NOVA_CELLS_CONF db sync
+ time_stop "dbsync"
$NOVA_BIN_DIR/nova-manage --config-file $NOVA_CELLS_CONF cell create --name=region --cell_type=parent --username=$RABBIT_USERID --hostname=$RABBIT_HOST --port=5672 --password=$RABBIT_PASSWORD --virtual_host=/ --woffset=0 --wscale=1
$NOVA_BIN_DIR/nova-manage cell create --name=child --cell_type=child --username=$RABBIT_USERID --hostname=$RABBIT_HOST --port=5672 --password=$RABBIT_PASSWORD --virtual_host=child_cell --woffset=0 --wscale=1
diff --git a/lib/nova_plugins/hypervisor-xenserver b/lib/nova_plugins/hypervisor-xenserver
index 880b87f..6f79e4f 100644
--- a/lib/nova_plugins/hypervisor-xenserver
+++ b/lib/nova_plugins/hypervisor-xenserver
@@ -84,14 +84,6 @@
* * * * * /root/rotate_xen_guest_logs.sh >/dev/null 2>&1
CRONTAB
- # Create directories for kernels and images
- {
- echo "set -eux"
- cat $TOP_DIR/tools/xen/functions
- echo "create_directory_for_images"
- echo "create_directory_for_kernels"
- echo "install_conntrack_tools"
- } | $ssh_dom0
}
# install_nova_hypervisor() - Install external components
diff --git a/lib/os_brick b/lib/os_brick
deleted file mode 100644
index d1cca4a..0000000
--- a/lib/os_brick
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/bin/bash
-#
-# lib/os_brick
-# Install **os-brick** python module from source
-
-# Dependencies:
-#
-# - functions
-# - DEST, DATA_DIR must be defined
-
-# stack.sh
-# ---------
-# - install_os_brick
-
-# Save trace setting
-_XTRACE_OS_BRICK=$(set +o | grep xtrace)
-set +o xtrace
-
-
-GITDIR["os-brick"]=$DEST/os-brick
-
-# Install os_brick from git only if requested, otherwise it will be pulled from
-# pip repositories by requirements of projects that need it.
-function install_os_brick {
- if use_library_from_git "os-brick"; then
- git_clone_by_name "os-brick"
- setup_dev_lib "os-brick"
- fi
-}
-
-# Restore xtrace
-$_XTRACE_OS_BRICK
\ No newline at end of file
diff --git a/lib/oslo b/lib/oslo
index 2895503..3ae64c8 100644
--- a/lib/oslo
+++ b/lib/oslo
@@ -6,110 +6,6 @@
#
# We need this to handle the fact that projects would like to use
# pre-released versions of oslo libraries.
-
-# Dependencies:
#
-# - ``functions`` file
-
-# ``stack.sh`` calls the entry points in this order:
-#
-# - install_oslo
-
-# Save trace setting
-_XTRACE_LIB_OSLO=$(set +o | grep xtrace)
-set +o xtrace
-
-
-# Defaults
-# --------
-GITDIR["automaton"]=$DEST/automaton
-GITDIR["castellan"]=$DEST/castellan
-GITDIR["cliff"]=$DEST/cliff
-GITDIR["cursive"]=$DEST/cursive
-GITDIR["debtcollector"]=$DEST/debtcollector
-GITDIR["futurist"]=$DEST/futurist
-GITDIR["os-client-config"]=$DEST/os-client-config
-GITDIR["osc-lib"]=$DEST/osc-lib
-GITDIR["oslo.cache"]=$DEST/oslo.cache
-GITDIR["oslo.concurrency"]=$DEST/oslo.concurrency
-GITDIR["oslo.config"]=$DEST/oslo.config
-GITDIR["oslo.context"]=$DEST/oslo.context
-GITDIR["oslo.db"]=$DEST/oslo.db
-GITDIR["oslo.i18n"]=$DEST/oslo.i18n
-GITDIR["oslo.log"]=$DEST/oslo.log
-GITDIR["oslo.messaging"]=$DEST/oslo.messaging
-GITDIR["oslo.middleware"]=$DEST/oslo.middleware
-GITDIR["oslo.policy"]=$DEST/oslo.policy
-GITDIR["oslo.privsep"]=$DEST/oslo.privsep
-GITDIR["oslo.reports"]=$DEST/oslo.reports
-GITDIR["oslo.rootwrap"]=$DEST/oslo.rootwrap
-GITDIR["oslo.serialization"]=$DEST/oslo.serialization
-GITDIR["oslo.service"]=$DEST/oslo.service
-GITDIR["oslo.utils"]=$DEST/oslo.utils
-GITDIR["oslo.versionedobjects"]=$DEST/oslo.versionedobjects
-GITDIR["oslo.vmware"]=$DEST/oslo.vmware
-GITDIR["osprofiler"]=$DEST/osprofiler
-GITDIR["pycadf"]=$DEST/pycadf
-GITDIR["python-openstacksdk"]=$DEST/python-openstacksdk
-GITDIR["stevedore"]=$DEST/stevedore
-GITDIR["taskflow"]=$DEST/taskflow
-GITDIR["tooz"]=$DEST/tooz
-
-# Support entry points installation of console scripts
-OSLO_BIN_DIR=$(get_python_exec_prefix)
-
-
-# Functions
-# ---------
-
-function _do_install_oslo_lib {
- local name=$1
- if use_library_from_git "$name"; then
- git_clone_by_name "$name"
- setup_dev_lib "$name"
- fi
-}
-
-# install_oslo() - Collect source and prepare
-function install_oslo {
- _do_install_oslo_lib "automaton"
- _do_install_oslo_lib "castellan"
- _do_install_oslo_lib "cliff"
- _do_install_oslo_lib "cursive"
- _do_install_oslo_lib "debtcollector"
- _do_install_oslo_lib "futurist"
- _do_install_oslo_lib "osc-lib"
- _do_install_oslo_lib "os-client-config"
- _do_install_oslo_lib "oslo.cache"
- _do_install_oslo_lib "oslo.concurrency"
- _do_install_oslo_lib "oslo.config"
- _do_install_oslo_lib "oslo.context"
- _do_install_oslo_lib "oslo.db"
- _do_install_oslo_lib "oslo.i18n"
- _do_install_oslo_lib "oslo.log"
- _do_install_oslo_lib "oslo.messaging"
- _do_install_oslo_lib "oslo.middleware"
- _do_install_oslo_lib "oslo.policy"
- _do_install_oslo_lib "oslo.privsep"
- _do_install_oslo_lib "oslo.reports"
- _do_install_oslo_lib "oslo.rootwrap"
- _do_install_oslo_lib "oslo.serialization"
- _do_install_oslo_lib "oslo.service"
- _do_install_oslo_lib "oslo.utils"
- _do_install_oslo_lib "oslo.versionedobjects"
- _do_install_oslo_lib "oslo.vmware"
- _do_install_oslo_lib "osprofiler"
- _do_install_oslo_lib "pycadf"
- _do_install_oslo_lib "python-openstacksdk"
- _do_install_oslo_lib "stevedore"
- _do_install_oslo_lib "taskflow"
- _do_install_oslo_lib "tooz"
-}
-
-# Restore xtrace
-$_XTRACE_LIB_OSLO
-
-# Tell emacs to use shell-script-mode
-## Local variables:
-## mode: shell-script
-## End:
+# Included for compatibility with grenade, remove in Queens
+source $TOP_DIR/lib/libraries
diff --git a/lib/placement b/lib/placement
index ad12824..8adbbde 100644
--- a/lib/placement
+++ b/lib/placement
@@ -149,7 +149,9 @@
function init_placement {
if [ "$PLACEMENT_DB_ENABLED" != False ]; then
recreate_database placement
+ time_start "dbsync"
$NOVA_BIN_DIR/nova-manage --config-file $NOVA_CONF api_db sync
+ time_stop "dbsync"
fi
create_placement_accounts
}
diff --git a/lib/swift b/lib/swift
index 1472e44..e247f15 100644
--- a/lib/swift
+++ b/lib/swift
@@ -542,6 +542,10 @@
else
iniset ${testfile} func_test auth_prefix /identity/v2.0/
fi
+ if is_service_enabled tls-proxy; then
+ iniset ${testfile} func_test cafile ${SSL_BUNDLE_FILE}
+ iniset ${testfile} func_test web_front_end apache2
+ fi
fi
local user_group
diff --git a/stack.sh b/stack.sh
index 5148ae1..326d186 100755
--- a/stack.sh
+++ b/stack.sh
@@ -27,11 +27,37 @@
# Make sure custom grep options don't get in the way
unset GREP_OPTIONS
-# Sanitize language settings to avoid commands bailing out
-# with "unsupported locale setting" errors.
+# NOTE(sdague): why do we explicitly set locale when running stack.sh?
+#
+# Devstack is written in bash, and many functions used throughout
+# devstack process text comming off a command (like the ip command)
+# and do transforms using grep, sed, cut, awk on the strings that are
+# returned. Many of these programs are interationalized, which is
+# great for end users, but means that the strings that devstack
+# functions depend upon might not be there in other locales. We thus
+# need to pin the world to an english basis during the runs.
+#
+# Previously we used the C locale for this, every system has it, and
+# it gives us a stable sort order. It does however mean that we
+# effectively drop unicode support.... boo! :(
+#
+# With python3 being more unicode aware by default, that's not the
+# right option. While there is a C.utf8 locale, some distros are
+# shipping it as C.UTF8 for extra confusingness. And it's support
+# isn't super clear across distros. This is made more challenging when
+# trying to support both out of the box distros, and the gate which
+# uses diskimage builder to build disk images in a different way than
+# the distros do.
+#
+# So... en_US.utf8 it is. That's existed for a very long time. It is a
+# compromise position, but it is the least worse idea at the time of
+# this comment.
+#
+# We also have to unset other variables that might impact LC_ALL
+# taking effect.
unset LANG
unset LANGUAGE
-LC_ALL=C
+LC_ALL=en_US.utf8
export LC_ALL
# Make sure umask is sane
@@ -498,6 +524,11 @@
kill 2>&1 $jobs
fi
+ #Remove timing data file
+ if [ -f "$OSCWRAP_TIMER_FILE" ] ; then
+ rm "$OSCWRAP_TIMER_FILE"
+ fi
+
# Kill the last spinner process
kill_spinner
@@ -561,7 +592,7 @@
# Source project function libraries
source $TOP_DIR/lib/infra
-source $TOP_DIR/lib/oslo
+source $TOP_DIR/lib/libraries
source $TOP_DIR/lib/lvm
source $TOP_DIR/lib/horizon
source $TOP_DIR/lib/keystone
@@ -574,7 +605,6 @@
source $TOP_DIR/lib/ldap
source $TOP_DIR/lib/dstat
source $TOP_DIR/lib/etcd3
-source $TOP_DIR/lib/os_brick
# Extras Source
# --------------
@@ -782,13 +812,17 @@
install_neutron_agent_packages
fi
+if is_service_enabled etcd3; then
+ install_etcd3
+fi
+
# Check Out and Install Source
# ----------------------------
echo_summary "Installing OpenStack project source"
-# Install Oslo libraries
-install_oslo
+# Install additional libraries
+install_libs
# Install uwsgi
install_apache_uwsgi
@@ -806,11 +840,6 @@
install_neutronclient
fi
-# Install shared libraries
-if is_service_enabled cinder nova; then
- install_os_brick
-fi
-
# Setup TLS certs
if is_service_enabled tls-proxy; then
configure_CA
@@ -913,6 +942,10 @@
pip_install_gr python-openstackclient
fi
+# Installs alias for osc so that we can collect timing for all
+# osc commands. Alias dies with stack.sh.
+install_oscwrap
+
if [[ $TRACK_DEPENDS = True ]]; then
$DEST/.venv/bin/pip freeze > $DEST/requires-post-pip
if ! diff -Nru $DEST/requires-pre-pip $DEST/requires-post-pip > $DEST/requires.diff; then
diff --git a/stackrc b/stackrc
index 80990b0..50f7c89 100644
--- a/stackrc
+++ b/stackrc
@@ -341,6 +341,11 @@
GITREPO["python-brick-cinderclient-ext"]=${BRICK_CINDERCLIENT_REPO:-${GIT_BASE}/openstack/python-brick-cinderclient-ext.git}
GITBRANCH["python-brick-cinderclient-ext"]=${BRICK_CINDERCLIENT_BRANCH:-master}
+# python barbican client library
+GITREPO["python-barbicanclient"]=${BARBICANCLIENT_REPO:-${GIT_BASE}/openstack/python-barbicanclient.git}
+GITBRANCH["python-barbicanclient"]=${BARBICANCLIENT_BRANCH:-master}
+GITDIR["python-barbicanclient"]=$DEST/python-barbicanclient
+
# python glance client library
GITREPO["python-glanceclient"]=${GLANCECLIENT_REPO:-${GIT_BASE}/openstack/python-glanceclient.git}
GITBRANCH["python-glanceclient"]=${GLANCECLIENT_BRANCH:-master}
@@ -570,6 +575,10 @@
GITBRANCH["neutron-lib"]=${NEUTRON_LIB_BRANCH:-master}
GITDIR["neutron-lib"]=$DEST/neutron-lib
+# os-traits library for resource provider traits in the placement service
+GITREPO["os-traits"]=${OS_TRAITS_REPO:-${GIT_BASE}/openstack/os-traits.git}
+GITBRANCH["os-traits"]=${OS_TRAITS_BRANCH:-master}
+
##################
#
# TripleO / Heat Agent Components
diff --git a/tests/test_libs_from_pypi.sh b/tests/test_libs_from_pypi.sh
index 608ef6a..5b4ff32 100755
--- a/tests/test_libs_from_pypi.sh
+++ b/tests/test_libs_from_pypi.sh
@@ -39,11 +39,11 @@
ALL_LIBS+=" python-openstackclient osc-lib os-client-config oslo.rootwrap"
ALL_LIBS+=" oslo.i18n oslo.utils python-openstacksdk python-swiftclient"
ALL_LIBS+=" python-neutronclient tooz ceilometermiddleware oslo.policy"
-ALL_LIBS+=" debtcollector os-brick automaton futurist oslo.service"
+ALL_LIBS+=" debtcollector os-brick os-traits automaton futurist oslo.service"
ALL_LIBS+=" oslo.cache oslo.reports osprofiler cursive"
ALL_LIBS+=" keystoneauth ironic-lib neutron-lib oslo.privsep"
ALL_LIBS+=" diskimage-builder os-vif python-brick-cinderclient-ext"
-ALL_LIBS+=" castellan"
+ALL_LIBS+=" castellan python-barbicanclient"
# Generate the above list with
# echo ${!GITREPO[@]}
diff --git a/tools/install_prereqs.sh b/tools/install_prereqs.sh
index a77eae6..da59093 100755
--- a/tools/install_prereqs.sh
+++ b/tools/install_prereqs.sh
@@ -60,14 +60,6 @@
# Install Packages
# ================
-if [[ "${DISTRO}" == "opensuse-42.2" ]]; then
- # temporary workaround until https://bugzilla.suse.com/show_bug.cgi?id=1041161 is fixed
- sudo zypper ar -f http://download.opensuse.org/update/leap/42.2-test/ leap42.2-test-updates
- sudo zypper --non-interactive --gpg-auto-import-keys --no-gpg-checks ref
- sudo zypper --non-interactive install liberasurecode-devel
- sudo zypper rr leap42.2-test-updates
-fi
-
# Install package requirements
PACKAGES=$(get_packages general,$ENABLED_SERVICES)
PACKAGES="$PACKAGES $(get_plugin_packages)"