Add option to install everything in global venvs
Since we are python3 only for openstack we create a single python3
virtualenv to install all the packages into. This gives us the benefits
of installing into a virtualenv while still ensuring coinstallability.
This is a major change and will likely break many things.
There are several reasons for this. The change that started this effort
was pip stopped uninstalling packages which used distutils to generate
their package installation. Many distro packages do this which meant
that pip installed packages and distro packages could not coexist in the
global install space. More recently git has made pip installing repos as
root more difficult due to file ownership concerns.
Currently the switch to the global venv is optional, but if we go down
this path we should very quickly remove the old global installation
method as it has only caused us problems.
Major hurdles we have to get over are convincing rootwrap to trust
binaries in the virtualenvs (so you'll notice we update rootwrap
configs).
Some distros still have issues, keep them using the old setup for now.
Depends-On: https://review.opendev.org/c/openstack/grenade/+/880266
Co-Authored-By: Dr. Jens Harbott <frickler@offenerstapel.de>
Change-Id: If9bc7ba45522189d03f19b86cb681bb150ee2f25
diff --git a/.zuul.yaml b/.zuul.yaml
index a7be671..803db3a 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -700,6 +700,9 @@
# TODO(kopecmartin) n-v until the following is resolved:
# https://bugs.launchpad.net/neutron/+bug/1979047
voting: false
+ vars:
+ devstack_localrc:
+ GLOBAL_VENV: false
- job:
name: devstack-platform-debian-bullseye
@@ -709,6 +712,9 @@
timeout: 9000
vars:
configure_swap_size: 4096
+ devstack_localrc:
+ # TODO(frickler): drop this once wheel build is fixed
+ MYSQL_GATHER_PERFORMANCE: false
- job:
name: devstack-platform-rocky-blue-onyx
@@ -718,6 +724,8 @@
timeout: 9000
vars:
configure_swap_size: 4096
+ devstack_localrc:
+ GLOBAL_VENV: false
- job:
name: devstack-platform-ubuntu-focal
diff --git a/files/apache-horizon.template b/files/apache-horizon.template
index efcfc03..da7a7d2 100644
--- a/files/apache-horizon.template
+++ b/files/apache-horizon.template
@@ -39,4 +39,5 @@
CustomLog /var/log/%APACHE_NAME%/horizon_access.log combined
</VirtualHost>
+%WSGIPYTHONHOME%
WSGISocketPrefix /var/run/%APACHE_NAME%
diff --git a/functions-common b/functions-common
index 5e1aa34..f752271 100644
--- a/functions-common
+++ b/functions-common
@@ -1522,6 +1522,7 @@
mkdir -p $SYSTEMD_DIR
iniset -sudo $unitfile "Unit" "Description" "Devstack $service"
+ iniset -sudo $unitfile "Service" "Environment" "\"PATH=$PATH\""
iniset -sudo $unitfile "Service" "User" "$user"
iniset -sudo $unitfile "Service" "ExecStart" "$command"
iniset -sudo $unitfile "Service" "KillMode" "process"
@@ -1549,6 +1550,7 @@
mkdir -p $SYSTEMD_DIR
iniset -sudo $unitfile "Unit" "Description" "Devstack $service"
+ iniset -sudo $unitfile "Service" "Environment" "\"PATH=$PATH\""
iniset -sudo $unitfile "Service" "SyslogIdentifier" "$service"
iniset -sudo $unitfile "Service" "User" "$user"
iniset -sudo $unitfile "Service" "ExecStart" "$command"
@@ -1614,6 +1616,9 @@
fi
local env_vars="$5"
if [[ "$command" =~ "uwsgi" ]] ; then
+ if [[ "$GLOBAL_VENV" == "True" ]] ; then
+ cmd="$cmd --venv $DEVSTACK_VENV"
+ fi
write_uwsgi_user_unit_file $systemd_service "$cmd" "$group" "$user" "$env_vars"
else
write_user_unit_file $systemd_service "$cmd" "$group" "$user" "$env_vars"
diff --git a/inc/python b/inc/python
index a24f4e9..cc6e01f 100644
--- a/inc/python
+++ b/inc/python
@@ -32,6 +32,23 @@
# Python Functions
# ================
+# Setup the global devstack virtualenvs and the associated environment
+# updates.
+function setup_devstack_virtualenv {
+ # We run devstack out of a global virtualenv.
+ if [[ ! -d $DEVSTACK_VENV ]] ; then
+ # Using system site packages to enable nova to use libguestfs.
+ # This package is currently installed via the distro and not
+ # available on pypi.
+ python$PYTHON3_VERSION -m venv --system-site-packages $DEVSTACK_VENV
+ pip_install -U pip
+ fi
+ if [[ ":$PATH:" != *":$DEVSTACK_VENV/bin:"* ]] ; then
+ export PATH="$DEVSTACK_VENV/bin:$PATH"
+ export PYTHON="$DEVSTACK_VENV/bin/python3"
+ fi
+}
+
# Get the path to the pip command.
# get_pip_command
function get_pip_command {
@@ -60,8 +77,11 @@
fi
$xtrace
- local PYTHON_PATH=/usr/local/bin
- echo $PYTHON_PATH
+ if [[ "$GLOBAL_VENV" == "True" ]] ; then
+ echo "$DEVSTACK_VENV/bin"
+ else
+ echo "/usr/local/bin"
+ fi
}
# Wrapper for ``pip install`` that only installs versions of libraries
@@ -166,6 +186,14 @@
if [[ -n ${PIP_VIRTUAL_ENV:=} && -d ${PIP_VIRTUAL_ENV} ]]; then
local cmd_pip=$PIP_VIRTUAL_ENV/bin/pip
local sudo_pip="env"
+ elif [[ "${GLOBAL_VENV}" == "True" && -d ${DEVSTACK_VENV} ]] ; then
+ # We have to check that the DEVSTACK_VENV exists because early
+ # devstack boostrapping needs to operate in a system context
+ # too bootstrap pip. Once pip is bootstrapped we create the
+ # global venv and can start to use it.
+ local cmd_pip=$DEVSTACK_VENV/bin/pip
+ local sudo_pip="env"
+ echo "Using python $PYTHON3_VERSION to install $package_dir"
else
local cmd_pip="python$PYTHON3_VERSION -m pip"
# See
@@ -439,7 +467,7 @@
pip_install $flags "$project_dir$extras"
# ensure that further actions can do things like setup.py sdist
- if [[ "$flags" == "-e" ]]; then
+ if [[ "$flags" == "-e" && "$GLOBAL_VENV" == "False" ]]; then
safe_chown -R $STACK_USER $1/*.egg-info
fi
}
diff --git a/inc/rootwrap b/inc/rootwrap
index 2a6e4b6..4c65440 100644
--- a/inc/rootwrap
+++ b/inc/rootwrap
@@ -60,6 +60,11 @@
sudo install -o root -g root -m 644 $rootwrap_conf_src_dir/rootwrap.conf /etc/${project}/rootwrap.conf
sudo sed -e "s:^filters_path=.*$:filters_path=/etc/${project}/rootwrap.d:" -i /etc/${project}/rootwrap.conf
+ # Rely on $PATH set by devstack to determine what is safe to execute
+ # by rootwrap rather than use explicit whitelist of paths in
+ # rootwrap.conf
+ sudo sed -e 's/^exec_dirs=.*/#&/' -i /etc/${project}/rootwrap.conf
+
# Set up the rootwrap sudoers
local tempfile
tempfile=$(mktemp)
diff --git a/lib/glance b/lib/glance
index 430d94d..e64f000 100644
--- a/lib/glance
+++ b/lib/glance
@@ -47,6 +47,9 @@
# from CINDER_ENABLED_BACKENDS
GLANCE_CINDER_DEFAULT_BACKEND=${GLANCE_CINDER_DEFAULT_BACKEND:-lvmdriver-1}
GLANCE_STORE_ROOTWRAP_BASE_DIR=/usr/local/etc/glance
+if [[ "$GLOBAL_VENV" == "True" ]] ; then
+ GLANCE_STORE_ROOTWRAP_BASE_DIR=${DEVSTACK_VENV}/etc/glance
+fi
# When Cinder is used as a glance store, you can optionally configure cinder to
# optimize bootable volume creation by allowing volumes to be cloned directly
# in the backend instead of transferring data via Glance. To use this feature,
diff --git a/lib/horizon b/lib/horizon
index f76f9e5..611329d 100644
--- a/lib/horizon
+++ b/lib/horizon
@@ -115,6 +115,11 @@
local horizon_conf
horizon_conf=$(apache_site_config_for horizon)
+ local wsgi_venv_config=""
+ if [[ "$GLOBAL_VENV" == "True" ]] ; then
+ wsgi_venv_config="WSGIPythonHome $DEVSTACK_VENV"
+ fi
+
# Configure apache to run horizon
# Set up the django horizon application to serve via apache/wsgi
sudo sh -c "sed -e \"
@@ -124,6 +129,7 @@
s,%APACHE_NAME%,$APACHE_NAME,g;
s,%DEST%,$DEST,g;
s,%WEBROOT%,$HORIZON_APACHE_ROOT,g;
+ s,%WSGIPYTHONHOME%,$wsgi_venv_config,g;
\" $FILES/apache-horizon.template >$horizon_conf"
if is_ubuntu; then
diff --git a/lib/tls b/lib/tls
index a1e162d..d35e9e2 100644
--- a/lib/tls
+++ b/lib/tls
@@ -364,8 +364,11 @@
function fix_system_ca_bundle_path {
if is_service_enabled tls-proxy; then
local capath
- capath=$(python3 -c $'try:\n from requests import certs\n print (certs.where())\nexcept ImportError: pass')
-
+ if [[ "$GLOBAL_VENV" == "True" ]] ; then
+ capath=$($DEVSTACK_VENV/bin/python3 -c $'try:\n from requests import certs\n print (certs.where())\nexcept ImportError: pass')
+ else
+ capath=$(python3 -c $'try:\n from requests import certs\n print (certs.where())\nexcept ImportError: pass')
+ fi
if [[ ! $capath == "" && ! $capath =~ ^/etc/.* && ! -L $capath ]]; then
if is_fedora; then
sudo rm -f $capath
diff --git a/stack.sh b/stack.sh
index ad88eab..c8f7c9d 100755
--- a/stack.sh
+++ b/stack.sh
@@ -1,5 +1,6 @@
#!/usr/bin/env bash
+
# ``stack.sh`` is an opinionated OpenStack developer installation. It
# installs and configures various combinations of **Cinder**, **Glance**,
# **Horizon**, **Keystone**, **Nova**, **Neutron**, and **Swift**
@@ -824,6 +825,17 @@
source $TOP_DIR/tools/fixup_stuff.sh
fixup_all
+if [[ "$GLOBAL_VENV" == "True" ]] ; then
+ # TODO(frickler): find a better solution for this
+ sudo ln -sf /opt/stack/data/venv/bin/privsep-helper /usr/local/bin
+ sudo ln -sf /opt/stack/data/venv/bin/cinder-rtstool /usr/local/bin
+ sudo ln -sf /opt/stack/data/venv/bin/openstack /usr/local/bin
+ sudo ln -sf /opt/stack/data/venv/bin/tox /usr/local/bin
+ sudo ln -sf /opt/stack/data/venv/bin/nova-manage /usr/local/bin
+
+ setup_devstack_virtualenv
+fi
+
# Install subunit for the subunit output stream
pip_install -U os-testr
diff --git a/stackrc b/stackrc
index dcc0ce4..0d1880c 100644
--- a/stackrc
+++ b/stackrc
@@ -183,6 +183,14 @@
# each services ${SERVICE}_ENFORCE_SCOPE variables
ENFORCE_SCOPE=$(trueorfalse False ENFORCE_SCOPE)
+# Devstack supports the use of a global virtualenv. These variables enable
+# and disable this functionality as well as set the path to the virtualenv.
+# Note that the DATA_DIR is selected because grenade testing uses a shared
+# DATA_DIR but different DEST dirs and we don't want two sets of venvs,
+# instead we want one global set.
+GLOBAL_VENV=$(trueorfalse True GLOBAL_VENV)
+DEVSTACK_VENV=${DEVSTACK_VENV:-$DATA_DIR/venv}
+
# Enable use of Python virtual environments. Individual project use of
# venvs are controlled by the PROJECT_VENV array; every project with
# an entry in the array will be installed into the named venv.
diff --git a/tools/install_prereqs.sh b/tools/install_prereqs.sh
index f2d57c8..bb470b2 100755
--- a/tools/install_prereqs.sh
+++ b/tools/install_prereqs.sh
@@ -79,6 +79,8 @@
fi
fi
+# TODO(clarkb) remove these once we are switched to global venv by default
+export PYTHON=$(which python${PYTHON3_VERSION} 2>/dev/null || which python3 2>/dev/null)
# Mark end of run
# ---------------
diff --git a/tools/memory_tracker.sh b/tools/memory_tracker.sh
index 6c36534..2f404c2 100755
--- a/tools/memory_tracker.sh
+++ b/tools/memory_tracker.sh
@@ -14,7 +14,12 @@
set -o errexit
-PYTHON=${PYTHON:-python3}
+# TODO(frickler): make this use stackrc variables
+if [ -x /opt/stack/data/venv/bin/python ]; then
+ PYTHON=/opt/stack/data/venv/bin/python
+else
+ PYTHON=${PYTHON:-python3}
+fi
# time to sleep between checks
SLEEP_TIME=20