Build wheel cache for venvs
Building a bunch of virtual envs later is going to be tedious if we do not
pre-cache certain annoying-to-build packages.
* tools/build_wheels.sh: pre-build some wheels for annoying package installs
* list distro package dependencies in files/*/venv
* list packages to pre-build as wheels in files/venv-requirements.txt
* install database Python modules when setting up the database
Change-Id: Idff1ea69a5ca12ba56098e664dbf6924fe6a2e47
diff --git a/clean.sh b/clean.sh
index 50d414c..fea1230 100755
--- a/clean.sh
+++ b/clean.sh
@@ -119,6 +119,10 @@
sudo rm -rf $SCREEN_LOGDIR
fi
+# Clean up venvs
+DIRS_TO_CLEAN="$WHEELHOUSE"
+rm -rf $DIRS_TO_CLEAN
+
# Clean up files
FILES_TO_CLEAN=".localrc.auto docs/files docs/html shocco/ stack-screenrc test*.conf* test.ini*"
diff --git a/files/debs/devlibs b/files/debs/devlibs
new file mode 100644
index 0000000..0446ceb
--- /dev/null
+++ b/files/debs/devlibs
@@ -0,0 +1,7 @@
+libffi-dev # pyOpenSSL
+libmysqlclient-dev # MySQL-python
+libpq-dev # psycopg2
+libssl-dev # pyOpenSSL
+libxml2-dev # lxml
+libxslt1-dev # lxml
+python-dev # pyOpenSSL
diff --git a/files/debs/postgresql b/files/debs/postgresql
deleted file mode 100644
index bf19d39..0000000
--- a/files/debs/postgresql
+++ /dev/null
@@ -1 +0,0 @@
-python-psycopg2
diff --git a/files/rpms-suse/devlibs b/files/rpms-suse/devlibs
new file mode 100644
index 0000000..11722ad
--- /dev/null
+++ b/files/rpms-suse/devlibs
@@ -0,0 +1,6 @@
+libffi-devel # pyOpenSSL
+libopenssl-devel # pyOpenSSL
+libxml2-devel # lxml
+libxslt1-dev # lxml
+postgresql-devel # psycopg2
+python-devel # pyOpenSSL
diff --git a/files/rpms-suse/postgresql b/files/rpms-suse/postgresql
deleted file mode 100644
index bf19d39..0000000
--- a/files/rpms-suse/postgresql
+++ /dev/null
@@ -1 +0,0 @@
-python-psycopg2
diff --git a/files/rpms/devlibs b/files/rpms/devlibs
new file mode 100644
index 0000000..6010c9f
--- /dev/null
+++ b/files/rpms/devlibs
@@ -0,0 +1,8 @@
+libffi-devel # pyOpenSSL
+libxml2-devel # lxml
+libxslt1-devel # lxml
+mariadb-devel # MySQL-python f20,f21,rhel7
+mysql-devel # MySQL-python rhel6
+openssl-devel # pyOpenSSL
+postgresql-devel # psycopg2
+python-devel # pyOpenSSL
diff --git a/files/rpms/postgresql b/files/rpms/postgresql
deleted file mode 100644
index bf19d39..0000000
--- a/files/rpms/postgresql
+++ /dev/null
@@ -1 +0,0 @@
-python-psycopg2
diff --git a/files/venv-requirements.txt b/files/venv-requirements.txt
new file mode 100644
index 0000000..3c50061
--- /dev/null
+++ b/files/venv-requirements.txt
@@ -0,0 +1,10 @@
+lxml
+MySQL-python
+netifaces
+numpy
+posix-ipc
+psycopg2
+pycrypto
+pyOpenSSL
+PyYAML
+xattr
diff --git a/lib/databases/mysql b/lib/databases/mysql
index c8ceec2..70073c4 100644
--- a/lib/databases/mysql
+++ b/lib/databases/mysql
@@ -151,6 +151,9 @@
else
exit_distro_not_supported "mysql installation"
fi
+
+ # Install Python client module
+ pip_install MySQL-python
}
function database_connection_url_mysql {
diff --git a/lib/databases/postgresql b/lib/databases/postgresql
index 317e0eb..dfda9ad 100644
--- a/lib/databases/postgresql
+++ b/lib/databases/postgresql
@@ -100,6 +100,9 @@
else
exit_distro_not_supported "postgresql installation"
fi
+
+ # Install Python client module
+ pip_install MySQL-python psycopg2
}
function database_connection_url_postgresql {
diff --git a/stack.sh b/stack.sh
index 753135b..49288a7 100755
--- a/stack.sh
+++ b/stack.sh
@@ -671,6 +671,21 @@
source $TOP_DIR/tools/fixup_stuff.sh
+# Virtual Environment
+# -------------------
+
+# Temporary hack for testing
+# This belongs in d-g functions.sh setup_host() or devstack-vm-gate.sh
+if [[ -d /var/cache/pip ]]; then
+ sudo chown -R $STACK_USER:$STACK_USER /var/cache/pip
+fi
+
+# Pre-build some problematic wheels
+if [[ ! -d ${WHEELHOUSE:-} ]]; then
+ source tools/build_wheels.sh
+fi
+
+
# Extras Pre-install
# ------------------
diff --git a/stackrc b/stackrc
index 386c5d5..ee601a9 100644
--- a/stackrc
+++ b/stackrc
@@ -112,6 +112,11 @@
source $RC_DIR/.localrc.auto
fi
+# Configure wheel cache location
+export WHEELHOUSE=${WHEELHOUSE:-$DEST/.wheelhouse}
+export PIP_WHEEL_DIR=${PIP_WHEEL_DIR:-$WHEELHOUSE}
+export PIP_FIND_LINKS=${PIP_FIND_LINKS:-file://$WHEELHOUSE}
+
# This can be used to turn database query logging on and off
# (currently only implemented for MySQL backend)
DATABASE_QUERY_LOGGING=$(trueorfalse True DATABASE_QUERY_LOGGING)
diff --git a/tools/build_wheels.sh b/tools/build_wheels.sh
new file mode 100755
index 0000000..c87b6da
--- /dev/null
+++ b/tools/build_wheels.sh
@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+#
+# **tools/build_wheels.sh** - Build a cache of Python wheels
+#
+# build_wheels.sh [package [...]]
+#
+# System package prerequisites listed in files/*/devlibs will be installed
+#
+# Builds wheels for all virtual env requirements listed in
+# ``venv-requirements.txt`` plus any supplied on the command line.
+#
+# Assumes ``tools/install_pip.sh`` has been run and a suitable pip/setuptools is available.
+
+# If TOP_DIR is set we're being sourced rather than running stand-alone
+# or in a sub-shell
+if [[ -z "$TOP_DIR" ]]; then
+
+ set -o errexit
+ set -o nounset
+
+ # Keep track of the devstack directory
+ TOP_DIR=$(cd $(dirname "$0")/.. && pwd)
+ FILES=$TOP_DIR/files
+
+ # Import common functions
+ source $TOP_DIR/functions
+
+ GetDistro
+
+ source $TOP_DIR/stackrc
+
+ trap err_trap ERR
+
+fi
+
+# Get additional packages to build
+MORE_PACKAGES="$@"
+
+# Set a fall-back default, assume that since this script is being called
+# package builds are to happen even if WHEELHOUSE is not configured
+export WHEELHOUSE=${WHEELHOUSE:-.wheelhouse}
+
+# Exit on any errors so that errors don't compound
+function err_trap {
+ local r=$?
+ set +o xtrace
+
+ rm -rf $TMP_VENV_PATH
+
+ exit $r
+}
+
+# Get system prereqs
+install_package $(get_packages devlibs)
+
+# Get a modern ``virtualenv``
+pip_install virtualenv
+
+# Prepare the workspace
+TMP_VENV_PATH=$(mktemp -d tmp-venv-XXXX)
+virtualenv $TMP_VENV_PATH
+
+# Install modern pip and wheel
+$TMP_VENV_PATH/bin/pip install -U pip wheel
+
+# VENV_PACKAGES is a list of packages we want to pre-install
+VENV_PACKAGE_FILE=$FILES/venv-requirements.txt
+if [[ -r $VENV_PACKAGE_FILE ]]; then
+ VENV_PACKAGES=$(grep -v '^#' $VENV_PACKAGE_FILE)
+fi
+
+for pkg in ${VENV_PACKAGES,/ } ${MORE_PACKAGES}; do
+ $TMP_VENV_PATH/bin/pip wheel $pkg
+done
+
+# Clean up wheel workspace
+rm -rf $TMP_VENV_PATH