Merge "Automatically set LIBS_FROM_GIT based on required projects"
diff --git a/inc/python b/inc/python
index ec4233b..96be107 100644
--- a/inc/python
+++ b/inc/python
@@ -429,22 +429,6 @@
[[ -n $(pip list --format=columns 2>/dev/null | awk "/^$safe_name/ {print \$3}") ]]
}
-# check that everything that's in LIBS_FROM_GIT was actually installed
-# correctly, this helps double check issues with library fat fingering.
-function check_libs_from_git {
- local lib=""
- local not_installed=""
- for lib in $(echo ${LIBS_FROM_GIT} | tr "," " "); do
- if ! lib_installed_from_git "$lib"; then
- not_installed+=" $lib"
- fi
- done
- # if anything is not installed, say what it is.
- if [[ -n "$not_installed" ]]; then
- die $LINENO "The following LIBS_FROM_GIT were not installed correct: $not_installed"
- fi
-}
-
# setup a library by name. If we are trying to use the library from
# git, we'll do a git based install, otherwise we'll punt and the
# library should be installed by a requirements pull from another
@@ -555,6 +539,13 @@
setup_package $project_dir "$flags" $extras
+ # If this project is in LIBS_FROM_GIT, verify it was actually installed
+ # correctly. This helps catch errors caused by constraints mismatches.
+ if use_library_from_git "$project_dir"; then
+ if ! lib_installed_from_git "$project_dir"; then
+ die $LINENO "The following LIBS_FROM_GIT was not installed correctly: $project_dir"
+ fi
+ fi
}
# ``pip install -e`` the package, which processes the dependencies
diff --git a/roles/write-devstack-local-conf/README.rst b/roles/write-devstack-local-conf/README.rst
index 73f9f0d..bfce9c9 100644
--- a/roles/write-devstack-local-conf/README.rst
+++ b/roles/write-devstack-local-conf/README.rst
@@ -20,6 +20,14 @@
bash shell variables, and will be ordered so that variables used by
later entries appear first.
+ As a special case, the variable ``LIBS_FROM_GIT`` will be
+ constructed automatically from the projects which appear in the
+ ``required-projects`` list defined by the job. To instruct
+ devstack to install a library from source rather than pypi, simply
+ add that library to the job's ``required-projects`` list. To
+ override the automatically-generated value, set ``LIBS_FROM_GIT``
+ in ``devstack_localrc`` to the desired value.
+
.. zuul:rolevar:: devstack_local_conf
:type: dict
@@ -75,3 +83,7 @@
A dictionary mapping a plugin name to a git repo location. If the
location is a non-empty string, then an ``enable_plugin`` line will
be emmitted for the plugin name.
+
+ If a plugin declares a dependency on another plugin (via
+ ``plugin_requires`` in the plugin's settings file), this role will
+ automatically emit ``enable_plugin`` lines in the correct order.
diff --git a/roles/write-devstack-local-conf/library/devstack_local_conf.py b/roles/write-devstack-local-conf/library/devstack_local_conf.py
index 746f54f..9728fef 100644
--- a/roles/write-devstack-local-conf/library/devstack_local_conf.py
+++ b/roles/write-devstack-local-conf/library/devstack_local_conf.py
@@ -207,17 +207,17 @@
class LocalConf(object):
def __init__(self, localrc, localconf, base_services, services, plugins,
- base_dir):
+ base_dir, projects):
self.localrc = []
self.meta_sections = {}
self.plugin_deps = {}
self.base_dir = base_dir
+ self.projects = projects
if plugins:
self.handle_plugins(plugins)
if services or base_services:
self.handle_services(base_services, services or {})
- if localrc:
- self.handle_localrc(localrc)
+ self.handle_localrc(localrc)
if localconf:
self.handle_localconf(localconf)
@@ -241,9 +241,22 @@
self.localrc.append('enable_service {}'.format(k))
def handle_localrc(self, localrc):
- vg = VarGraph(localrc)
- for k, v in vg.getVars():
- self.localrc.append('{}={}'.format(k, v))
+ lfg = False
+ if localrc:
+ vg = VarGraph(localrc)
+ for k, v in vg.getVars():
+ self.localrc.append('{}={}'.format(k, v))
+ if k == 'LIBS_FROM_GIT':
+ lfg = True
+
+ if not lfg and self.projects:
+ required_projects = []
+ for project_name, project_info in self.projects.items():
+ if project_info.get('required'):
+ required_projects.append(project_info['short_name'])
+ if required_projects:
+ self.localrc.append('LIBS_FROM_GIT={}'.format(
+ ','.join(required_projects)))
def handle_localconf(self, localconf):
for phase, phase_data in localconf.items():
@@ -277,6 +290,7 @@
local_conf=dict(type='dict'),
base_dir=dict(type='path'),
path=dict(type='str'),
+ projects=dict(type='dict'),
)
)
@@ -286,7 +300,8 @@
p.get('base_services'),
p.get('services'),
p.get('plugins'),
- p.get('base_dir'))
+ p.get('base_dir'),
+ p.get('projects'))
lc.write(p['path'])
module.exit_json()
diff --git a/roles/write-devstack-local-conf/library/test.py b/roles/write-devstack-local-conf/library/test.py
index 843ca6e..7ccb68f 100644
--- a/roles/write-devstack-local-conf/library/test.py
+++ b/roles/write-devstack-local-conf/library/test.py
@@ -56,7 +56,8 @@
p.get('base_services'),
p.get('services'),
p.get('plugins'),
- p.get('base_dir'))
+ p.get('base_dir'),
+ p.get('projects'))
lc.write(p['path'])
plugins = []
@@ -66,6 +67,7 @@
plugins.append(line.split()[1])
self.assertEqual(['bar', 'baz', 'foo'], plugins)
+
def test_plugin_deps(self):
"Test that plugins with dependencies work"
os.makedirs(os.path.join(self.tmpdir, 'foo-plugin', 'devstack'))
@@ -101,20 +103,80 @@
plugins=plugins,
base_dir=self.tmpdir,
path=os.path.join(self.tmpdir, 'test.local.conf'))
+
+ def test_libs_from_git(self):
+ "Test that LIBS_FROM_GIT is auto-generated"
+ projects = {
+ 'git.openstack.org/openstack/nova': {
+ 'required': True,
+ 'short_name': 'nova',
+ },
+ 'git.openstack.org/openstack/oslo.messaging': {
+ 'required': True,
+ 'short_name': 'oslo.messaging',
+ },
+ 'git.openstack.org/openstack/devstack-plugin': {
+ 'required': False,
+ 'short_name': 'devstack-plugin',
+ },
+ }
+ p = dict(base_services=[],
+ base_dir='./test',
+ path=os.path.join(self.tmpdir, 'test.local.conf'),
+ projects=projects)
lc = LocalConf(p.get('localrc'),
p.get('local_conf'),
p.get('base_services'),
p.get('services'),
p.get('plugins'),
- p.get('base_dir'))
+ p.get('base_dir'),
+ p.get('projects'))
lc.write(p['path'])
- plugins = []
+ lfg = None
with open(p['path']) as f:
for line in f:
- if line.startswith('enable_plugin'):
- plugins.append(line.split()[1])
- self.assertEqual(['foo', 'bar'], plugins)
+ if line.startswith('LIBS_FROM_GIT'):
+ lfg = line.strip().split('=')[1]
+ self.assertEqual('nova,oslo.messaging', lfg)
+
+ def test_overridelibs_from_git(self):
+ "Test that LIBS_FROM_GIT can be overridden"
+ localrc = {'LIBS_FROM_GIT': 'oslo.db'}
+ projects = {
+ 'git.openstack.org/openstack/nova': {
+ 'required': True,
+ 'short_name': 'nova',
+ },
+ 'git.openstack.org/openstack/oslo.messaging': {
+ 'required': True,
+ 'short_name': 'oslo.messaging',
+ },
+ 'git.openstack.org/openstack/devstack-plugin': {
+ 'required': False,
+ 'short_name': 'devstack-plugin',
+ },
+ }
+ p = dict(localrc=localrc,
+ base_services=[],
+ base_dir='./test',
+ path=os.path.join(self.tmpdir, 'test.local.conf'),
+ projects=projects)
+ lc = LocalConf(p.get('localrc'),
+ p.get('local_conf'),
+ p.get('base_services'),
+ p.get('services'),
+ p.get('plugins'),
+ p.get('base_dir'),
+ p.get('projects'))
+ lc.write(p['path'])
+
+ lfg = None
+ with open(p['path']) as f:
+ for line in f:
+ if line.startswith('LIBS_FROM_GIT'):
+ lfg = line.strip().split('=')[1]
+ self.assertEqual('oslo.db', lfg)
def test_plugin_circular_deps(self):
"Test that plugins with circular dependencies fail"
diff --git a/roles/write-devstack-local-conf/tasks/main.yaml b/roles/write-devstack-local-conf/tasks/main.yaml
index 2a9f898..a294cae 100644
--- a/roles/write-devstack-local-conf/tasks/main.yaml
+++ b/roles/write-devstack-local-conf/tasks/main.yaml
@@ -9,3 +9,4 @@
localrc: "{{ devstack_localrc|default(omit) }}"
local_conf: "{{ devstack_local_conf|default(omit) }}"
base_dir: "{{ devstack_base_dir|default(omit) }}"
+ projects: "{{ zuul.projects }}"
diff --git a/stack.sh b/stack.sh
index ce6e6fe..6899fa0 100755
--- a/stack.sh
+++ b/stack.sh
@@ -1398,11 +1398,6 @@
# Check the status of running services
service_check
-# ensure that all the libraries we think we installed from git,
-# actually were.
-check_libs_from_git
-
-
# Configure nova cellsv2
# ----------------------