| # Copyright (C) 2017 Red Hat, Inc. |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or |
| # implied. |
| # |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| import os |
| import re |
| |
| |
| class DependencyGraph(object): |
| # This is based on the JobGraph from Zuul. |
| |
| def __init__(self): |
| self._names = set() |
| self._dependencies = {} # dependent_name -> set(parent_names) |
| |
| def add(self, name, dependencies): |
| # Append the dependency information |
| self._dependencies.setdefault(name, set()) |
| try: |
| for dependency in dependencies: |
| # Make sure a circular dependency is never created |
| ancestors = self._getParentNamesRecursively( |
| dependency, soft=True) |
| ancestors.add(dependency) |
| if name in ancestors: |
| raise Exception("Dependency cycle detected in {}". |
| format(name)) |
| self._dependencies[name].add(dependency) |
| except Exception: |
| del self._dependencies[name] |
| raise |
| |
| def getDependenciesRecursively(self, parent): |
| dependencies = [] |
| |
| current_dependencies = self._dependencies[parent] |
| for current in current_dependencies: |
| if current not in dependencies: |
| dependencies.append(current) |
| for dep in self.getDependenciesRecursively(current): |
| if dep not in dependencies: |
| dependencies.append(dep) |
| return dependencies |
| |
| def _getParentNamesRecursively(self, dependent, soft=False): |
| all_parent_items = set() |
| items_to_iterate = set([dependent]) |
| while len(items_to_iterate) > 0: |
| current_item = items_to_iterate.pop() |
| current_parent_items = self._dependencies.get(current_item) |
| if current_parent_items is None: |
| if soft: |
| current_parent_items = set() |
| else: |
| raise Exception("Dependent item {} not found: ".format( |
| dependent)) |
| new_parent_items = current_parent_items - all_parent_items |
| items_to_iterate |= new_parent_items |
| all_parent_items |= new_parent_items |
| return all_parent_items |
| |
| |
| class VarGraph(DependencyGraph): |
| def __init__(self, vars): |
| super(VarGraph, self).__init__() |
| self.vars = {} |
| self._varnames = set() |
| for k, v in vars.items(): |
| self._varnames.add(k) |
| for k, v in vars.items(): |
| self._addVar(k, str(v)) |
| |
| bash_var_re = re.compile(r'\$\{?(\w+)') |
| def getDependencies(self, value): |
| return self.bash_var_re.findall(value) |
| |
| def _addVar(self, key, value): |
| if key in self.vars: |
| raise Exception("Variable {} already added".format(key)) |
| self.vars[key] = value |
| # Append the dependency information |
| dependencies = set() |
| for dependency in self.getDependencies(value): |
| if dependency == key: |
| # A variable is allowed to reference itself; no |
| # dependency link needed in that case. |
| continue |
| if dependency not in self._varnames: |
| # It's not necessary to create a link for an |
| # external variable. |
| continue |
| dependencies.add(dependency) |
| try: |
| self.add(key, dependencies) |
| except Exception: |
| del self.vars[key] |
| raise |
| |
| def getVars(self): |
| ret = [] |
| keys = sorted(self.vars.keys()) |
| seen = set() |
| for key in keys: |
| dependencies = self.getDependenciesRecursively(key) |
| for var in dependencies + [key]: |
| if var not in seen: |
| ret.append((var, self.vars[var])) |
| seen.add(var) |
| return ret |
| |
| |
| class PluginGraph(DependencyGraph): |
| def __init__(self, base_dir, plugins): |
| super(PluginGraph, self).__init__() |
| # The dependency trees expressed by all the plugins we found |
| # (which may be more than those the job is using). |
| self._plugin_dependencies = {} |
| self.loadPluginNames(base_dir) |
| |
| self.plugins = {} |
| self._pluginnames = set() |
| for k, v in plugins.items(): |
| self._pluginnames.add(k) |
| for k, v in plugins.items(): |
| self._addPlugin(k, str(v)) |
| |
| def loadPluginNames(self, base_dir): |
| if base_dir is None: |
| return |
| git_roots = [] |
| for root, dirs, files in os.walk(base_dir): |
| if '.git' not in dirs: |
| continue |
| # Don't go deeper than git roots |
| dirs[:] = [] |
| git_roots.append(root) |
| for root in git_roots: |
| devstack = os.path.join(root, 'devstack') |
| if not (os.path.exists(devstack) and os.path.isdir(devstack)): |
| continue |
| settings = os.path.join(devstack, 'settings') |
| if not (os.path.exists(settings) and os.path.isfile(settings)): |
| continue |
| self.loadDevstackPluginInfo(settings) |
| |
| define_re = re.compile(r'^define_plugin\s+(\S+).*') |
| require_re = re.compile(r'^plugin_requires\s+(\S+)\s+(\S+).*') |
| def loadDevstackPluginInfo(self, fn): |
| name = None |
| reqs = set() |
| with open(fn) as f: |
| for line in f: |
| m = self.define_re.match(line) |
| if m: |
| name = m.group(1) |
| m = self.require_re.match(line) |
| if m: |
| if name == m.group(1): |
| reqs.add(m.group(2)) |
| if name and reqs: |
| self._plugin_dependencies[name] = reqs |
| |
| def getDependencies(self, value): |
| return self._plugin_dependencies.get(value, []) |
| |
| def _addPlugin(self, key, value): |
| if key in self.plugins: |
| raise Exception("Plugin {} already added".format(key)) |
| self.plugins[key] = value |
| # Append the dependency information |
| dependencies = set() |
| for dependency in self.getDependencies(key): |
| if dependency == key: |
| continue |
| dependencies.add(dependency) |
| try: |
| self.add(key, dependencies) |
| except Exception: |
| del self.plugins[key] |
| raise |
| |
| def getPlugins(self): |
| ret = [] |
| keys = sorted(self.plugins.keys()) |
| seen = set() |
| for key in keys: |
| dependencies = self.getDependenciesRecursively(key) |
| for plugin in dependencies + [key]: |
| if plugin not in seen: |
| ret.append((plugin, self.plugins[plugin])) |
| seen.add(plugin) |
| return ret |
| |
| |
| class LocalConf(object): |
| |
| def __init__(self, localrc, localconf, base_services, services, plugins, |
| base_dir, projects, project, tempest_plugins): |
| self.localrc = [] |
| self.warnings = [] |
| self.meta_sections = {} |
| self.plugin_deps = {} |
| self.base_dir = base_dir |
| self.projects = projects |
| self.project = project |
| self.tempest_plugins = tempest_plugins |
| if services or base_services: |
| self.handle_services(base_services, services or {}) |
| self.handle_localrc(localrc) |
| # Plugins must be the last items in localrc, otherwise |
| # the configuration lines which follows them in the file are |
| # not applied to the plugins (for example, the value of DEST.) |
| if plugins: |
| self.handle_plugins(plugins) |
| if localconf: |
| self.handle_localconf(localconf) |
| |
| def handle_plugins(self, plugins): |
| pg = PluginGraph(self.base_dir, plugins) |
| for k, v in pg.getPlugins(): |
| if v: |
| self.localrc.append('enable_plugin {} {}'.format(k, v)) |
| |
| def handle_services(self, base_services, services): |
| enable_base_services = services.pop('base', True) |
| if enable_base_services and base_services: |
| self.localrc.append('ENABLED_SERVICES={}'.format( |
| ",".join(base_services))) |
| else: |
| self.localrc.append('disable_all_services') |
| for k, v in services.items(): |
| if v is False: |
| self.localrc.append('disable_service {}'.format(k)) |
| elif v is True: |
| self.localrc.append('enable_service {}'.format(k)) |
| |
| def handle_localrc(self, localrc): |
| lfg = False |
| tp = False |
| if localrc: |
| vg = VarGraph(localrc) |
| for k, v in vg.getVars(): |
| # Avoid double quoting |
| if len(v) and v[0]=='"': |
| self.localrc.append('{}={}'.format(k, v)) |
| else: |
| self.localrc.append('{}="{}"'.format(k, v)) |
| if k == 'LIBS_FROM_GIT': |
| lfg = True |
| elif k == 'TEMPEST_PLUGINS': |
| tp = True |
| |
| if not lfg and (self.projects or self.project): |
| required_projects = [] |
| if self.projects: |
| for project_name, project_info in self.projects.items(): |
| if project_info.get('required'): |
| required_projects.append(project_info['short_name']) |
| if self.project: |
| if self.project['short_name'] not in required_projects: |
| required_projects.append(self.project['short_name']) |
| if required_projects: |
| self.localrc.append('LIBS_FROM_GIT={}'.format( |
| ','.join(required_projects))) |
| |
| if self.tempest_plugins: |
| if not tp: |
| tp_dirs = [] |
| for tempest_plugin in self.tempest_plugins: |
| tp_dirs.append(os.path.join(self.base_dir, tempest_plugin)) |
| self.localrc.append('TEMPEST_PLUGINS="{}"'.format( |
| ' '.join(tp_dirs))) |
| else: |
| self.warnings.append('TEMPEST_PLUGINS already defined ({}),' |
| 'requested value {} ignored'.format( |
| tp, self.tempest_plugins)) |
| |
| |
| def handle_localconf(self, localconf): |
| for phase, phase_data in localconf.items(): |
| for fn, fn_data in phase_data.items(): |
| ms_name = '[[{}|{}]]'.format(phase, fn) |
| ms_data = [] |
| for section, section_data in fn_data.items(): |
| ms_data.append('[{}]'.format(section)) |
| for k, v in section_data.items(): |
| ms_data.append('{} = {}'.format(k, v)) |
| ms_data.append('') |
| self.meta_sections[ms_name] = ms_data |
| |
| def write(self, path): |
| with open(path, 'w') as f: |
| f.write('[[local|localrc]]\n') |
| f.write('\n'.join(self.localrc)) |
| f.write('\n\n') |
| for section, lines in self.meta_sections.items(): |
| f.write('{}\n'.format(section)) |
| f.write('\n'.join(lines)) |
| |
| |
| def main(): |
| module = AnsibleModule( |
| argument_spec=dict( |
| plugins=dict(type='dict'), |
| base_services=dict(type='list'), |
| services=dict(type='dict'), |
| localrc=dict(type='dict'), |
| local_conf=dict(type='dict'), |
| base_dir=dict(type='path'), |
| path=dict(type='str'), |
| projects=dict(type='dict'), |
| project=dict(type='dict'), |
| tempest_plugins=dict(type='list'), |
| ) |
| ) |
| |
| p = module.params |
| lc = LocalConf(p.get('localrc'), |
| p.get('local_conf'), |
| p.get('base_services'), |
| p.get('services'), |
| p.get('plugins'), |
| p.get('base_dir'), |
| p.get('projects'), |
| p.get('project'), |
| p.get('tempest_plugins')) |
| lc.write(p['path']) |
| |
| module.exit_json(warnings=lc.warnings) |
| |
| |
| try: |
| from ansible.module_utils.basic import * # noqa |
| from ansible.module_utils.basic import AnsibleModule |
| except ImportError: |
| pass |
| |
| if __name__ == '__main__': |
| main() |