Merge "install mod_ssl on centos 9 stream by default"
diff --git a/.zuul.yaml b/.zuul.yaml
index 0dda262..8fd4d02 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -177,6 +177,36 @@
- compute1
- nodeset:
+ name: openstack-two-node-centos-9-stream
+ nodes:
+ - name: controller
+ label: centos-9-stream
+ - name: compute1
+ label: centos-9-stream
+ groups:
+ # Node where tests are executed and test results collected
+ - name: tempest
+ nodes:
+ - controller
+ # Nodes running the compute service
+ - name: compute
+ nodes:
+ - controller
+ - compute1
+ # Nodes that are not the controller
+ - name: subnode
+ nodes:
+ - compute1
+ # Switch node for multinode networking setup
+ - name: switch
+ nodes:
+ - controller
+ # Peer nodes for multinode networking setup
+ - name: peers
+ nodes:
+ - compute1
+
+- nodeset:
name: openstack-two-node-focal
nodes:
- name: controller
@@ -389,6 +419,7 @@
'{{ devstack_log_dir }}/worlddump-latest.txt': logs
'{{ devstack_full_log}}': logs
'{{ stage_dir }}/verify_tempest_conf.log': logs
+ '{{ stage_dir }}/performance.json': logs
'{{ stage_dir }}/apache': logs
'{{ stage_dir }}/apache_config': logs
'{{ stage_dir }}/etc': logs
diff --git a/doc/source/configuration.rst b/doc/source/configuration.rst
index dd8f21f..40a8725 100644
--- a/doc/source/configuration.rst
+++ b/doc/source/configuration.rst
@@ -279,7 +279,7 @@
::
- LOGDAYS=1
+ LOGDAYS=2
Some coloring is used during the DevStack runs to make it easier to
see what is going on. This can be disabled with::
diff --git a/functions-common b/functions-common
index ddef2e4..8651604 100644
--- a/functions-common
+++ b/functions-common
@@ -677,8 +677,13 @@
# about how we clone and work with repos. Mark them safe globally
# as a work-around.
#
+ # NOTE(danms): On bionic (and likely others) git-config may write
+ # ~stackuser/.gitconfig if not run with sudo -H. Using --system
+ # writes these changes to /etc/gitconfig which is more
+ # discoverable anyway.
+ #
# [1] https://github.com/git/git/commit/8959555cee7ec045958f9b6dd62e541affb7e7d9
- sudo git config --global --add safe.directory ${git_dest}
+ sudo git config --system --add safe.directory ${git_dest}
# print out the results so we know what change was used in the logs
cd $git_dest
diff --git a/lib/databases/mysql b/lib/databases/mysql
index 0f45273..6b3ea02 100644
--- a/lib/databases/mysql
+++ b/lib/databases/mysql
@@ -150,6 +150,15 @@
iniset -sudo $my_conf mysqld log-queries-not-using-indexes 1
fi
+ if [[ "$MYSQL_GATHER_PERFORMANCE" == "True" ]]; then
+ echo "enabling MySQL performance_schema items"
+ # Enable long query history
+ iniset -sudo $my_conf mysqld \
+ performance-schema-consumer-events-statements-history-long TRUE
+ iniset -sudo $my_conf mysqld \
+ performance_schema_events_stages_history_long_size 1000000
+ fi
+
restart_service $MYSQL_SERVICE_NAME
}
diff --git a/playbooks/post.yaml b/playbooks/post.yaml
index 9e66f20..d8d5f68 100644
--- a/playbooks/post.yaml
+++ b/playbooks/post.yaml
@@ -20,6 +20,9 @@
roles:
- export-devstack-journal
- apache-logs-conf
+ # This should run as early as possible to make sure we don't skew
+ # the post-tempest results with other activities.
+ - capture-performance-data
- devstack-project-conf
# capture-system-logs should be the last role before stage-output
- capture-system-logs
diff --git a/roles/capture-performance-data/README.rst b/roles/capture-performance-data/README.rst
new file mode 100644
index 0000000..b7a37c2
--- /dev/null
+++ b/roles/capture-performance-data/README.rst
@@ -0,0 +1,25 @@
+Generate performance logs for staging
+
+Captures usage information from mysql, systemd, apache logs, and other
+parts of the system and generates a performance.json file in the
+staging directory.
+
+**Role Variables**
+
+.. zuul:rolevar:: stage_dir
+ :default: {{ ansible_user_dir }}
+
+ The base stage directory
+
+.. zuul:rolevar:: devstack_conf_dir
+ :default: /opt/stack
+
+ The base devstack destination directory
+
+.. zuul:rolevar:: debian_suse_apache_deref_logs
+
+ The apache logs found in the debian/suse locations
+
+.. zuul:rolevar:: redhat_apache_deref_logs
+
+ The apache logs found in the redhat locations
diff --git a/roles/capture-performance-data/defaults/main.yaml b/roles/capture-performance-data/defaults/main.yaml
new file mode 100644
index 0000000..7bd79f4
--- /dev/null
+++ b/roles/capture-performance-data/defaults/main.yaml
@@ -0,0 +1,3 @@
+devstack_base_dir: /opt/stack
+devstack_conf_dir: "{{ devstack_base_dir }}"
+stage_dir: "{{ ansible_user_dir }}"
diff --git a/roles/capture-performance-data/tasks/main.yaml b/roles/capture-performance-data/tasks/main.yaml
new file mode 100644
index 0000000..2d2cfe4
--- /dev/null
+++ b/roles/capture-performance-data/tasks/main.yaml
@@ -0,0 +1,15 @@
+- name: Generate statistics
+ shell:
+ executable: /bin/bash
+ cmd: |
+ source {{ devstack_conf_dir }}/stackrc
+ python3 {{ devstack_conf_dir }}/tools/get-stats.py \
+ --db-user="$DATABASE_USER" \
+ --db-pass="$DATABASE_PASSWORD" \
+ --db-host="$DATABASE_HOST" \
+ {{ apache_logs }} > {{ stage_dir }}/performance.json
+ vars:
+ apache_logs: >-
+ {% for i in debian_suse_apache_deref_logs.results | default([]) + redhat_apache_deref_logs.results | default([]) %}
+ --apache-log="{{ i.stat.path }}"
+ {% endfor %}
diff --git a/samples/local.conf b/samples/local.conf
index 8b76137..55b7298 100644
--- a/samples/local.conf
+++ b/samples/local.conf
@@ -49,7 +49,7 @@
# path of the destination log file. A timestamp will be appended to the given name.
LOGFILE=$DEST/logs/stack.sh.log
-# Old log files are automatically removed after 7 days to keep things neat. Change
+# Old log files are automatically removed after 2 days to keep things neat. Change
# the number of days by setting ``LOGDAYS``.
LOGDAYS=2
diff --git a/stackrc b/stackrc
index d22fa88..c3254dc 100644
--- a/stackrc
+++ b/stackrc
@@ -193,6 +193,10 @@
# (currently only implemented for MySQL backend)
DATABASE_QUERY_LOGGING=$(trueorfalse False DATABASE_QUERY_LOGGING)
+# This can be used to turn on various non-default items in the
+# performance_schema that are of interest to us
+MYSQL_GATHER_PERFORMANCE=$(trueorfalse True MYSQL_GATHER_PERFORMANCE)
+
# Set a timeout for git operations. If git is still running when the
# timeout expires, the command will be retried up to 3 times. This is
# in the format for timeout(1);
diff --git a/tools/get-stats.py b/tools/get-stats.py
new file mode 100755
index 0000000..dc0bd0f
--- /dev/null
+++ b/tools/get-stats.py
@@ -0,0 +1,155 @@
+#!/usr/bin/python3
+
+import argparse
+import datetime
+import glob
+import itertools
+import json
+import os
+import psutil
+import re
+import socket
+import subprocess
+import sys
+import pymysql
+
+# https://www.elastic.co/blog/found-crash-elasticsearch#mapping-explosion
+
+
+def tryint(value):
+ try:
+ return int(value)
+ except (ValueError, TypeError):
+ return value
+
+
+def get_service_stats(service):
+ stats = {'MemoryCurrent': 0}
+ output = subprocess.check_output(['/usr/bin/systemctl', 'show', service] +
+ ['-p%s' % stat for stat in stats])
+ for line in output.decode().split('\n'):
+ if not line:
+ continue
+ stat, val = line.split('=')
+ stats[stat] = int(val)
+
+ return stats
+
+
+def get_services_stats():
+ services = [os.path.basename(s) for s in
+ glob.glob('/etc/systemd/system/devstack@*.service')]
+ return [dict(service=service, **get_service_stats(service))
+ for service in services]
+
+
+def get_process_stats(proc):
+ cmdline = proc.cmdline()
+ if 'python' in cmdline[0]:
+ cmdline = cmdline[1:]
+ return {'cmd': cmdline[0],
+ 'pid': proc.pid,
+ 'args': ' '.join(cmdline[1:]),
+ 'rss': proc.memory_info().rss}
+
+
+def get_processes_stats(matches):
+ me = os.getpid()
+ procs = psutil.process_iter()
+
+ def proc_matches(proc):
+ return me != proc.pid and any(
+ re.search(match, ' '.join(proc.cmdline()))
+ for match in matches)
+
+ return [
+ get_process_stats(proc)
+ for proc in procs
+ if proc_matches(proc)]
+
+
+def get_db_stats(host, user, passwd):
+ dbs = []
+ db = pymysql.connect(host=host, user=user, password=passwd,
+ database='performance_schema',
+ cursorclass=pymysql.cursors.DictCursor)
+ with db:
+ with db.cursor() as cur:
+ cur.execute(
+ 'SELECT COUNT(*) AS queries,current_schema AS db FROM '
+ 'events_statements_history_long GROUP BY current_schema')
+ for row in cur:
+ dbs.append({k: tryint(v) for k, v in row.items()})
+ return dbs
+
+
+def get_http_stats_for_log(logfile):
+ stats = {}
+ for line in open(logfile).readlines():
+ m = re.search('"([A-Z]+) /([^" ]+)( HTTP/1.1)?" ([0-9]{3}) ([0-9]+)',
+ line)
+ if m:
+ method = m.group(1)
+ path = m.group(2)
+ status = m.group(4)
+ size = int(m.group(5))
+
+ try:
+ service, rest = path.split('/', 1)
+ except ValueError:
+ # Root calls like "GET /identity"
+ service = path
+ rest = ''
+
+ stats.setdefault(service, {'largest': 0})
+ stats[service].setdefault(method, 0)
+ stats[service][method] += 1
+ stats[service]['largest'] = max(stats[service]['largest'], size)
+
+ # Flatten this for ES
+ return [{'service': service, 'log': os.path.basename(logfile),
+ **vals}
+ for service, vals in stats.items()]
+
+
+def get_http_stats(logfiles):
+ return list(itertools.chain.from_iterable(get_http_stats_for_log(log)
+ for log in logfiles))
+
+
+def get_report_info():
+ return {
+ 'timestamp': datetime.datetime.now().isoformat(),
+ 'hostname': socket.gethostname(),
+ }
+
+
+if __name__ == '__main__':
+ process_defaults = ['privsep', 'mysqld', 'erlang', 'etcd']
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--db-user', default='root',
+ help=('MySQL user for collecting stats '
+ '(default: "root")'))
+ parser.add_argument('--db-pass', default=None,
+ help='MySQL password for db-user')
+ parser.add_argument('--db-host', default='localhost',
+ help='MySQL hostname')
+ parser.add_argument('--apache-log', action='append', default=[],
+ help='Collect API call stats from this apache log')
+ parser.add_argument('--process', action='append',
+ default=process_defaults,
+ help=('Include process stats for this cmdline regex '
+ '(default is %s)' % ','.join(process_defaults)))
+ args = parser.parse_args()
+
+ data = {
+ 'services': get_services_stats(),
+ 'db': args.db_pass and get_db_stats(args.db_host,
+ args.db_user,
+ args.db_pass) or [],
+ 'processes': get_processes_stats(args.process),
+ 'api': get_http_stats(args.apache_log),
+ 'report': get_report_info(),
+ }
+
+ print(json.dumps(data, indent=2))
diff --git a/unstack.sh b/unstack.sh
index 4b57b6e..813f9a8 100755
--- a/unstack.sh
+++ b/unstack.sh
@@ -181,3 +181,8 @@
clean_pyc_files
rm -Rf $DEST/async
+
+# Clean any safe.directory items we wrote into the global
+# gitconfig. We can identify the relevant ones by checking that they
+# point to somewhere in our $DEST directory.
+sudo sed -i "/directory=${DEST}/ d" /etc/gitconfig