Added list of mlock-using processes to peakmem_tracker output
The change makes peakmem_tracker list processes that lock memory pages
from swapping to disk. It may be helpful when debugging oom-killer job
failures in gate in case when dstat shows that swap is not fully used
when oom-killer is triggered.
The peakmem_tracker service was renamed into memory_tracker to reflect
its new broader scope.
Needed-By: I5862d92478397eac2e61b8a61ce3437b698678be
Change-Id: I1dca120448ee87930fe903fd81277b58efaefc92
diff --git a/tools/mlock_report.py b/tools/mlock_report.py
new file mode 100755
index 0000000..1d23af9
--- /dev/null
+++ b/tools/mlock_report.py
@@ -0,0 +1,59 @@
+#!/usr/bin/env python
+
+# This tool lists processes that lock memory pages from swapping to disk.
+
+import re
+import subprocess
+
+import psutil
+
+
+SUMMARY_REGEX = re.compile(r".*\s+(?P<locked>[\d]+)\s+KB")
+
+
+def main():
+ try:
+ print _get_report()
+ except Exception as e:
+ print "Failure listing processes locking memory: %s" % str(e)
+
+
+def _get_report():
+ mlock_users = []
+ for proc in psutil.process_iter():
+ pid = proc.pid
+ # sadly psutil does not expose locked pages info, that's why we
+ # call to pmap and parse the output here
+ try:
+ out = subprocess.check_output(['pmap', '-XX', str(pid)])
+ except subprocess.CalledProcessError as e:
+ # 42 means process just vanished, which is ok
+ if e.returncode == 42:
+ continue
+ raise
+ last_line = out.splitlines()[-1]
+
+ # some processes don't provide a memory map, for example those
+ # running as kernel services, so we need to skip those that don't
+ # match
+ result = SUMMARY_REGEX.match(last_line)
+ if result:
+ locked = int(result.group('locked'))
+ if locked:
+ mlock_users.append({'name': proc.name(),
+ 'pid': pid,
+ 'locked': locked})
+
+ # produce a single line log message with per process mlock stats
+ if mlock_users:
+ return "; ".join(
+ "[%(name)s (pid:%(pid)s)]=%(locked)dKB" % args
+ # log heavy users first
+ for args in sorted(mlock_users, key=lambda d: d['locked'])
+ )
+ else:
+ return "no locked memory"
+
+
+if __name__ == "__main__":
+ main()