From 4a346b779ad973f742ae0131c59217a761618887 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Fri, 22 Nov 2024 22:51:45 +0100 Subject: [PATCH] test: Dump coredumps from journal in the integration test wrapper Fixes #35277 --- test/TEST-02-UNITTESTS/meson.build | 1 + test/TEST-16-EXTEND-TIMEOUT/meson.build | 1 + test/TEST-17-UDEV/meson.build | 1 + test/TEST-59-RELOADING-RESTART/meson.build | 1 + test/TEST-73-LOCALE/meson.build | 2 + test/TEST-74-AUX-UTILS/meson.build | 1 + test/integration-test-wrapper.py | 120 +++++++++++++++------ test/meson.build | 2 + 8 files changed, 95 insertions(+), 34 deletions(-) diff --git a/test/TEST-02-UNITTESTS/meson.build b/test/TEST-02-UNITTESTS/meson.build index b0003391b5c..9be6f2a6d95 100644 --- a/test/TEST-02-UNITTESTS/meson.build +++ b/test/TEST-02-UNITTESTS/meson.build @@ -3,6 +3,7 @@ integration_tests += [ integration_test_template + { 'name' : fs.name(meson.current_source_dir()), + 'coredump-exclude-regex' : '/(bash|python3.[0-9]+|systemd-executor)$', 'cmdline' : integration_test_template['cmdline'] + [ ''' diff --git a/test/TEST-16-EXTEND-TIMEOUT/meson.build b/test/TEST-16-EXTEND-TIMEOUT/meson.build index 15986acfb45..2a06f9a151f 100644 --- a/test/TEST-16-EXTEND-TIMEOUT/meson.build +++ b/test/TEST-16-EXTEND-TIMEOUT/meson.build @@ -4,6 +4,7 @@ integration_tests += [ integration_test_template + { 'name' : fs.name(meson.current_source_dir()), 'unit' : files('TEST-16-EXTEND-TIMEOUT.service'), + 'coredump-exclude-regex' : '/(bash|sleep)$', }, ] diff --git a/test/TEST-17-UDEV/meson.build b/test/TEST-17-UDEV/meson.build index 77370ce4588..1d72c56e8c6 100644 --- a/test/TEST-17-UDEV/meson.build +++ b/test/TEST-17-UDEV/meson.build @@ -4,5 +4,6 @@ integration_tests += [ integration_test_template + { 'name' : fs.name(meson.current_source_dir()), 'vm' : true, + 'coredump-exclude-regex' : '/(sleep|udevadm)$', }, ] diff --git a/test/TEST-59-RELOADING-RESTART/meson.build b/test/TEST-59-RELOADING-RESTART/meson.build index 8dec5f37e73..3d12024cef3 100644 --- a/test/TEST-59-RELOADING-RESTART/meson.build +++ b/test/TEST-59-RELOADING-RESTART/meson.build @@ -3,5 +3,6 @@ integration_tests += [ integration_test_template + { 'name' : fs.name(meson.current_source_dir()), + 'coredump-exclude-regex' : '/(sleep|bash|systemd-notify)$', }, ] diff --git a/test/TEST-73-LOCALE/meson.build b/test/TEST-73-LOCALE/meson.build index 4f50d66e550..e5b9f190151 100644 --- a/test/TEST-73-LOCALE/meson.build +++ b/test/TEST-73-LOCALE/meson.build @@ -4,5 +4,7 @@ integration_tests += [ integration_test_template + { 'name' : fs.name(meson.current_source_dir()), 'priority' : 10, + # TODO: Remove when https://github.com/systemd/systemd/issues/35335 is fixed. + 'coredump-exclude-regex' : '/systemd-localed', }, ] diff --git a/test/TEST-74-AUX-UTILS/meson.build b/test/TEST-74-AUX-UTILS/meson.build index 543eee195ff..ee24cd8f786 100644 --- a/test/TEST-74-AUX-UTILS/meson.build +++ b/test/TEST-74-AUX-UTILS/meson.build @@ -5,6 +5,7 @@ integration_tests += [ 'name' : fs.name(meson.current_source_dir()), 'storage': 'persistent', 'vm' : true, + 'coredump-exclude-regex' : '/(test-usr-dump|test-dump|bash)$', }, ] diff --git a/test/integration-test-wrapper.py b/test/integration-test-wrapper.py index 737bbd42724..a3f90dc9faf 100755 --- a/test/integration-test-wrapper.py +++ b/test/integration-test-wrapper.py @@ -6,6 +6,7 @@ import argparse import json import os +import re import shlex import subprocess import sys @@ -32,6 +33,59 @@ ExecStart=false """ +def process_coredumps(args: argparse.Namespace, journal_file: Path) -> bool: + # Collect executable paths of all coredumps and filter out the expected ones. + + if args.coredump_exclude_regex: + exclude_regex = re.compile(args.coredump_exclude_regex) + else: + exclude_regex = None + + result = subprocess.run( + [ + args.mkosi, + '--directory', os.fspath(args.meson_source_dir), + '--extra-search-path', os.fspath(args.meson_build_dir), + 'sandbox', + 'coredumpctl', + '--file', journal_file, + '--json=short', + ], + stdout=subprocess.PIPE, + text=True, + ) # fmt: skip + + # coredumpctl returns a non-zero exit status if there are no coredumps. + if result.returncode != 0: + return False + + coredumps = json.loads(result.stdout) + + coredumps = [ + coredump for coredump in coredumps if not exclude_regex or not exclude_regex.search(coredump['exe']) + ] + + if not coredumps: + return False + + subprocess.run( + [ + args.mkosi, + '--directory', os.fspath(args.meson_source_dir), + '--extra-search-path', os.fspath(args.meson_build_dir), + 'sandbox', + 'coredumpctl', + '--file', journal_file, + '--no-pager', + 'info', + *(coredump['exe'] for coredump in coredumps), + ], + check=True, + ) # fmt: skip + + return True + + def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--mkosi', required=True) @@ -44,6 +98,7 @@ def main() -> None: parser.add_argument('--slow', action=argparse.BooleanOptionalAction) parser.add_argument('--vm', action=argparse.BooleanOptionalAction) parser.add_argument('--exit-code', required=True, type=int) + parser.add_argument('--coredump-exclude-regex', required=True) parser.add_argument('mkosi_args', nargs='*') args = parser.parse_args() @@ -114,7 +169,9 @@ def main() -> None: """ ) - journal_file = None + journal_file = (args.meson_build_dir / (f'test/journal/{name}.journal')).absolute() + journal_file.unlink(missing_ok=True) + if not sys.stderr.isatty(): dropin += textwrap.dedent( """ @@ -122,9 +179,6 @@ def main() -> None: FailureAction=exit """ ) - - journal_file = (args.meson_build_dir / (f'test/journal/{name}.journal')).absolute() - journal_file.unlink(missing_ok=True) elif not shell: dropin += textwrap.dedent( """ @@ -194,44 +248,42 @@ def main() -> None: ) exit(77) - if journal_file and ( - keep_journal == '0' or (result.returncode in (args.exit_code, 77) and keep_journal == 'fail') + coredumps = process_coredumps(args, journal_file) + + if keep_journal == '0' or ( + keep_journal == 'fail' and result.returncode in (args.exit_code, 77) and not coredumps ): journal_file.unlink(missing_ok=True) - if shell or result.returncode in (args.exit_code, 77): + if shell or (result.returncode in (args.exit_code, 77) and not coredumps): exit(0 if shell or result.returncode == args.exit_code else 77) - if journal_file: - ops = [] + ops = [] - if os.getenv('GITHUB_ACTIONS'): - id = os.environ['GITHUB_RUN_ID'] - iteration = os.environ['GITHUB_RUN_ATTEMPT'] - j = json.loads( - subprocess.run( - [ - args.mkosi, - '--directory', os.fspath(args.meson_source_dir), - '--json', - 'summary', - ], - stdout=subprocess.PIPE, - text=True, - ).stdout - ) # fmt: skip - distribution = j['Images'][-1]['Distribution'] - release = j['Images'][-1]['Release'] - artifact = f'ci-mkosi-{id}-{iteration}-{distribution}-{release}-failed-test-journals' - ops += [f'gh run download {id} --name {artifact} -D ci/{artifact}'] - journal_file = Path(f'ci/{artifact}/test/journal/{name}.journal') + if os.getenv('GITHUB_ACTIONS'): + id = os.environ['GITHUB_RUN_ID'] + iteration = os.environ['GITHUB_RUN_ATTEMPT'] + j = json.loads( + subprocess.run( + [ + args.mkosi, + '--directory', os.fspath(args.meson_source_dir), + '--json', + 'summary', + ], + stdout=subprocess.PIPE, + text=True, + ).stdout + ) # fmt: skip + distribution = j['Images'][-1]['Distribution'] + release = j['Images'][-1]['Release'] + artifact = f'ci-mkosi-{id}-{iteration}-{distribution}-{release}-failed-test-journals' + ops += [f'gh run download {id} --name {artifact} -D ci/{artifact}'] + journal_file = Path(f'ci/{artifact}/test/journal/{name}.journal') - ops += [f'journalctl --file {journal_file} --no-hostname -o short-monotonic -u {args.unit} -p info'] + ops += [f'journalctl --file {journal_file} --no-hostname -o short-monotonic -u {args.unit} -p info'] - print( - "Test failed, relevant logs can be viewed with: \n\n" f"{(' && '.join(ops))}\n", - file=sys.stderr, - ) + print("Test failed, relevant logs can be viewed with: \n\n" f"{(' && '.join(ops))}\n", file=sys.stderr) # 0 also means we failed so translate that to a non-zero exit code to mark the test as failed. exit(result.returncode or 1) diff --git a/test/meson.build b/test/meson.build index ceebb72bae0..6d3fdef1fc4 100644 --- a/test/meson.build +++ b/test/meson.build @@ -297,6 +297,7 @@ integration_test_template = { 'qemu-args' : [], 'exit-code' : 123, 'vm' : false, + 'coredump-exclude-regex' : '', } testdata_subdirs = [ 'auxv', @@ -391,6 +392,7 @@ foreach integration_test : integration_tests '--storage', integration_test['storage'], '--firmware', integration_test['firmware'], '--exit-code', integration_test['exit-code'].to_string(), + '--coredump-exclude-regex', integration_test['coredump-exclude-regex'], ] if 'unit' in integration_test