test: Use meson add_test_setup() instead of environment variables

We add a default test setup that excludes the integration-tests suite
so that the integration tests don't run by default. This allows us to
get rid of $SYSTEMD_INTEGRATION_TESTS. Then, we add two extra setups:
'integration' and 'shell'. The 'integration' setup does not exclude the
integration-tests suite, and so can be used to run the integration tests.
The 'shell' setup does the same, but additionally sets $TEST_SHELL=1,
allowing to get rid of $TEST_SHELL in the docs.

Author: DaanDeMeyer

.github/workflows/coverage.yml

@@ -90,7 +90,6 @@ jobs:
           sudo mkosi sandbox -- \
             meson setup \
             --buildtype=debugoptimized \
-            -Dintegration-tests=true \
             build
 
       - name: Build image
@@ -120,7 +119,8 @@ jobs:
             meson test \
             -C build \
             --no-rebuild \
-            --suite integration-tests \
+            --setup=integration \
+            --suite=integration-tests \
             --print-errorlogs \
             --no-stdsplit \
             --num-processes "$(($(nproc) - 1))" \

.github/workflows/mkosi.yml

@@ -197,7 +197,6 @@ jobs:
           sudo mkosi sandbox -- \
             meson setup \
             --buildtype=debugoptimized \
-            -Dintegration-tests=true \
             -Dbpf-framework=disabled \
             build
 
@@ -233,7 +232,8 @@ jobs:
             meson test \
             -C build \
             --no-rebuild \
-            --suite integration-tests \
+            --setup=integration \
+            --suite=integration-tests \
             --print-errorlogs \
             --no-stdsplit \
             --num-processes "$(($(nproc) - 1))" \

meson.build

@@ -13,6 +13,12 @@ project('systemd', 'c',
         meson_version : '>= 0.62.0',
 )
 
+add_test_setup(
+        'default',
+        exclude_suites : ['integration-tests'],
+        is_default : true,
+)
+
 project_major_version = meson.project_version().split('.')[0].split('~')[0]
 if meson.project_version().contains('.')
         project_minor_version = meson.project_version().split('.')[-1].split('~')[0]
@@ -339,7 +345,6 @@ meson_build_sh = find_program('tools/meson-build.sh')
 want_tests = get_option('tests')
 want_slow_tests = want_tests != 'false' and get_option('slow-tests')
 want_fuzz_tests = want_tests != 'false' and get_option('fuzz-tests')
-want_integration_tests = want_tests != 'false' and get_option('integration-tests')
 install_tests = want_tests != 'false' and get_option('install-tests')
 
 if add_languages('cpp', native : false, required : fuzzer_build)
@@ -2661,10 +2666,6 @@ endif
 #####################################################################
 
 mkosi = find_program('mkosi', required : false)
-if want_integration_tests and not mkosi.found()
-        error('Could not find mkosi which is required to run the integration tests')
-endif
-
 mkosi_depends = public_programs
 
 foreach executable : ['systemd-journal-remote', 'systemd-sbsign', 'systemd-keyutil']

meson_options.txt

@@ -509,7 +509,7 @@ option('install-tests', type : 'boolean', value : false,
        description : 'install test executables')
 option('log-message-verification', type : 'feature', deprecated : { 'true' : 'enabled', 'false' : 'disabled' },
        description : 'do fake printf() calls to verify format strings')
-option('integration-tests', type : 'boolean', value : false,
+option('integration-tests', type : 'boolean', value : false, deprecated : true,
        description : 'run the integration tests')
 
 option('ok-color', type : 'combo',

test/integration-tests/README.md

@@ -38,14 +38,14 @@ directory (`OutputDirectory=`) to point to the other directory using `mkosi/mkos
 After the image has been built, the integration tests can be run with:
 
 ```shell
-$ env SYSTEMD_INTEGRATION_TESTS=1 mkosi -f sandbox -- meson test -C build --suite integration-tests --num-processes "$(($(nproc) / 4))"
+$ mkosi -f sandbox -- meson test -C build --setup=integration --suite integration-tests --num-processes "$(($(nproc) / 4))"
 ```
 
 As usual, specific tests can be run in meson by appending the name of the test
 which is usually the name of the directory e.g.
 
 ```shell
-$ env SYSTEMD_INTEGRATION_TESTS=1 mkosi -f sandbox -- meson test -C build -v TEST-01-BASIC
+$ mkosi -f sandbox -- meson test -C build --setup=integration -v TEST-01-BASIC
 ```
 
 See `mkosi -f sandbox -- meson introspect build --tests` for a list of tests.
@@ -55,7 +55,7 @@ To interactively debug a failing integration test, the `--interactive` option
 newer:
 
 ```shell
-$ env SYSTEMD_INTEGRATION_TESTS=1 mkosi -f sandbox -- meson test -C build -i TEST-01-BASIC
+$ mkosi -f sandbox -- meson test -C build --setup=integration -i TEST-01-BASIC
 ```
 
 Due to limitations in meson, the integration tests do not yet depend on the
@@ -64,7 +64,7 @@ running the integration tests. To rebuild the image and rerun a test, the
 following command can be used:
 
 ```shell
-$ mkosi -f sandbox -- meson compile -C build mkosi && env SYSTEMD_INTEGRATION_TESTS=1 mkosi -f sandbox -- meson test -C build -v TEST-01-BASIC
+$ mkosi -f sandbox -- meson compile -C build mkosi && mkosi -f sandbox -- meson test -C build --setup=integration -v TEST-01-BASIC
 ```
 
 The integration tests use the same mkosi configuration that's used when you run
@@ -78,7 +78,7 @@ To iterate on an integration test, let's first get a shell in the integration te
 the following:
 
 ```shell
-$ mkosi -f sandbox -- meson compile -C build mkosi && env SYSTEMD_INTEGRATION_TESTS=1 TEST_SHELL=1 mkosi -f sandbox -- meson test -C build -i TEST-01-BASIC
+$ mkosi -f sandbox -- meson compile -C build mkosi && mkosi -f sandbox -- meson test -C build --setup=shell -i TEST-01-BASIC
 ```
 
 This will get us a shell in the integration test environment after booting the machine without running the
@@ -107,7 +107,7 @@ re-running the test will first install the new packages we just built, make a ne
 the test again. You can keep running the loop of `mkosi -R`, `systemctl soft-reboot` and
 `systemctl start ...` until the changes to the integration test are working.
 
-If you're debugging a failing integration test (running `meson test --interactive` without `TEST_SHELL`),
+If you're debugging a failing integration test (running `meson test --interactive`),
 there's no need to run `systemctl start ...`, running `systemctl soft-reboot` on its own is sufficient to
 rerun the test.
 
@@ -120,10 +120,6 @@ rerun the test.
 `TEST_NO_KVM=1`: Disable qemu KVM auto-detection (may be necessary when you're
 trying to run the *vanilla* qemu and have both qemu and qemu-kvm installed)
 
-`TEST_SHELL=1`: Configure the machine to be more *user-friendly* for
-interactive debugging (e.g. by setting a usable default terminal, suppressing
-the shutdown after the test, etc.).
-
 `TEST_MATCH_SUBTEST=subtest`:  If the test makes use of `run_subtests` use this
 variable to provide a POSIX extended regex to run only subtests matching the
 expression.

test/integration-tests/integration-test-wrapper.py

@@ -361,7 +361,7 @@ def statfs(path: Path) -> str:
 
 def main() -> None:
     parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument('--mkosi', required=True)
+    parser.add_argument('--mkosi', default=None)
     parser.add_argument('--meson-source-dir', required=True, type=Path)
     parser.add_argument('--meson-build-dir', required=True, type=Path)
     parser.add_argument('--name', required=True)
@@ -379,6 +379,12 @@ def main() -> None:
     parser.add_argument('mkosi_args', nargs='*')
     args = parser.parse_args()
 
+    if not args.mkosi:
+        args.mkosi = shutil.which('mkosi')
+        if not args.mkosi:
+            print('Could not find mkosi which is required to run the integration tests', file=sys.stderr)
+            sys.exit(1)
+
     # The meson source directory can either be the top-level repository directory or the
     # test/integration-tests/standalone subdirectory in the repository directory. The mkosi configuration
     # will always be a parent directory of one of these directories and at most 4 levels upwards, so don't
@@ -395,13 +401,6 @@ def main() -> None:
         )
         exit(1)
 
-    if not bool(int(os.getenv('SYSTEMD_INTEGRATION_TESTS', '0'))):
-        print(
-            f'SYSTEMD_INTEGRATION_TESTS=1 not found in environment, skipping {args.name}',
-            file=sys.stderr,
-        )
-        exit(77)
-
     if args.slow and not bool(int(os.getenv('SYSTEMD_SLOW_TESTS', '0'))):
         print(
             f'SYSTEMD_SLOW_TESTS=1 not found in environment, skipping {args.name}',

test/integration-tests/meson.build

@@ -1,5 +1,9 @@
 # SPDX-License-Identifier: LGPL-2.1-or-later
 
+# We'd give these more descriptive names but only alphanumeric characters are allowed.
+add_test_setup('integration')
+add_test_setup('shell', env : {'TEST_SHELL' : '1'})
+
 integration_test_wrapper = find_program('integration-test-wrapper.py')
 integration_tests = []
 integration_test_template = {
@@ -129,11 +133,11 @@ foreach integration_test : integration_tests
                 integration_test_args += ['--skip']
         endif
 
-        if not mkosi.found()
-                continue
+        if mkosi.found()
+                integration_test_args += ['--mkosi', mkosi.full_path()]
         endif
 
-        integration_test_args += ['--mkosi', mkosi.full_path(), '--']
+        integration_test_args += ['--']
 
         if integration_test['cmdline'].length() > 0
                 integration_test_args += [
@@ -151,19 +155,12 @@ foreach integration_test : integration_tests
 
         integration_test_args += integration_test['mkosi-args']
 
-        integration_test_env = {}
-
-        if want_integration_tests
-                integration_test_env += {'SYSTEMD_INTEGRATION_TESTS': '1'}
-        endif
-
         # We don't explicitly depend on the "mkosi" target because that means the image is rebuilt on every
         # "ninja -C build". Instead, the mkosi target has to be rebuilt manually before running the
         # integration tests with mkosi.
         test(
                 integration_test['name'],
                 integration_test_wrapper,
-                env : integration_test_env,
                 args : integration_test_args,
                 timeout : integration_test['timeout'],
                 priority : integration_test['priority'],

test/integration-tests/standalone/meson.build

@@ -16,7 +16,6 @@ project('systemd-testsuite',
 
 fs = import('fs')
 mkosi = find_program('mkosi', required : true)
-want_integration_tests = true
 
 # meson refuses .. in subdir() so we use a symlink to trick it into accepting it anyway.
 subdir('integration-tests')