diff --git a/plans/integration/main.fmf b/plans/integration/main.fmf
index e40d17f8c3..d209166485 100644
--- a/plans/integration/main.fmf
+++ b/plans/integration/main.fmf
@@ -5,7 +5,7 @@ description:
 discover:
     how: fmf
     filter: 'tag: integration'
-prepare:
+prepare+:
   - name: Install pip
     how: install
     package: python3-pip
diff --git a/plans/main.fmf b/plans/main.fmf
index 0ad8910cfa..94e5a4cec3 100644
--- a/plans/main.fmf
+++ b/plans/main.fmf
@@ -12,6 +12,13 @@ prepare+:
       - python3-pip
       - python3-pytest
 
+  # Install *Python bindings for jq* - jq itself was installed above
+  # as a regular package. But Python tests use jq to inspect structures,
+  # and need jq Python bindings.
+  - how: shell
+    script: |
+      pip3 install jq
+
   - how: shell
     summary: Make sure we run koji commands with automatic retry to avoid 50x errors
     script: |
diff --git a/pyproject.toml b/pyproject.toml
index f182e020e8..e840a62926 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -137,6 +137,7 @@ platforms = ["linux"]
 description = "Development environment"
 dependencies = [
     "autopep8",
+    "jq",
     "ruff",
     "mypy",
     "pytest",
diff --git a/tests/__init__.py b/tests/__init__.py
index ac22aca297..2c2b7aff98 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -1,11 +1,25 @@
-from collections.abc import Mapping
-from typing import IO, Any, Optional, Protocol, Union
+import contextlib
+import functools
+import os
+from collections.abc import Iterable, Iterator, Mapping
+from typing import IO, Any, Callable, Optional, Protocol, TypeVar, Union, cast
 
 import click.core
 import click.testing
+import jq as _jq
 
 import tmt.__main__
+import tmt._compat.importlib.metadata
 import tmt.cli._root
+from tmt._compat.typing import ParamSpec
+from tmt.utils import Path
+
+_CLICK_VERSION = tuple(
+    int(_s) for _s in tmt._compat.importlib.metadata.version('click').split('.')
+)
+
+T = TypeVar('T')
+P = ParamSpec('P')
 
 
 def reset_common() -> None:
@@ -30,6 +44,52 @@ def reset_common() -> None:
         klass.cli_invocation = None
 
 
+@contextlib.contextmanager
+def cwd(path: Path) -> Iterator[Path]:
+    """
+    A context manager switching the current working directory to a given path.
+
+    .. warning::
+
+        Changing the current working directory can have unexpected
+        consequences in a multithreaded environment.
+    """
+
+    cwd = Path.cwd()
+
+    os.chdir(path)
+
+    try:
+        yield path
+
+    finally:
+        os.chdir(cwd)
+
+
+def with_cwd(path: Path) -> Callable[[Callable[P, T]], Callable[P, T]]:
+    """
+    Decorate a test to have it run in the given path as its CWD.
+    """
+
+    def _with_cwd(fn: Callable[P, T]) -> Callable[P, T]:
+        @functools.wraps(fn)
+        def __with_cwd(*args: P.args, **kwargs: P.kwargs) -> T:
+            with cwd(path):
+                return fn(*args, **kwargs)
+
+        return __with_cwd
+
+    return _with_cwd
+
+
+def jq_all(data: Any, query: str) -> Iterable[Any]:
+    """
+    Apply a jq filter on given data, and return the product.
+    """
+
+    return cast(Iterable[Any], _jq.compile(query).input(data).all())
+
+
 class RunTmt(Protocol):
     """
     A type representing :py:meth:`CliRunner.invoke`.
@@ -40,7 +100,7 @@ class RunTmt(Protocol):
 
     def __call__(
         self,
-        *args: str,
+        *args: Union[str, Path],
         command: Optional[click.BaseCommand] = None,
         input: Optional[Union[str, bytes, IO[Any]]] = None,
         env: Optional[Mapping[str, Optional[str]]] = None,
@@ -53,11 +113,15 @@ def __call__(
 
 class CliRunner(click.testing.CliRunner):
     def __init__(self) -> None:
-        super().__init__(charset='utf-8', echo_stdin=False)
+        if _CLICK_VERSION >= (8, 2, 0):
+            super().__init__(charset='utf-8', echo_stdin=False)
+
+        else:
+            super().__init__(charset='utf-8', echo_stdin=False, mix_stderr=False)
 
     def invoke(  # type: ignore[override]
         self,
-        *args: str,
+        *args: Union[str, Path],
         command: Optional[click.BaseCommand] = None,
         input: Optional[Union[str, bytes, IO[Any]]] = None,
         env: Optional[Mapping[str, Optional[str]]] = None,
@@ -73,7 +137,7 @@ def invoke(  # type: ignore[override]
 
         return super().invoke(
             command,
-            args=args,
+            args=[str(arg) for arg in args],
             input=input,
             env=env,
             catch_exceptions=catch_exceptions,
diff --git a/tests/policy/main.fmf b/tests/policy/main.fmf
index 475fc13da8..4faa246d51 100644
--- a/tests/policy/main.fmf
+++ b/tests/policy/main.fmf
@@ -4,4 +4,5 @@ summary: Verify tmt policies work as expected
   test: ./test-test.sh
 
 /plan:
-  test: ./test-plan.sh
+  framework: shell
+  test: $PYTEST ./test_plan.py
diff --git a/tests/policy/test-plan.sh b/tests/policy/test-plan.sh
deleted file mode 100755
index d09bbdabbc..0000000000
--- a/tests/policy/test-plan.sh
+++ /dev/null
@@ -1,44 +0,0 @@
-#!/bin/bash
-. /usr/share/beakerlib/beakerlib.sh || exit 1
-
-rlJournalStart
-    rlPhaseStartSetup
-        rlRun "pushd data/plan"
-    rlPhaseEnd
-
-    rlPhaseStartTest "Export"
-        # Not doing anything complex, test-level policy test covers plenty
-        # of cases. Focusing on plan-specific modifications only.
-        rlRun -s "tmt -vv plan export --policy-file ../../policies/plan/plan.yaml"
-        rlAssertGrep "Apply tmt policy '../../policies/plan/plan.yaml' to plans." $rlRun_LOG
-
-        rlRun -s "tmt -vv plan export --policy-file ../../policies/plan/plan.yaml 2> /dev/null"
-
-        rlAssertEquals \
-            "Verify that discover key is empty" \
-            "$(yq -o json '.[] | .discover' $rlRun_LOG | jq -cSr)" \
-            "null"
-        rlAssertEquals \
-            "Verify that prepare step contains two phases" \
-            "$(yq -o json '.[] | .prepare | .[] | "\(.how):\(.order)"' $rlRun_LOG | jq -cSr)" \
-            "feature:17
-shell:null"
-        rlAssertEquals \
-            "Verify that contact key was populated" \
-            "$(yq -o json '.[] | .contact | .[]' $rlRun_LOG | jq -cSr)" \
-            "xyzzy"
-    rlPhaseEnd
-
-    rlPhaseStartTest "Run"
-        # Not doing anything complex, just try to run a plan that should
-        # be modified by a policy.
-        rlRun -s "tmt -vv run -a --policy-file ../../policies/plan/simple.yaml" 3
-
-        rlAssertGrep "Apply tmt policy '../../policies/plan/simple.yaml' to plans." $rlRun_LOG
-        rlAssertGrep "No tests found, finishing plan." $rlRun_LOG
-    rlPhaseEnd
-
-    rlPhaseStartCleanup
-        rlRun "popd"
-    rlPhaseEnd
-rlJournalEnd
diff --git a/tests/policy/test_plan.py b/tests/policy/test_plan.py
new file mode 100644
index 0000000000..fae83824b9
--- /dev/null
+++ b/tests/policy/test_plan.py
@@ -0,0 +1,63 @@
+from typing import TYPE_CHECKING
+
+from tests import jq_all, with_cwd
+
+import tmt.cli
+from tmt.utils import Path, from_yaml
+
+if TYPE_CHECKING:
+    from tests import RunTmt
+
+
+TEST_DIR = Path(__file__).absolute().parent
+DATA_DIR = TEST_DIR / 'data/plan'
+POLICIES_DIR = TEST_DIR / 'policies'
+
+
+@with_cwd(DATA_DIR)
+def test_export_modified_plan(run_tmt: 'RunTmt') -> None:
+    """
+    Verify a plan export is affected by the policy.
+
+    .. note::
+
+        Not doing anything complex, test-level policy tests cover plenty
+        of policy instructions and behavior. Focusing on plan-specific
+        modifications only.
+    """
+
+    result = run_tmt(
+        '-vv',
+        'plan',
+        'export',
+        '--policy-file',
+        POLICIES_DIR / 'plan/plan.yaml',
+    )
+
+    assert f"Apply tmt policy '{POLICIES_DIR}/plan/plan.yaml' to plans." in result.stderr
+
+    plans_exported = from_yaml(result.stdout)
+
+    assert jq_all(plans_exported, '.[] | .discover') == [None], "Verify that discover key is empty"
+
+    assert jq_all(plans_exported, '.[] | .prepare | .[] | [.how, .order]') == [
+        ['feature', 17],
+        ['shell', None],
+    ], 'Verify that prepare step contains two phases'
+
+    assert jq_all(plans_exported, '.[] | .contact') == [["xyzzy"]], (
+        'Verify that contact key was populated'
+    )
+
+
+@with_cwd(DATA_DIR)
+def test_run_modified_plan(run_tmt: 'RunTmt') -> None:
+    """
+    Verify a run is affected by the policy.
+    """
+
+    result = run_tmt('-vv', 'run', '-a', '--policy-file', POLICIES_DIR / 'plan/simple.yaml')
+
+    assert result.exit_code == tmt.cli.TmtExitCode.NO_RESULTS_FOUND
+    assert f"Apply tmt policy '{POLICIES_DIR}/plan/simple.yaml' to plans." in result.stderr
+    assert "No tests found, finishing plan." in result.stderr
diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py
index 78b788bde3..336ad27c9e 100644
--- a/tests/unit/test_cli.py
+++ b/tests/unit/test_cli.py
@@ -38,9 +38,9 @@ def test_mini(run_tmt: 'RunTmt'):
     tmp = tempfile.mkdtemp()
     result = run_tmt('--root', example('mini'), 'run', '-i', tmp, '-dv', 'discover')
     assert result.exit_code == 0
-    assert 'Found 1 plan.' in result.output
-    assert '1 test selected' in result.output
-    assert '/ci' in result.output
+    assert 'Found 1 plan.' in result.stderr
+    assert '1 test selected' in result.stderr
+    assert '/ci' in result.stderr
     shutil.rmtree(tmp)
 
 
@@ -111,8 +111,8 @@ def test_step(run_tmt: 'RunTmt'):
         tmp = tempfile.mkdtemp()
         result = run_tmt('--feeling-safe', '--root', example('local'), 'run', '-i', tmp, step)
         assert result.exit_code == 0
-        assert step in result.output
-        assert 'finish' not in result.output
+        assert step in result.stderr
+        assert 'finish' not in result.stderr
         shutil.rmtree(tmp)
 
 
@@ -132,8 +132,8 @@ def test_step_execute(run_tmt: 'RunTmt'):
     # As we started using 'from' everywhere, '__cause__' must be set
     assert result.exception.__cause__ is not None
     assert isinstance(result.exception.__cause__, tmt.utils.ExecuteError)
-    assert step in result.output
-    assert 'provision' not in result.output
+    assert step in result.stderr
+    assert 'provision' not in result.stderr
     shutil.rmtree(tmp)
 
 
diff --git a/tmt/_compat/importlib/metadata.py b/tmt/_compat/importlib/metadata.py
index 2cda1c3f26..7d21ffdc0e 100644
--- a/tmt/_compat/importlib/metadata.py
+++ b/tmt/_compat/importlib/metadata.py
@@ -1,10 +1,14 @@
 import sys
 
 if sys.version_info >= (3, 10):
-    from importlib.metadata import entry_points
+    from importlib.metadata import entry_points, version
 else:
-    from importlib_metadata import entry_points  # pyright: ignore[reportUnknownVariableType]
+    from importlib_metadata import (
+        entry_points,  # pyright: ignore[reportUnknownVariableType]
+        version,
+    )
 
 __all__ = [
     "entry_points",
+    "version",
 ]