run_python_test: Start tcpdump during test case execution (#43490)

* run_python_test: Start tcpdump during test case execution

Signed-off-by: Maciej Grela <m.grela@samsung.com>

* Some improvements, changes suggested by automated review, try to fix missing tcpdump:

- bump image version
- don't break when pcap is already deleted
- distinguish pcaps from different runs of the same TC
- use non-interactive sudo
- make exit_code handling more robust in case of exceptions
- make tcpdump start/stop conditions simpler

Signed-off-by: Maciej Grela <m.grela@samsung.com>

* Restyled by autopep8

* Make sure tcpdump's Subprocess object is None before start() is called

Signed-off-by: Maciej Grela <m.grela@samsung.com>

* Remove stray dash, harmonize upload action version

Signed-off-by: Maciej Grela <m.grela@samsung.com>

* Fixes suggested by automated review

- reverse `keep_dumpfile` logic to ensure dumps are kept in case of any
exceptions
- move pcap upload action to the proper job in tests.yaml

---------

Signed-off-by: Maciej Grela <m.grela@samsung.com>
Co-authored-by: Restyled.io <commits@restyled.io>

Author: enkiusz

.github/workflows/tests.yaml

@@ -61,7 +61,7 @@ jobs:
         runs-on: ubuntu-latest
 
         container:
-            image: ghcr.io/project-chip/chip-build:184
+            image: ghcr.io/project-chip/chip-build:190
             options: >-
                 --privileged --sysctl "net.ipv6.conf.all.disable_ipv6=0
                 net.ipv4.conf.all.forwarding=1 net.ipv6.conf.all.forwarding=1"
@@ -796,7 +796,7 @@ jobs:
         runs-on: namespace-profile-16x32
 
         container:
-            image: ghcr.io/project-chip/chip-build:181
+            image: ghcr.io/project-chip/chip-build:190
             volumes:
                 - /var/run/nsc/:/var/run/nsc/
             env:
@@ -1054,12 +1054,13 @@ jobs:
             BUILD_VARIANT: ipv6only-no-ble-no-wifi
             DISABLE_CCACHE: ${{ (github.event_name == 'workflow_dispatch' && inputs.disable_ccache == 'true') && 'true' || (contains(github.event.head_commit.message, '[no-ccache]') && 'true') || 'false' }}
             CCACHE_KEY_SUFFIX: ${{ github.event_name == 'workflow_dispatch' && inputs.cache_suffix || '' }}
+            CHIP_IP_PACKET_CAPTURE: 1
 
         # Tests are I/O bound, so run on a smaller machine
         runs-on: namespace-profile-1x2
 
         container:
-            image: ghcr.io/project-chip/chip-build:182
+            image: ghcr.io/project-chip/chip-build:190
             volumes:
                 - /var/run/nsc/:/var/run/nsc/
             env:
@@ -1191,6 +1192,21 @@ jobs:
                   docs/development_controllers/matter-repl/Matter_Multi_Fabric_Commissioning.ipynb \
                   "
 
+            - name: Uploading core files
+              uses: actions/upload-artifact@v7
+              if: ${{ failure() && !env.ACT }}
+              with:
+                  name: crash-core-linux-repl-${{ matrix.filter }}
+                  path: /tmp/cores/
+                  # Cores are big; don't hold on to them too long.
+                  retention-days: 5
+            - name: Uploading packet captures for debugging
+              uses: actions/upload-artifact@v7
+              if: ${{ failure() && !env.ACT }}
+              with:
+                  name: pcaps-linux-repl-${{ matrix.filter }}
+                  path: out/ip_packet_captures
+
             # Example debug breakpoint:
             #
             # - name: Breakpoint if tests failed

scripts/tests/run_python_test.py

@@ -18,6 +18,7 @@
 import dataclasses
 import datetime
 import enum
+import getpass
 import glob
 import io
 import logging
@@ -47,6 +48,7 @@
 
 MATTER_DEVELOPMENT_PAA_ROOT_CERTS = "credentials/development/paa-root-certs"
 
+TAG_PROCESS_MON = f"[{Fore.GREEN}MON {Style.RESET_ALL}]".encode()
 TAG_PROCESS_APP = f"[{Fore.GREEN}APP {Style.RESET_ALL}]".encode()
 TAG_PROCESS_TEST = f"[{Fore.GREEN}TEST{Style.RESET_ALL}]".encode()
 TAG_STDOUT = f"[{Fore.YELLOW}STDOUT{Style.RESET_ALL}]".encode()
@@ -70,6 +72,10 @@ def process_chip_output(line: bytes, is_stderr: bool, process_tag: bytes = b"")
     return f"[{timestamp}]".encode() + process_tag + (TAG_STDERR if is_stderr else TAG_STDOUT) + line
 
 
+def process_mon_output(line, is_stderr):
+    return process_chip_output(line, is_stderr, TAG_PROCESS_MON)
+
+
 def process_chip_app_output(line, is_stderr):
     return process_chip_output(line, is_stderr, TAG_PROCESS_APP)
 
@@ -150,6 +156,33 @@ def get_process(self):
         return self.app_process
 
 
+class IpPacketCaptureManager():
+    def __init__(self, dump_filename: pathlib.Path):
+        self.tcpdump_process = None
+        self.dump_filename = dump_filename
+        self.interface = 'any'
+        self.keep_dumpfile = True
+
+    def start(self):
+        # Create directory for dump files
+        self.dump_filename.parent.mkdir(parents=True, exist_ok=True)
+
+        cmd = ['tcpdump', '-qn', '-i', self.interface, '-w', str(self.dump_filename), '-Z', getpass.getuser()]
+        if os.getuid() != 0:
+            cmd = ['sudo', '-n'] + cmd
+        self.tcpdump_process = Subprocess(cmd[0], *cmd[1:], output_cb=process_mon_output)
+
+        self.tcpdump_process.start()
+
+    def stop(self):
+        if self.tcpdump_process:
+            self.tcpdump_process.terminate()
+            self.tcpdump_process = None
+        if not self.keep_dumpfile:
+            log.info("Deleting capture file '%s'", self.dump_filename)
+            self.dump_filename.unlink(missing_ok=True)
+
+
 @click.command()
 @click.option("--app", type=click.Path(exists=True), default=None,
               help='Path to local application to use, omit to use external apps.')
@@ -178,10 +211,14 @@ def get_process(self):
               help="Do not print output from passing tests. Use this flag in CI to keep GitHub log size manageable.")
 @click.option("--load-from-env", default=None, help="YAML file that contains values for environment variables.")
 @click.option("--run", type=str, multiple=True, help="Run only the specified test run(s).")
+@click.option("--ip-packet-capture/--no-ip-packet-capture", is_flag=True, default=False, help="Enable IP packet capture.")
+@click.option("--ip-packet-capture-dir", type=click.Path(file_okay=False, writable=True, path_type=pathlib.Path),
+              default=pathlib.Path.cwd() / "out/ip_packet_captures", help="Storage for capture files.")
 @click.option("--app-filter", type=str, default=None, help="Run only for the specified app(s). Comma separated.")
 def main(app: str, factory_reset: bool, factory_reset_app_only: bool, app_args: str,
          app_ready_pattern: str, app_stdin_pipe: str, script: str, script_args: str,
-         script_gdb: bool, quiet: bool, load_from_env, run, app_filter):
+         script_gdb: bool, quiet: bool, load_from_env, run, ip_packet_capture: bool, ip_packet_capture_dir: pathlib.Path,
+         app_filter):
     if load_from_env:
         reader = MetadataReader(load_from_env)
         runs = reader.parse_script(script)
@@ -228,12 +265,14 @@ def main(app: str, factory_reset: bool, factory_reset_app_only: bool, app_args:
     for run in runs:
         log.info("Executing '%s' '%s'", run.py_script_path.split('/')[-1], run.run)
         main_impl(run.app, run.factory_reset, run.factory_reset_app_only, run.app_args or "", run.app_ready_pattern,
-                  run.app_stdin_pipe, run.py_script_path, run.script_args or "", run.script_gdb, run.quiet)
+                  run.app_stdin_pipe, run.py_script_path, run.script_args or "", run.script_gdb, ip_packet_capture,
+                  ip_packet_capture_dir, run.quiet, run.run)
 
 
 def main_impl(app: str, factory_reset: bool, factory_reset_app_only: bool, app_args: str,
               app_ready_pattern: str, app_stdin_pipe: str, script: str, script_args: str,
-              script_gdb: bool, quiet: bool):
+              script_gdb: bool, ip_packet_capture: bool, ip_packet_capture_dir: pathlib.Path,
+              quiet: bool, run_name: str):
 
     app_args = app_args.replace('{SCRIPT_BASE_NAME}', os.path.splitext(os.path.basename(script))[0])
     script_args = script_args.replace('{SCRIPT_BASE_NAME}', os.path.splitext(os.path.basename(script))[0])
@@ -242,6 +281,14 @@ def main_impl(app: str, factory_reset: bool, factory_reset_app_only: bool, app_a
     test_run_id = str(uuid.uuid4())[:8]  # Use first 8 characters for shorter paths
     restart_flag_file = f"/tmp/chip_test_restart_app_{test_run_id}"
 
+    script_name = pathlib.Path(script).name.removesuffix('.py')
+    tcpdump_capture_filename = ip_packet_capture_dir / f"tcpdump_{script_name}-{os.getpid()}-{run_name}.pcap"
+
+    tcpdump = IpPacketCaptureManager(pathlib.Path(tcpdump_capture_filename))
+
+    if ip_packet_capture:
+        tcpdump.start()
+
     # Remove app config and storage if factory reset is requested
     if factory_reset or factory_reset_app_only:
         reset_type = FactoryResetType.AppAndController if factory_reset else FactoryResetType.AppOnly
@@ -328,6 +375,10 @@ def main_impl(app: str, factory_reset: bool, factory_reset_app_only: bool, app_a
         # We expect both app and test script should exit with 0
         exit_code = test_script_exit_code or app_exit_code
 
+        if tcpdump and exit_code == 0:
+            # Delete packet captures from successful runs
+            tcpdump.keep_dumpfile = False
+
         if quiet:
             if exit_code:
                 sys.stdout.write(stream_output.getvalue().decode('utf-8', errors='replace'))
@@ -346,6 +397,8 @@ def main_impl(app: str, factory_reset: bool, factory_reset_app_only: bool, app_a
             log.info("Stopping app restart monitor thread")
             restart_monitor_thread.join(2.0)
 
+        tcpdump.stop()
+
         # Clean up any leftover flag files if they exist - ensure this always executes
         log.info("Cleaning up flag files")
         if os.path.exists(restart_flag_file):