topic/ci/source test db

.github/workflows/db.yaml

@@ -0,0 +1,117 @@
+name: Create Test and Source DB
+
+on:
+  push:
+    branches:
+      - main
+      - v*-branch
+      - collab-*
+  pull_request:
+    branches:
+      - main
+      - v*-branch
+      - collab-*
+  schedule:
+    # Run at 17:00 UTC on every Saturday
+    - cron: '0 17 * * 6'
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.head_ref || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test-source-db:
+    runs-on: ubuntu-24.04
+    strategy:
+      fail-fast: false
+      matrix:
+        subset: [1,2,3,4,5,6,7,8,9,10 ,11,12,13,14,15,16,17,18,19,20]
+    timeout-minutes: 1440
+    env:
+      TWISTER_COMMON: ' --test-config tests/test_config_ci.yaml --no-detailed-test-id --force-color --inline-logs -v -N -M --cmake-only -j 8'
+      COMMIT_RANGE: ${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }}
+      BASE_REF: ${{ github.base_ref }}
+      LLVM_TOOLCHAIN_PATH: /usr/lib/llvm-16
+    steps:
+      - name: Checkout
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          path: zephyr
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Set Up Python 3.12
+        uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
+        with:
+          python-version: 3.12
+          cache: pip
+          cache-dependency-path: scripts/requirements-actions.txt
+
+      - name: install-packages
+        working-directory: zephyr
+        run: |
+          pip install -r scripts/requirements-actions.txt --require-hashes
+          sudo apt-get update -y
+          sudo apt-get install -y lcov gperf
+
+      - name: Setup Zephyr project
+        uses: zephyrproject-rtos/action-zephyr-setup@f7b70269a8eb01f70c8e710891e4c94972a2f6b4 # v1.0.6
+        with:
+          app-path: zephyr
+          toolchains: all
+
+      - name: Run Tests with Twister
+        working-directory: zephyr
+        id: run_twister
+        run: |
+          export ZEPHYR_BASE=${PWD}
+          export ZEPHYR_TOOLCHAIN_VARIANT=zephyr
+          ./scripts/twister --subset ${{matrix.subset}}/${{ strategy.job-total }} ${TWISTER_COMMON} ${PUSH_OPTIONS}
+          python ./scripts/ci/db/gen_test_database.py --directory twister-out/ --output db_${{matrix.subset}}.json
+
+      - name: Upload Database
+        if: always()
+        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+        with:
+          name: Test and Source DB (Subset ${{ matrix.subset }})
+          if-no-files-found: ignore
+          path: |
+            zephyr/db_${{matrix.subset}}.json
+
+  test-db-results:
+    name: "Publish Merged DB"
+    needs:
+      - test-source-db
+    runs-on: ubuntu-22.04
+    permissions:
+      checks: write # to create the check run entry with Twister test results
+    # the build-and-test job might be skipped, we don't need to run this job then
+    if: success() || failure()
+
+    steps:
+      - name: Install packages
+        run: |
+          sudo apt-get update -y
+          sudo apt install -y jq
+
+      - name: Download Artifacts
+        uses: actions/download-artifact@cc203385981b70ca67e1cc392babf9cc229d5806 # v4.1.9
+        with:
+          path: artifacts
+
+      - name: Merge DB files
+        run: |
+          jq -s '.' artifacts/*/*.json > db.json
+
+      - name: Upload Merged DB
+        if: always()
+        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+        with:
+          name: Merged DB
+          if-no-files-found: ignore
+          path: |
+            db.json

.github/workflows/footprint-tracking.yml

@@ -25,12 +25,8 @@ concurrency:
 
 jobs:
   footprint-tracking:
-    runs-on:
-      group: zephyr-runner-v2-linux-x64-4xlarge
+    runs-on: ubuntu-24.04
     if: github.repository_owner == 'zephyrproject-rtos'
-    container:
-      image: ghcr.io/zephyrproject-rtos/ci-repo-cache:v0.27.4.20241026
-      options: '--entrypoint /bin/bash'
     defaults:
       run:
         shell: bash
@@ -39,32 +35,10 @@ jobs:
     env:
       ZEPHYR_TOOLCHAIN_VARIANT: zephyr
     steps:
-      - name: Apply container owner mismatch workaround
-        run: |
-          # FIXME: The owner UID of the GITHUB_WORKSPACE directory may not
-          #        match the container user UID because of the way GitHub
-          #        Actions runner is implemented. Remove this workaround when
-          #        GitHub comes up with a fundamental fix for this problem.
-          git config --global --add safe.directory ${GITHUB_WORKSPACE}
-
-      - name: Print cloud service information
-        run: |
-          echo "ZEPHYR_RUNNER_CLOUD_PROVIDER = ${ZEPHYR_RUNNER_CLOUD_PROVIDER}"
-          echo "ZEPHYR_RUNNER_CLOUD_NODE = ${ZEPHYR_RUNNER_CLOUD_NODE}"
-          echo "ZEPHYR_RUNNER_CLOUD_POD = ${ZEPHYR_RUNNER_CLOUD_POD}"
-
-      - name: Update PATH for west
-        run: |
-          echo "$HOME/.local/bin" >> $GITHUB_PATH
-
-      - name: Install packages
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y python3-venv
-
       - name: checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
+          path: zephyr
           ref: ${{ github.event.pull_request.head.sha }}
           fetch-depth: 0
 
@@ -76,40 +50,43 @@ jobs:
           cache-dependency-path: scripts/requirements-actions.txt
 
       - name: Install Python packages
+        working-directory: zephyr
         run: |
             pip install -r scripts/requirements-actions.txt --require-hashes
 
-      - name: Environment Setup
-        run: |
-          echo "ZEPHYR_SDK_INSTALL_DIR=/opt/toolchains/zephyr-sdk-$( cat SDK_VERSION )" >> $GITHUB_ENV
-
-      - name: west setup
-        run: |
-          west init -l . || true
-          west config --global update.narrow true
-          west update 2>&1 1> west.update.log || west update 2>&1 1> west.update2.log
-
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 # v4.1.0
         with:
           aws-access-key-id: ${{ vars.AWS_TESTING_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_TESTING_SECRET_ACCESS_KEY }}
           aws-region: us-east-1
 
+      - name: Setup Zephyr project
+        uses: zephyrproject-rtos/action-zephyr-setup@f7b70269a8eb01f70c8e710891e4c94972a2f6b4 # v1.0.6
+        with:
+          app-path: zephyr
+          toolchains: all
+
+      - name: Install additional packages
+        run: |
+          sudo apt-get update -y
+          sudo apt-get install -y gperf
+
       - name: Record Footprint
+        working-directory: zephyr
         env:
           BASE_REF: ${{ github.base_ref }}
         run: |
           export ZEPHYR_BASE=${PWD}
           ./scripts/footprint/track.py  -p scripts/footprint/plan.txt
 
       - name: Upload footprint data
+        working-directory: zephyr
         run: |
-          python3 -m venv .venv
-          . .venv/bin/activate
           aws s3 sync  --quiet footprint_data/ s3://testing.zephyrproject.org/footprint_data/
 
       - name: Transform Footprint data to Twister JSON reports
+        working-directory: zephyr
         run: |
           shopt -s globstar
           export ZEPHYR_BASE=${PWD}
@@ -119,6 +96,7 @@ jobs:
             ./footprint_data/**/
 
       - name: Upload to ElasticSearch
+        working-directory: zephyr
         env:
           ELASTICSEARCH_KEY: ${{ secrets.ELASTICSEARCH_KEY }}
           ELASTICSEARCH_SERVER: "https://elasticsearch.zephyrproject.io:443"

.github/workflows/manifest.yml

@@ -16,7 +16,7 @@ jobs:
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           path: zephyrproject/zephyr
-          ref: ${{ github.event.pull_request.head.sha }}
+          #ref: ${{ github.event.pull_request.head.sha }}
           fetch-depth: 0
           persist-credentials: false
 

.github/workflows/twister-prep.yaml

@@ -20,7 +20,7 @@ jobs:
   prep_pr:
     if: github.repository_owner == 'zephyrproject-rtos' && github.event_name == 'pull_request'
     runs-on:
-      group: zephyr-runner-v2-linux-x64-4xlarge
+      group: test-runner-v2-linux-x64-4xlarge
     container:
       image: ghcr.io/zephyrproject-rtos/ci-repo-cache:v0.27.4.20241026
       options: '--entrypoint /bin/bash'

.github/workflows/twister.yaml

@@ -28,7 +28,7 @@ jobs:
 
   twister-build:
     runs-on:
-      group: zephyr-runner-v2-linux-x64-4xlarge
+      group: test-runner-v2-linux-x64-4xlarge
     needs: twister-build-prep
     if: needs.twister-build-prep.outputs.size != 0
     container:
@@ -168,13 +168,7 @@ jobs:
         run: |
           export ZEPHYR_BASE=${PWD}
           export ZEPHYR_TOOLCHAIN_VARIANT=zephyr
-          ./scripts/twister --subset ${{matrix.subset}}/${{ strategy.job-total }} ${TWISTER_COMMON} ${WEEKLY_OPTIONS}
-          if [ "${{matrix.subset}}" = "1" ]; then
-            ./scripts/zephyr_module.py --twister-out module_tests.args
-            if [ -s module_tests.args ]; then
-              ./scripts/twister +module_tests.args --outdir module_tests ${TWISTER_COMMON} ${WEEKLY_OPTIONS}
-            fi
-          fi
+          ./scripts/twister --cmake-only --subset ${{matrix.subset}}/${{ strategy.job-total }} ${TWISTER_COMMON} ${WEEKLY_OPTIONS}
 
       - name: Print ccache stats
         if: always()

scripts/ci/db/find_test_combo.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+
+import json
+import os
+import re
+import sys
+from pathlib import Path
+from collections import defaultdict
+import argparse
+
+if "ZEPHYR_BASE" not in os.environ:
+    exit("$ZEPHYR_BASE environment variable undefined.")
+
+# These are globally used variables. They are assigned in __main__ and are visible in further methods
+# however, pylint complains that it doesn't recognize them when used (used-before-assignment).
+zephyr_base = Path(os.environ['ZEPHYR_BASE'])
+
+sys.path.insert(0, os.path.join(zephyr_base / "scripts"))
+from get_maintainer import Maintainers
+
+def load_database(database_path):
+    with open(database_path, 'r') as file:
+        return json.load(file)
+
+def find_areas(files):
+    maintf = zephyr_base / "MAINTAINERS.yml"
+    maintainer_file = Maintainers(maintf)
+
+    num_files = 0
+    all_areas = set()
+
+    for changed_file in files:
+        num_files += 1
+        print(f"file: {changed_file}")
+        areas = maintainer_file.path2areas(changed_file)
+
+        if not areas:
+            continue
+        all_areas.update(areas)
+    tests = []
+    for area in all_areas:
+        for suite in area.tests:
+            tests.append(f"{suite}.*")
+    return tests
+
+def find_best_coverage(database, changed_files, tests):
+    coverage = defaultdict(lambda: defaultdict(int))
+
+    for file in changed_files:
+        if file in database:
+            for entry in database[file]:
+                if not any(re.search(f"^{test}", entry["testsuite_id"]) for test in tests):
+                    print("skip")
+                    continue
+                testsuite_id = entry["testsuite_id"]
+                platform = entry["platform"]
+                coverage[testsuite_id][platform] += 1
+
+    best_coverage = []
+    for testsuite_id, platforms in coverage.items():
+        for platform, count in platforms.items():
+            best_coverage.append((testsuite_id, platform, count))
+
+    best_coverage.sort(key=lambda x: x[2], reverse=True)
+    return best_coverage
+
+def main(database_path, changed_files):
+    tests = find_areas(changed_files)
+    database = load_database(database_path)
+    best_coverage = find_best_coverage(database, changed_files, tests)
+
+    if best_coverage:
+        print("Best coverage testsuites and platforms:")
+        for testsuite_id, platform, count in best_coverage:
+            print(f"Testsuite: {testsuite_id}, Platform: {platform}, Coverage: {count}")
+    else:
+        print("No matching testsuites found for the provided files.")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Find the best coverage testsuites and platforms for changed files.")
+    parser.add_argument("--database", help="Path to the testsuite database JSON file")
+    parser.add_argument("--changed-files", action="append", help="List of changed files")
+
+    args = parser.parse_args()
+
+    main(args.database, args.changed_files)