Merge pull request sherlock-project#2548 from sherlock-project/feature/automatic-testing

Automatically test modified targets upon PR

Author: ppfeister

.github/workflows/validate_modified_targets.yml

@@ -0,0 +1,64 @@
+name: Modified Target Validation
+
+on:
+  pull_request:
+    branches:
+      - master
+    paths:
+      - "sherlock_project/resources/data.json"
+
+jobs:
+  validate-modified-targets:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: '3.13'
+
+      - name: Install Poetry
+        uses: abatilo/actions-poetry@v4
+        with:
+          poetry-version: 'latest'
+
+      - name: Install dependencies
+        run: |
+          poetry install --no-interaction --with dev
+
+      - name: Discover modified targets
+        id: discover-modified
+        run: |
+          # Fetch the upstream branch
+          git fetch origin ${{ github.base_ref }} --depth=1
+
+          # Discover changes
+          git show origin/${{ github.base_ref }}:sherlock_project/resources/data.json > data.json.base
+          CHANGED=$(
+            jq --slurpfile base data.json.base --slurpfile head sherlock_project/resources/data.json '
+              [
+                ($head[0] | keys_unsorted[]) as $key
+                | select(($base[0][$key] != $head[0][$key]) or ($base[0][$key] | not))
+                | $key
+              ] | unique | join(",")'
+          )
+
+          # Preserve changelist
+          echo ">>> Changed targets: \n$(echo $CHANGED | tr ',' '\n')"
+          echo "changed_targets=$CHANGED" >> "$GITHUB_OUTPUT"
+
+      - name: Validate modified targets
+        if: steps.discover-modified.outputs.changed_targets != ''
+        run: |
+          $(poetry env activate)
+          pytest -q --tb no -rA -m validate_targets -n 20 --chunked-sites "${{ steps.discover-modified.outputs.changed_targets }}"
+          deactivate
+
+      - name: Announce skip if no modified targets
+        if: steps.discover-modified.outputs.changed_targets == ''
+        run: |
+          echo "No modified targets found"

tests/conftest.py

@@ -6,7 +6,7 @@
 
 def fetch_local_manifest(honor_exclusions: bool = True) -> dict[str, dict[str, str]]:
     sites_obj = SitesInformation(data_file_path=os.path.join(os.path.dirname(__file__), "../sherlock_project/resources/data.json"), honor_exclusions=honor_exclusions)
-    sites_iterable = {site.name: site.information for site in sites_obj}
+    sites_iterable: dict[str, dict[str, str]] = {site.name: site.information for site in sites_obj}
     return sites_iterable
 
 @pytest.fixture()
@@ -25,9 +25,27 @@ def remote_schema():
         schemadat = json.load(remoteschema)
     yield schemadat
 
+def pytest_addoption(parser):
+    parser.addoption(
+        "--chunked-sites",
+        action="store",
+        default=None,
+        help="For tests utilizing chunked sites, include only the (comma-separated) site(s) specified.",
+    )
+
 def pytest_generate_tests(metafunc):
     if "chunked_sites" in metafunc.fixturenames:
         sites_info = fetch_local_manifest(honor_exclusions=False)
+
+        # Ingest and apply site selections
+        site_filter: str | None = metafunc.config.getoption("--chunked-sites")
+        if site_filter:
+            selected_sites: list[str] = [site.strip() for site in site_filter.split(",")]
+            sites_info = {
+                site: data for site, data in sites_info.items()
+                if site in selected_sites
+            }
+
         params = [{name: data} for name, data in sites_info.items()]
         ids = list(sites_info.keys())
         metafunc.parametrize("chunked_sites", params, ids=ids)