BFD-4351: Python CI improvements

.github/workflows/ci-python.yml

@@ -4,7 +4,7 @@ on:
   pull_request:
     paths:
       - "**/*.py"
-      - "**/*requirement*.txt"
+      - "**/pyproject.toml"
       - "**/uv.lock"
       - ".github/workflows/ci-python.yml"
 
@@ -13,65 +13,57 @@ on:
 env:
   BASE_COMMIT: ""
   GH_TOKEN: ${{ github.token }}
+  # Pin tool versions to match workspace configuration
+  PYTHON_VERSION: "3.13"
+  UV_VERSION: "0.5.18"
 
 jobs:
-  matrix-pytest:
+  workspace-checks:
+    name: Workspace Tests & Checks
     runs-on: ubuntu-24.04
-    strategy:
-      matrix:
-        projects: [
-            {
-              projectName: "sftp-outbound-transfer-lambda",
-              projectRoot: "ops/services/02-eft/lambda_src/sftp_outbound_transfer",
-              testsRoot: "ops/services/02-eft/lambda_src/sftp_outbound_transfer/tests",
-              uvCommand: "uv sync",
-              pytestCommand: "uv run pytest",
-              pythonVersion: "3.13",
-            },
-            {
-              projectName: "locust",
-              projectRoot: "apps/utils/locust_tests",
-              testsRoot: "apps/utils/locust_tests",
-              uvCommand: "uv sync --group lambda-run-locust",
-              # no tests
-              pytestCommand: "echo 'no tests'",
-              pythonVersion: "3.13",
-            },
-            {
-              projectName: "idr-pipeline",
-              projectRoot: "apps/bfd-pipeline/bfd-pipeline-idr",
-              testsRoot: "apps/bfd-pipeline/bfd-pipeline-idr",
-              uvCommand: "uv sync",
-              pytestCommand: "uv run pytest",
-              pythonVersion: "3.13",
-            },
-          ]
-      fail-fast: false
     steps:
       - name: Checkout code
         id: code-checkout
         uses: actions/checkout@v4
         with:
           ref: ${{ inputs.branch }}
-      # Maven is needed to run the migrator
+
+      # Maven is needed to run the migrator for bfd-pipeline-idr tests
       - name: "Setup JDK"
         uses: actions/setup-java@v4
         with:
           java-version: "25"
           distribution: "corretto"
-      - uses: astral-sh/setup-uv@v6
-      - name: Run Python Tests per project
-        id: run-pytest-set
+
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          version: ${{ env.UV_VERSION }}
+
+      - name: Set up Python
+        run: uv python install ${{ env.PYTHON_VERSION }}
+
+      - name: Install workspace dependencies
+        run: |
+          cd ${{ github.workspace }}/apps
+          uv sync --all-groups
+
+      - name: Run linting (ruff check)
         run: |
-          echo "Testing ${{ matrix.projects.projectName }}"
-          cd ${{ github.workspace }}/${{ matrix.projects.projectRoot }}
-          uv python install
-          ${{ matrix.projects.uvCommand }}
-          # check linting
+          cd ${{ github.workspace }}/apps
           uv run ruff check
-          # check formatting
+
+      - name: Run formatting check (ruff format)
+        run: |
+          cd ${{ github.workspace }}/apps
           uv run ruff format --check
-          # check types
-          uv run pyright .
-          # tests
-          ${{ matrix.projects.pytestCommand }} ${{ github.workspace }}/${{ matrix.projects.testsRoot }}
+
+      - name: Run type checking (pyright)
+        run: |
+          cd ${{ github.workspace }}/apps
+          uv run pyright
+
+      - name: Run tests (pytest)
+        run: |
+          cd ${{ github.workspace }}/apps
+          uv run pytest

apps/bfd-model-idr/claims_generator.py

@@ -12,9 +12,8 @@
 import pandas as pd
 import yaml
 from faker import Faker
-from pydantic import BaseModel, ConfigDict, Field, TypeAdapter
-
 from generator_util import GeneratorUtil
+from pydantic import BaseModel, ConfigDict, Field, TypeAdapter
 
 
 class SecurityLabelModel(BaseModel):
@@ -684,11 +683,13 @@ def gen_claim(bene_sk="-1", min_date="2018-01-01", max_date=str(now)):
     # generate claim header financial elements here
     claim["CLM"]["CLM_SBMT_CHRG_AMT"] = round(random.uniform(1, 1000000), 2)
     if clm_type_cd == 71 or clm_type_cd == 72:
-        claim["CLM"]["CLM_RFRG_PRVDR_PIN_NUM"] = random.choice([
-            9181272397,
-            9181272391,
-            918127239123,
-        ])
+        claim["CLM"]["CLM_RFRG_PRVDR_PIN_NUM"] = random.choice(
+            [
+                9181272397,
+                9181272391,
+                918127239123,
+            ]
+        )
     if clm_type_cd > 70 and clm_type_cd <= 82:
         claim["CLM"]["CLM_ALOWD_CHRG_AMT"] = round(random.uniform(1, 1000000), 2)
         claim["CLM"]["CLM_BENE_PD_AMT"] = round(random.uniform(1, 1000000), 2)
@@ -931,10 +932,12 @@ def gen_claim(bene_sk="-1", min_date="2018-01-01", max_date=str(now)):
 
             if random.randint(0, 10) == 6:
                 claim_line_prfnl["CLM_LINE_HCT_HGB_TYPE_CD"] = random.choice(["R1", "R2"])
-                claim_line_prfnl["CLM_LINE_CARR_CLNCL_LAB_NUM"] = random.choice([
-                    "11D1111111",
-                    "22D2222222",
-                ])
+                claim_line_prfnl["CLM_LINE_CARR_CLNCL_LAB_NUM"] = random.choice(
+                    [
+                        "11D1111111",
+                        "22D2222222",
+                    ]
+                )
 
             # these don't have much variance in our synthetic data, but they are not strictly
             # the same in actual data!
@@ -1024,17 +1027,19 @@ def gen_claim(bene_sk="-1", min_date="2018-01-01", max_date=str(now)):
         claim_line_inst["CLM_REV_CNTR_STUS_CD"] = random.choice(
             generator.code_systems["CLM_REV_CNTR_STUS_CD"]
         )
-        claim_line_inst["CLM_ANSI_SGNTR_SK"] = random.choice([
-            "8585",
-            "1",
-            "4365",
-            "1508",
-            "5555",
-            "9204",
-            "6857",
-            "5816",
-            "11978",
-        ])
+        claim_line_inst["CLM_ANSI_SGNTR_SK"] = random.choice(
+            [
+                "8585",
+                "1",
+                "4365",
+                "1508",
+                "5555",
+                "9204",
+                "6857",
+                "5816",
+                "11978",
+            ]
+        )
         add_meta_timestamps(claim_line_inst, claim["CLM"], max_date)
 
         claim_line["CLM_UNIQ_ID"] = claim["CLM"]["CLM_UNIQ_ID"]
@@ -1101,18 +1106,20 @@ def gen_pac_version_of_claim(claim, max_date):
     pac_claim["CLM_FISS"]["GEO_BENE_SK"] = pac_claim["CLM"]["GEO_BENE_SK"]
     pac_claim["CLM_FISS"]["CLM_NUM_SK"] = pac_claim["CLM"]["CLM_NUM_SK"]
     pac_claim["CLM_FISS"]["CLM_TYPE_CD"] = pac_claim["CLM"]["CLM_TYPE_CD"]
-    pac_claim["CLM_FISS"]["CLM_CRNT_STUS_CD"] = random.choice([
-        "A",
-        "F",
-        "I",
-        "S",
-        "M",
-        "P",
-        "R",
-        "D",
-        "T",
-        "U",
-    ])
+    pac_claim["CLM_FISS"]["CLM_CRNT_STUS_CD"] = random.choice(
+        [
+            "A",
+            "F",
+            "I",
+            "S",
+            "M",
+            "P",
+            "R",
+            "D",
+            "T",
+            "U",
+        ]
+    )
     add_meta_timestamps(pac_claim["CLM_FISS"], claim["CLM"], max_date)
 
     pac_claim["CLM_LCTN_HSTRY"] = {}
@@ -1121,22 +1128,24 @@ def gen_pac_version_of_claim(claim, max_date):
     pac_claim["CLM_LCTN_HSTRY"]["CLM_NUM_SK"] = pac_claim["CLM"]["CLM_NUM_SK"]
     pac_claim["CLM_LCTN_HSTRY"]["CLM_TYPE_CD"] = pac_claim["CLM"]["CLM_TYPE_CD"]
     pac_claim["CLM_LCTN_HSTRY"]["CLM_LCTN_CD_SQNC_NUM"] = "1"
-    pac_claim["CLM_LCTN_HSTRY"]["CLM_AUDT_TRL_STUS_CD"] = random.choice([
-        "A",
-        "F",
-        "I",
-        "S",
-        "M",
-        "P",
-        "R",
-        "D",
-        "T",
-        "U",
-        "1",
-        "2",
-        "4",
-        "8",
-    ])
+    pac_claim["CLM_LCTN_HSTRY"]["CLM_AUDT_TRL_STUS_CD"] = random.choice(
+        [
+            "A",
+            "F",
+            "I",
+            "S",
+            "M",
+            "P",
+            "R",
+            "D",
+            "T",
+            "U",
+            "1",
+            "2",
+            "4",
+            "8",
+        ]
+    )
     add_meta_timestamps(pac_claim["CLM_LCTN_HSTRY"], claim["CLM"], max_date)
 
     for i in range(len(pac_claim["CLM_LINE"])):