ci.yml

name: CI

on:
  pull_request:
    branches: [ main ]
  push:
    branches: [ main ]
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

permissions: read-all

env:
  REGISTRY_NAME: ${{ secrets.ACR_NAME }}.azurecr.io
  TAG: gha${{ github.run_number }}

jobs:
  lint:
    name: Linting
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v6
        with:
          persist-credentials: false

      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: '3.11'

      - name: Install ruff
        run: pip install ruff

      - name: Run ruff check
        run: ruff check

  dependency-review:
    name: Dependency Review
    runs-on: ubuntu-latest
    if: github.event_name == 'pull_request'
    permissions:
      contents: read
      pull-requests: write
    steps:
      - name: Checkout code
        uses: actions/checkout@v6
        with:
          persist-credentials: false

      - name: Dependency Review
        uses: actions/dependency-review-action@v4
        with:
          fail-on-severity: low
          allow-licenses: MIT, Apache-2.0, BSD-3-Clause
          comment-summary-in-pr: on-failure

  test:
    name: Test ${{ matrix.component.name }} (Python ${{ matrix.python-version }})
    runs-on: ubuntu-latest
    permissions:
      contents: write
      pull-requests: write
    strategy:
      matrix:
        python-version: ['3.10', '3.11', '3.12', '3.13']
        component: 
          - name: 'Analyzer'
            path: 'presidio-analyzer'
            extras: '--all-extras'
            spacy-models: 'en_core_web_lg en_core_web_sm'
          - name: 'Anonymizer'
            path: 'presidio-anonymizer'
            extras: ''
            spacy-models: ''
          - name: 'Image Redactor'
            path: 'presidio-image-redactor'
            extras: ''
            spacy-models: 'en_core_web_lg'
          - name: 'CLI'
            path: 'presidio-cli'
            extras: ''
            spacy-models: ''
          - name: 'Structured'
            path: 'presidio-structured'
            extras: ''
            spacy-models: ''
    env:
      POETRY_CACHE_DIR: /mnt/poetry_cache
      COVERAGE_THRESHOLD: 90
      PRIMARY_PYTHON: '3.13'
    steps:
      - name: Checkout code
        uses: actions/checkout@v6
        with:
          persist-credentials: true
          fetch-depth: 0

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v6
        with:
          python-version: ${{ matrix.python-version }}
          cache: 'pip'
          cache-dependency-path: ${{ matrix.component.path }}/pyproject.toml

      - name: Install Poetry
        run: |
          python -m pip install --upgrade pip
          python -m pip install poetry

      - name: Setup Poetry Cache Directory
        run: |
          sudo mkdir -p $POETRY_CACHE_DIR
          sudo chown -R runner $POETRY_CACHE_DIR    

      - name: Install system dependencies for Image Redactor
        if: matrix.component.name == 'Image Redactor'
        run: |
          sudo apt-get update
          sudo apt-get install tesseract-ocr -y

      - name: Install dependencies
        working-directory: ${{ matrix.component.path }}
        run: |
          poetry install ${{ matrix.component.extras }}

      - name: Install presidio-analyzer for Image Redactor
        if: matrix.component.name == 'Image Redactor'
        working-directory: ${{ matrix.component.path }}
        run: |
          poetry run pip install -e ../presidio-analyzer/.

      - name: Download spaCy models
        if: matrix.component.spacy-models != ''
        working-directory: ${{ matrix.component.path }}
        run: |
          for model in ${{ matrix.component.spacy-models }}; do
            poetry run python -m spacy download $model
          done

      - name: Run tests with coverage
        working-directory: ${{ matrix.component.path }}
        run: |
          PACKAGE_NAME=$(echo "${{ matrix.component.path }}" | sed 's/-/_/g')
          poetry run pytest --cov=$PACKAGE_NAME --cov-report=xml:coverage.xml --cov-report=term --cov-report=html -vv --tb=short
          
          if [ "${{ github.event_name }}" = "pull_request" ]; then
            echo "Checking PR difference coverage (>= ${{ env.COVERAGE_THRESHOLD }}%) with diff-cover..."
            poetry run diff-cover coverage.xml \
              --compare-branch=origin/${{ github.base_ref }} \
              --fail-under=${{ env.COVERAGE_THRESHOLD }} \
              --show-uncovered
          fi

      - name: Comment PR with Coverage
        if: matrix.python-version == env.PRIMARY_PYTHON
        uses: py-cov-action/python-coverage-comment-action@v3
        with:
          GITHUB_TOKEN: ${{ github.token }}
          MINIMUM_GREEN: 85
          MINIMUM_ORANGE: 70
          COVERAGE_DATA_BRANCH: coverage-data
          SUBPROJECT_ID: ${{ matrix.component.path }}
          COVERAGE_PATH: ${{ matrix.component.path }}

      - name: Build wheel package
        working-directory: ${{ matrix.component.path }}
        run: |
          pip install build
          python -m build --wheel

  build-platform-images:
    name: Build ${{ matrix.image }} (${{ matrix.platform }})
    runs-on: ${{ matrix.runner }}
    needs: [lint]
    if: github.ref == 'refs/heads/main'
    permissions:
      id-token: write
      contents: read
    strategy:
      matrix:
        include:
          - image: presidio-anonymizer
            platform: linux/amd64
            runner: ubuntu-latest
          - image: presidio-analyzer
            platform: linux/amd64
            runner: ubuntu-latest
          - image: presidio-image-redactor
            platform: linux/amd64
            runner: ubuntu-latest
          - image: presidio-anonymizer
            platform: linux/arm64
            runner: ubuntu-24.04-arm
          - image: presidio-analyzer
            platform: linux/arm64
            runner: ubuntu-24.04-arm
          - image: presidio-image-redactor
            platform: linux/arm64
            runner: ubuntu-24.04-arm
    steps:
      - name: Checkout code
        uses: actions/checkout@v6
        with:
          persist-credentials: false

      - name: Azure Login using OIDC
        uses: azure/login@v2
        with:
          client-id: ${{ secrets.AZURE_CLIENT_ID }}
          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}

      - name: Log in to Azure Container Registry
        run: az acr login --name ${{ secrets.ACR_NAME }}

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Build and Push ${{ matrix.image }} for ${{ matrix.platform }}
        run: |
          # Create platform-specific tag
          PLATFORM_TAG=$(echo "${{ matrix.platform }}" | sed 's/\//-/g')
          docker buildx build \
            --platform ${{ matrix.platform }} \
            --push \
            --tag ${{ env.REGISTRY_NAME }}/${{ matrix.image }}:${{ env.TAG }}-${PLATFORM_TAG} \
            --cache-from type=registry,ref=${{ env.REGISTRY_NAME }}/public/${{ matrix.image }}:latest \
            --cache-to type=inline \
            --attest type=sbom \
            --attest type=provenance,mode=max \
            ./${{ matrix.image }}
        env:
          REGISTRY_NAME: ${{ env.REGISTRY_NAME }}
          TAG: ${{ env.TAG }}

  create-manifests:
    name: Create Multi-Platform Manifests
    runs-on: ubuntu-latest
    needs: build-platform-images
    if: github.ref == 'refs/heads/main'
    permissions:
      id-token: write
      contents: read
    steps:
      - name: Azure Login using OIDC
        uses: azure/login@v2
        with:
          client-id: ${{ secrets.AZURE_CLIENT_ID }}
          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}

      - name: Log in to Azure Container Registry
        run: az acr login --name ${{ secrets.ACR_NAME }}

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Create all multi-platform manifests
        run: |
          IMAGES=("presidio-anonymizer" "presidio-analyzer" "presidio-image-redactor")
          
          for image in "${IMAGES[@]}"; do
            echo "Creating manifest for $image"
            docker buildx imagetools create \
              --tag ${{ env.REGISTRY_NAME }}/${image}:${{ env.TAG }} \
              ${{ env.REGISTRY_NAME }}/${image}:${{ env.TAG }}-linux-amd64 \
              ${{ env.REGISTRY_NAME }}/${image}:${{ env.TAG }}-linux-arm64
          done
        env:
          REGISTRY_NAME: ${{ env.REGISTRY_NAME }}
          TAG: ${{ env.TAG }}

  e2e-tests:
    name: E2E Tests (${{ matrix.platform }})
    runs-on: ${{ matrix.runner }}
    needs: create-manifests
    if: github.ref == 'refs/heads/main'
    permissions:
      id-token: write
      contents: read
    strategy:
      matrix:
        include:
          - platform: linux/amd64
            runner: ubuntu-latest
          - platform: linux/arm64
            runner: ubuntu-24.04-arm
    steps:
      - name: Checkout code
        uses: actions/checkout@v6
        with:
          persist-credentials: false

      - name: Azure Login using OIDC
        uses: azure/login@v2
        with:
          client-id: ${{ secrets.AZURE_CLIENT_ID }}
          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}

      - name: Log in to Azure Container Registry
        run: az acr login --name ${{ secrets.ACR_NAME }}

      - name: Pull Presidio images from ACR
        run: |
          export DOCKER_DEFAULT_PLATFORM=${{ matrix.platform }}
          docker compose -f docker-compose.yml pull
        env:
          REGISTRY_NAME: ${{ env.REGISTRY_NAME }}
          TAG: :${{ env.TAG }}

      - name: Start Presidio cluster
        run: |
          export DOCKER_DEFAULT_PLATFORM=${{ matrix.platform }}
          docker compose -f docker-compose.yml up -d
        env:
          REGISTRY_NAME: ${{ env.REGISTRY_NAME }}
          TAG: :${{ env.TAG }}

      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: '3.10'

      - name: Cache E2E dependencies
        uses: actions/cache@v5
        with:
          path: |
            ~/.cache/pip
            e2e-tests/env
          key: ${{ runner.os }}-${{ matrix.platform }}-e2e-${{ hashFiles('e2e-tests/requirements.txt') }}
          restore-keys: |
            ${{ runner.os }}-${{ matrix.platform }}-e2e-

      - name: Install E2E test dependencies
        working-directory: e2e-tests
        run: |
          python -m venv env
          source env/bin/activate
          python -m pip install --upgrade pip
          pip install -r requirements.txt
          python -m spacy download en_core_web_lg

      - name: Wait for services to be ready
        run: sleep 60

      - name: Download Ollama model for Ollama LangExtract recognizer tests
        run: |
          docker exec $(docker ps -qf "name=ollama") ollama pull qwen2.5:1.5b
          docker exec $(docker ps -qf "name=ollama") ollama run qwen2.5:1.5b

      - name: Run E2E tests
        working-directory: e2e-tests
        run: |
          source env/bin/activate
          pytest -vv --tb=short

      - name: Capture Docker logs on failure (${{ matrix.platform }})
        if: failure()
        run: |
          echo "Capturing Docker logs for platform: ${{ matrix.platform }}"
          docker compose -f docker-compose.yml logs

      - name: Stop Docker containers
        if: always()
        run: |
          docker compose -f docker-compose.yml down

  # Local build and test jobs for external contributors (no secrets required) 
  local-build-and-test:
    name: Local Build and E2E Tests (${{ matrix.platform }})
    runs-on: ${{ matrix.runner }}
    needs: [lint]
    if: github.ref != 'refs/heads/main'
    strategy:
      matrix:
        include:
          - platform: linux/amd64
            runner: ubuntu-latest
          - platform: linux/arm64
            runner: ubuntu-24.04-arm
    steps:
      - name: Checkout code
        uses: actions/checkout@v6
        with:
          persist-credentials: false

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Build Presidio images locally
        run: |
          # Build all images locally without pushing
          docker compose -f docker-compose.yml build
        env:
          REGISTRY_NAME: 'mcr.microsoft.com'
          IMAGE_PREFIX: ''
          TAG: gha${{ github.run_number }}

      - name: Start Presidio cluster locally
        run: |
          docker compose -f docker-compose.yml up -d
        env:
          REGISTRY_NAME: 'mcr.microsoft.com'
          IMAGE_PREFIX: ''
          TAG: gha${{ github.run_number }}

      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: '3.10'

      - name: Cache E2E dependencies
        uses: actions/cache@v5
        with:
          path: |
            ~/.cache/pip
            e2e-tests/env
          key: ${{ runner.os }}-${{ matrix.platform }}-local-e2e-${{ hashFiles('e2e-tests/requirements.txt') }}
          restore-keys: |
            ${{ runner.os }}-${{ matrix.platform }}-local-e2e-

      - name: Install E2E test dependencies
        working-directory: e2e-tests
        run: |
          python -m venv env
          source env/bin/activate
          python -m pip install --upgrade pip
          pip install -r requirements.txt
          python -m spacy download en_core_web_lg

      - name: Wait for services to be ready
        run: sleep 60

      - name: Download Ollama model for Ollama LangExtract recognizer tests
        run: |
          docker exec $(docker ps -qf "name=ollama") ollama pull qwen2.5:1.5b
          docker exec $(docker ps -qf "name=ollama") ollama run qwen2.5:1.5b

      - name: Run E2E tests
        working-directory: e2e-tests
        run: |
          source env/bin/activate
          pytest -vv --tb=short

      - name: Capture Docker logs on failure
        if: failure()
        run: |
          echo "Capturing Docker logs for local build"
          docker compose -f docker-compose.yml logs

      - name: Stop Docker containers
        if: always()
        run: |
          docker compose -f docker-compose.yml down