test(DATAGO-129265): Migrate repository unit tests to real DB integration tests #2 #4792
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| types: [opened, synchronize] | |
| branches: | |
| - main | |
| - art/** | |
| permissions: | |
| repository-projects: read | |
| contents: write | |
| id-token: write | |
| packages: write | |
| checks: write | |
| pull-requests: write | |
| issues: write # For PR comments from FOSSA | |
| statuses: write # For FOSSA commit status | |
| actions: read # Required by sca-scan-and-guard reusable workflow | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| # ---------------------------------------------------- | |
| # FOSSA Scan | |
| # ---------------------------------------------------- | |
| fossa_scan: | |
| name: FOSSA Scan | |
| if: ${{ !github.event.repository.fork }} | |
| uses: SolaceDev/solace-public-workflows/.github/workflows/sca-scan-and-guard.yaml@main | |
| with: | |
| use_vault: false | |
| config_file: ".github/workflow-config.json" | |
| setup_actions: '["setup-uv"]' | |
| custom_setup_script: "uv export --format requirements-txt --no-dev --output-file requirements.txt" | |
| secrets: | |
| FOSSA_API_KEY: ${{ secrets.FOSSA_API_KEY }} | |
| # ---------------------------------------------------- | |
| # Test and SonarQube Analysis (Reusable Workflow) | |
| # ---------------------------------------------------- | |
| test-and-sonarqube: | |
| name: Test and SonarQube | |
| uses: ./.github/workflows/test-and-sonarqube.yml | |
| with: | |
| min-python-version: "3.10" | |
| max-python-version: "3.13" | |
| node-version: "25.5.0" | |
| ui-path: "client/webui/frontend" | |
| secrets: | |
| SONAR_TOKEN: ${{ secrets.SONARQUBE_TOKEN }} | |
| SONAR_HOST_URL: ${{ secrets.SONARQUBE_HOST_URL }} | |
| # ---------------------------------------------------- | |
| # PHASE 0: Prepare Metadata and Determine Build Strategy Test1 | |
| # ---------------------------------------------------- | |
| prepare-metadata: | |
| name: Prepare Build Metadata | |
| runs-on: ubuntu-latest | |
| outputs: | |
| version: ${{ steps.get_version.outputs.version }} | |
| short_sha: ${{ steps.get_commit_hash.outputs.short_sha }} | |
| commit_hash: ${{ steps.get_commit_hash.outputs.commit_hash }} | |
| release_tag: ${{ steps.release_tag.outputs.release_tag }} | |
| docker_push: ${{ steps.set_build_params.outputs.docker_push }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4.2.0 | |
| with: | |
| enable-cache: true | |
| - name: Sync and verify uv.lock | |
| run: | | |
| # Sync lock file with pyproject.toml | |
| uv lock | |
| - name: Get Commit Hash | |
| id: get_commit_hash | |
| run: | | |
| echo "commit_hash=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT | |
| echo "short_sha=${GITHUB_SHA:0:10}" >> $GITHUB_OUTPUT | |
| - name: Get Version from hatch | |
| id: get_version | |
| run: | | |
| uv tool install --with virtualenv==20.26.6 hatch | |
| echo "version=$(hatch version)" >> $GITHUB_OUTPUT | |
| echo "Found version: $(hatch version)" | |
| - name: Determine Release Tag | |
| id: release_tag | |
| run: | | |
| # Sanitize ref_name for Docker tag compatibility (replace / with -) | |
| sanitized_ref="${{ github.ref_name }}" | |
| sanitized_ref="${sanitized_ref//\//-}" | |
| echo "release_tag=${sanitized_ref}" >> $GITHUB_OUTPUT | |
| echo "Using sanitized branch name as release tag: ${sanitized_ref} (from ${{ github.ref_name }})" | |
| - name: Set Build Params | |
| id: set_build_params | |
| run: | | |
| docker_push=false | |
| # Only update manifest for default branch pushes | |
| if [[ "${{ github.event_name == 'push' }}" == "true" && "${{ github.ref_name }}" == "${{ github.event.repository.default_branch }}" ]]; then | |
| docker_push=true | |
| fi | |
| echo "Will Push Images: ${docker_push}" | |
| echo "docker_push=${docker_push}" >> $GITHUB_OUTPUT | |
| echo "Always building multiplatform for all scenarios" | |
| # ---------------------------------------------------- | |
| # PHASE 1 & 2: Multiplatform Builds (Always Both Platforms) | |
| # ---------------------------------------------------- | |
| build-platform: | |
| name: Build ${{ matrix.platform.name }} | |
| needs: prepare-metadata | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| platform: | |
| - name: linux/amd64 | |
| runner: ubuntu-24.04 | |
| tag_suffix: amd64 | |
| - name: linux/arm64 | |
| runner: ubuntu-24.04-arm | |
| tag_suffix: arm64 | |
| runs-on: ${{ matrix.platform.runner }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 | |
| - name: Setup Docker Buildx | |
| uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1 | |
| with: | |
| aws-access-key-id: ${{ secrets.SAM_AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.SAM_AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ secrets.AWS_DEFAULT_REGION }} | |
| - name: Login to Amazon ECR | |
| id: login-ecr | |
| uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1 | |
| - name: Determine Image Name and Cache Configuration | |
| id: image_name | |
| run: | | |
| image_name="${{ steps.login-ecr.outputs.registry }}/solace-agent-mesh" | |
| echo "image_name=${image_name}" >> $GITHUB_OUTPUT | |
| platform="${{ matrix.platform.tag_suffix }}" | |
| # For push to main: read and write only to buildcache-<platform> | |
| if [[ "${{ github.event_name == 'push' }}" == "true" && "${{ github.ref_name }}" == "${{ github.event.repository.default_branch }}" ]]; then | |
| cache_ref="${image_name}:buildcache-${platform}" | |
| echo "cache_from=type=registry,ref=${cache_ref}" >> $GITHUB_OUTPUT | |
| echo "cache_to=type=registry,ref=${cache_ref},mode=max" >> $GITHUB_OUTPUT | |
| echo "Main branch push: read/write buildcache-${platform}" | |
| else | |
| # For PR builds: read from main cache and PR cache, write only to PR cache | |
| # Sanitize branch name for Docker tag compatibility (replace / with -) | |
| pr_branch="${{ github.head_ref || github.ref_name }}" | |
| pr_branch="${pr_branch//\//-}" | |
| main_cache_ref="${image_name}:buildcache-${platform}" | |
| pr_cache_ref="${image_name}:buildcache-${pr_branch}-${platform}" | |
| # Read from both caches (will try in order) | |
| echo "cache_from=type=registry,ref=${pr_cache_ref}" >> $GITHUB_OUTPUT | |
| echo "cache_from_secondary=type=registry,ref=${main_cache_ref}" >> $GITHUB_OUTPUT | |
| # Write only to PR-specific cache | |
| echo "cache_to=type=registry,ref=${pr_cache_ref},mode=max" >> $GITHUB_OUTPUT | |
| echo "PR build: read from buildcache-${pr_branch}-${platform} and buildcache-${platform}, write to buildcache-${pr_branch}-${platform}" | |
| fi | |
| - name: Prepare Cache From List | |
| id: cache_list | |
| run: | | |
| cache_from="${{ steps.image_name.outputs.cache_from }}" | |
| if [[ -n "${{ steps.image_name.outputs.cache_from_secondary }}" ]]; then | |
| cache_from="${cache_from}"$'\n'"${{ steps.image_name.outputs.cache_from_secondary }}" | |
| fi | |
| # Use multiline output format for GitHub Actions | |
| echo "cache_from<<EOF" >> $GITHUB_OUTPUT | |
| echo "$cache_from" >> $GITHUB_OUTPUT | |
| echo "EOF" >> $GITHUB_OUTPUT | |
| - name: Build and Push Platform-Specific Image | |
| uses: docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1 # v6.16.0 | |
| with: | |
| context: . | |
| file: Dockerfile | |
| platforms: ${{ matrix.platform.name }} | |
| tags: ${{ steps.image_name.outputs.image_name }}:${{ needs.prepare-metadata.outputs.version }}-${{ needs.prepare-metadata.outputs.short_sha }}-${{ matrix.platform.tag_suffix }} | |
| push: ${{ fromJSON(needs.prepare-metadata.outputs.docker_push) }} | |
| cache-from: ${{ steps.cache_list.outputs.cache_from }} | |
| load: ${{ !fromJSON(needs.prepare-metadata.outputs.docker_push) }} | |
| cache-to: ${{ steps.image_name.outputs.cache_to }} | |
| provenance: false | |
| sbom: false | |
| build-args: | | |
| INSTALL_LIBREOFFICE=${{ secrets.INSTALL_LIBREOFFICE || 'false' }} | |
| - name: Run Prisma Cloud Scan on Local Image | |
| if: ${{ github.event_name == 'pull_request' }} | |
| id: prisma_scan | |
| continue-on-error: true | |
| uses: SolaceDev/solace-public-workflows/prisma-cloud-scan@main | |
| with: | |
| image_registry: ${{ steps.login-ecr.outputs.registry }} | |
| image_repo: ${{ github.event.repository.name }} | |
| image_tag: ${{ format('{0}-{1}-{2}', needs.prepare-metadata.outputs.version, needs.prepare-metadata.outputs.short_sha, matrix.platform.tag_suffix) }} | |
| pcc_console_url: ${{ vars.PRISMACLOUD_CONSOLE_URL }} | |
| pcc_user: ${{ secrets.PRISMA_ACCESS_KEY_ID }} | |
| pcc_pass: ${{ secrets.PRISMA_SECRET_ACCESS_KEY }} | |
| twistcli_publish: "true" | |
| vulnerability_grace_period_days: "7" | |
| skip_image_pull: ${{ !fromJSON(needs.prepare-metadata.outputs.docker_push) }} | |
| # ---------------------------------------------------- | |
| # PHASE 3: Merge Multi-Platform Manifest (Always) | |
| # ---------------------------------------------------- | |
| merge-manifest: | |
| name: Create Multi-Platform Manifest | |
| needs: [prepare-metadata, build-platform] | |
| if: ${{ fromJSON(needs.prepare-metadata.outputs.docker_push) }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1 | |
| with: | |
| aws-access-key-id: ${{ secrets.SAM_AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.SAM_AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ secrets.AWS_DEFAULT_REGION }} | |
| - name: Login to Amazon ECR | |
| uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1 | |
| - name: Setup Docker Buildx | |
| uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 | |
| - name: Determine Image Name and Tags | |
| id: image_tags | |
| run: | | |
| # Reconstruct image name based on registry type | |
| image_name="${{ secrets.SAM_AWS_ECR_REGISTRY }}/solace-agent-mesh" | |
| echo "Using ECR registry: ${image_name}" | |
| # Reconstruct tags | |
| version="${{ needs.prepare-metadata.outputs.version }}" | |
| release_tag="${{ needs.prepare-metadata.outputs.release_tag }}" | |
| short_sha="${{ needs.prepare-metadata.outputs.short_sha }}" | |
| tags="${image_name}:${version}-${short_sha}" | |
| if [[ "${{ github.ref_name }}" == "${{ github.event.repository.default_branch }}" ]]; then | |
| tags="${tags},${image_name}:${version}" | |
| tags="${tags},${image_name}:${release_tag}" | |
| fi | |
| tags="${tags},${image_name}:${release_tag}-${short_sha}" | |
| echo "image_name=${image_name}" >> $GITHUB_OUTPUT | |
| echo "tags=${tags}" >> $GITHUB_OUTPUT | |
| echo "Created tags: ${tags}" | |
| - name: Create and Push Multi-Platform Manifest | |
| if: ${{ fromJSON(needs.prepare-metadata.outputs.docker_push) }} | |
| run: | | |
| # Define the platform-specific tags | |
| AMD_TAG="${{ steps.image_tags.outputs.image_name }}:${{ needs.prepare-metadata.outputs.version }}-${{ needs.prepare-metadata.outputs.short_sha }}-amd64" | |
| ARM_TAG="${{ steps.image_tags.outputs.image_name }}:${{ needs.prepare-metadata.outputs.version }}-${{ needs.prepare-metadata.outputs.short_sha }}-arm64" | |
| echo "AMD64 Tag: ${AMD_TAG}" | |
| echo "ARM64 Tag: ${ARM_TAG}" | |
| # OCI annotations to ensure unique manifest digest per commit | |
| # This allows Prisma Cloud and other tools to index each tag uniquely | |
| # while still benefiting from layer caching | |
| COMMIT_SHA="${{ needs.prepare-metadata.outputs.commit_hash }}" | |
| BUILD_TIME="$(date -u +'%Y-%m-%dT%H:%M:%SZ')" | |
| # Convert comma-separated tags string to array and create manifest for each | |
| IFS=',' read -ra TAGS <<< "${{ steps.image_tags.outputs.tags }}" | |
| for TAG in "${TAGS[@]}"; do | |
| # Trim any whitespace | |
| TAG=$(echo "$TAG" | xargs) | |
| echo "Creating manifest for tag: $TAG" | |
| docker buildx imagetools create \ | |
| --annotation "index:org.opencontainers.image.revision=${COMMIT_SHA}" \ | |
| --annotation "index:org.opencontainers.image.created=${BUILD_TIME}" \ | |
| --annotation "index:org.opencontainers.image.source=https://github.com/${{ github.repository }}" \ | |
| --tag "$TAG" \ | |
| "$AMD_TAG" \ | |
| "$ARM_TAG" | |
| done | |
| echo "Multi-platform manifest created successfully" | |
| - name: Update Release Manifest | |
| if: fromJSON(needs.prepare-metadata.outputs.docker_push) | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.SAM_AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.SAM_AWS_SECRET_ACCESS_KEY }} | |
| AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} | |
| run: | | |
| # Set release manifest variables | |
| export release_version="${{ needs.prepare-metadata.outputs.version }}-${{ needs.prepare-metadata.outputs.short_sha }}" | |
| export version="${{ needs.prepare-metadata.outputs.version }}" | |
| export image_tag="${{ needs.prepare-metadata.outputs.version }}-${{ needs.prepare-metadata.outputs.short_sha }}" | |
| export chart_version="n/a" | |
| export sha="${{ needs.prepare-metadata.outputs.commit_hash }}" | |
| export squad="ai" | |
| export repository="solace-agent-mesh" | |
| export release_tag="${{ needs.prepare-metadata.outputs.release_tag }}" | |
| # Update DynamoDB manifest | |
| aws dynamodb update-item \ | |
| --table-name solace-cloud-manifest \ | |
| --key "{\"squad\":{\"S\":\"${squad}\"},\"repository\": {\"S\": \"${repository}\"} }"\ | |
| --update-expression "SET dev = :r" \ | |
| --expression-attribute-values \ | |
| """{\":r\": | |
| {\"M\": | |
| { | |
| \"version\":{\"S\":\"${version}\"}, | |
| \"image_tag\":{\"S\":\"${image_tag}\"}, | |
| \"chart_version\": {\"S\": \"${chart_version}\"}, | |
| \"sha\": {\"S\": \"${sha}\"}, | |
| \"release_version\": {\"S\": \"${release_version}\"} | |
| } | |
| } | |
| }""" | |
| - name: Trigger RC Workflow in rc-sam-community | |
| uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 | |
| with: | |
| github-token: ${{ secrets.RC_TOKEN }} | |
| script: | | |
| const owner = 'SolaceDev'; | |
| const repo = 'rc-sam-community'; | |
| const workflow_id = 'rc-workflow.yaml'; | |
| const ref = 'main'; | |
| const sha = '${{ needs.prepare-metadata.outputs.commit_hash }}'; | |
| const version = '${{ needs.prepare-metadata.outputs.version }}'; | |
| const image_tag = '${{ needs.prepare-metadata.outputs.version }}-${{ needs.prepare-metadata.outputs.short_sha }}'; | |
| console.log('Triggering RC workflow with:'); | |
| console.log(` SHA: ${sha}`); | |
| console.log(` Version: ${version}`); | |
| console.log(` Image Tag: ${image_tag}`); | |
| try { | |
| await github.rest.actions.createWorkflowDispatch({ | |
| owner, | |
| repo, | |
| workflow_id, | |
| ref, | |
| inputs: { | |
| sha: sha, | |
| version: version, | |
| image_tag: image_tag, | |
| environment: 'rc' | |
| } | |
| }); | |
| console.log('✓ Successfully triggered RC workflow'); | |
| console.log(` View at: https://github.com/${owner}/${repo}/actions/workflows/${workflow_id}`); | |
| } catch (error) { | |
| console.error('Failed to trigger RC workflow:', error.message); | |
| throw error; | |
| } |