diff --git a/.github/actions/notify-slack/action.yml b/.github/actions/notify-slack/action.yml index 10fb2208c..bf4c049d5 100644 --- a/.github/actions/notify-slack/action.yml +++ b/.github/actions/notify-slack/action.yml @@ -1,10 +1,18 @@ name: 'Notify Slack' -description: 'Send a notification to a Slack channel with an error message' +description: 'Send a notification to a Slack channel with an alert message based on priority level' inputs: OP_SERVICE_ACCOUNT_TOKEN: description: 'The service account token for 1Password' required: true type: string + PRIORITY: + description: 'Priority level of the alert (medium/high)' + required: true + type: string + STEP: + description: 'The step in which the issue occurred (optional)' + required: false + type: string runs: using: 'composite' @@ -19,30 +27,52 @@ runs: CREDENTIALS: "op://rbiv7rvkkrsdlpcrz3bmv7nmcu/ifkeehu5gzi7wy5ub5qvwkaire/credential" SLACK_WEBHOOK_URL: "op://rbiv7rvkkrsdlpcrz3bmv7nmcu/Slack webhook URLs/internal-github-action-alerts/internal-github-action-alerts" + - name: Set Alert Message + id: alert_message + shell: bash + run: | + if [[ "${{ inputs.PRIORITY }}" == "high" ]]; then + if [[ -n "${{ inputs.STEP }}" ]]; then + message="🚨 High Priority Alert: [${{ github.workflow }}] failed at step \"${{ inputs.STEP }}\". Immediate attention is required to avoid production impact." + else + message="🚨 High Priority Alert: [${{ github.workflow }}] failed. Immediate attention is required to avoid production impact." + fi + else + if [[ -n "${{ inputs.STEP }}" ]]; then + message="🚧 Medium Priority Alert: [${{ github.workflow }}] encountered an issue at step \"${{ inputs.STEP }}\". This may affect ongoing integration processes." + else + message="🚧 Medium Priority Alert: [${{ github.workflow }}] encountered an issue. This may affect ongoing integration processes." + fi + fi + + # Construct the JSON payload and save it to a file + jq -n --arg message "$message" --arg run_url "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" '{ + "channel": "C074SPVCH6H", + "blocks": [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": $message, + "emoji": true + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "See <\($run_url)|GitHub Action Run Details>" + } + } + ] + }' > slack_payload.json + - name: Post notification in Slack channel uses: slackapi/slack-github-action@v1.26.0 with: - payload: | - { - "channel": "C074SPVCH6H", - "blocks": [ - { - "type": "header", - "text": { - "type": "plain_text", - "text": ":x: ERROR running Github action \"${{ github.workflow }}\"", - "emoji": true - } - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "See https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" - } - } - ] - } + payload-file-path: ./slack_payload.json env: SLACK_WEBHOOK_URL: ${{ env.SLACK_WEBHOOK_URL }} SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK + + diff --git a/.github/workflows/datasets-batch-deployer-dev.yml b/.github/workflows/datasets-batch-deployer-dev.yml index 21d2c443d..7bd9aa735 100644 --- a/.github/workflows/datasets-batch-deployer-dev.yml +++ b/.github/workflows/datasets-batch-deployer-dev.yml @@ -21,3 +21,15 @@ jobs: DATASETS_BUCKET_NAME: 'mobilitydata-datasets-dev' secrets: GCP_MOBILITY_FEEDS_SA_KEY: ${{ secrets.DEV_GCP_MOBILITY_FEEDS_SA_KEY }} + notify-slack-on-failure: + needs: [ deploy ] + if: always() && (needs.deploy.result == 'failure') && (github.event_name != 'workflow_dispatch') + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Notify Slack + uses: ./.github/actions/notify-slack + with: + OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} + PRIORITY: "medium" diff --git a/.github/workflows/db-update-dev.yml b/.github/workflows/db-update-dev.yml index 1bdbc9d7c..7f0749913 100644 --- a/.github/workflows/db-update-dev.yml +++ b/.github/workflows/db-update-dev.yml @@ -27,4 +27,16 @@ jobs: DB_GCP_MOBILITY_FEEDS_SA_KEY: ${{ secrets.QA_GCP_MOBILITY_FEEDS_SA_KEY }} OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} OP_FEEDS_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_FEEDS_SERVICE_ACCOUNT_TOKEN }} - POSTGRE_SQL_INSTANCE_NAME: ${{ secrets.DB_INSTANCE_NAME }} \ No newline at end of file + POSTGRE_SQL_INSTANCE_NAME: ${{ secrets.DB_INSTANCE_NAME }} + notify-slack-on-failure: + needs: [ update ] + if: always() && (needs.update.result == 'failure') && (github.event_name == 'repository_dispatch') + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Notify Slack + uses: ./.github/actions/notify-slack + with: + OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} + PRIORITY: "high" diff --git a/.github/workflows/db-update-prod.yml b/.github/workflows/db-update-prod.yml index 1fbd9a497..634dd489e 100644 --- a/.github/workflows/db-update-prod.yml +++ b/.github/workflows/db-update-prod.yml @@ -22,4 +22,17 @@ jobs: DB_GCP_MOBILITY_FEEDS_SA_KEY: ${{ secrets.PROD_GCP_MOBILITY_FEEDS_SA_KEY }} OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} OP_FEEDS_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_FEEDS_SERVICE_ACCOUNT_TOKEN }} - POSTGRE_SQL_INSTANCE_NAME: ${{ secrets.DB_INSTANCE_NAME }} \ No newline at end of file + POSTGRE_SQL_INSTANCE_NAME: ${{ secrets.DB_INSTANCE_NAME }} + + notify-slack-on-failure: + needs: [ update ] + if: always() && (needs.update.result == 'failure') && (github.event_name == 'repository_dispatch') + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Notify Slack + uses: ./.github/actions/notify-slack + with: + OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} + PRIORITY: "high" diff --git a/.github/workflows/db-update-qa.yml b/.github/workflows/db-update-qa.yml index fa88a2e66..e4aa65d9f 100644 --- a/.github/workflows/db-update-qa.yml +++ b/.github/workflows/db-update-qa.yml @@ -22,4 +22,16 @@ jobs: DB_GCP_MOBILITY_FEEDS_SA_KEY: ${{ secrets.QA_GCP_MOBILITY_FEEDS_SA_KEY }} OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} OP_FEEDS_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_FEEDS_SERVICE_ACCOUNT_TOKEN }} - POSTGRE_SQL_INSTANCE_NAME: ${{ secrets.DB_INSTANCE_NAME }} \ No newline at end of file + POSTGRE_SQL_INSTANCE_NAME: ${{ secrets.DB_INSTANCE_NAME }} + notify-slack-on-failure: + needs: [ update ] + if: always() && (needs.update.result == 'failure') && (github.event_name == 'repository_dispatch') + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Notify Slack + uses: ./.github/actions/notify-slack + with: + OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} + PRIORITY: "high" \ No newline at end of file diff --git a/.github/workflows/release-qa.yml b/.github/workflows/release-qa.yml index 47105c314..6fa5fad54 100644 --- a/.github/workflows/release-qa.yml +++ b/.github/workflows/release-qa.yml @@ -39,3 +39,15 @@ jobs: needs: batch-deployment uses: ./.github/workflows/web-qa.yml secrets: inherit + notify-slack-on-failure: + needs: [ web-deployment, integration-tests ] + if: failure() && (github.event_name != 'workflow_dispatch') + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Notify Slack + uses: ./.github/actions/notify-slack + with: + OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} + PRIORITY: "high" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8291cca4f..17edc2c04 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -37,3 +37,15 @@ jobs: needs: batch-deployment uses: ./.github/workflows/web-prod.yml secrets: inherit + notify-slack-on-failure: + needs: [web-deployment, integration-tests] + if: failure() + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Notify Slack + uses: ./.github/actions/notify-slack + with: + OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} + PRIORITY: "high" \ No newline at end of file diff --git a/.github/workflows/validator-update.yml b/.github/workflows/validator-update.yml index f07c2d553..3a1b1a7b0 100644 --- a/.github/workflows/validator-update.yml +++ b/.github/workflows/validator-update.yml @@ -37,3 +37,15 @@ jobs: --schedule-time=$(date -u -d "+24 hours" +%Y-%m-%dT%H:%M:%SZ) \ --oidc-service-account-email=${DEPLOYER_SERVICE_ACCOUNT} \ --location=${{ vars.MOBILITY_FEEDS_REGION }} + notify-slack-on-failure: + needs: [ validator-update ] + if: failure() && (github.event_name != 'workflow_dispatch') + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Notify Slack + uses: ./.github/actions/notify-slack + with: + OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} + PRIORITY: "high" diff --git a/docs/GitHubWorkflows.md b/docs/GitHubWorkflows.md new file mode 100644 index 000000000..ca3283d1f --- /dev/null +++ b/docs/GitHubWorkflows.md @@ -0,0 +1,28 @@ +# GitHub Workflow Documentation + +| **Workflow Group** | **Environment** | **Description** | **Triggers** | **Workflow Dependencies** | **Failure Sends Slack Message** | +|-----------------------------|-------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------|------------------------------------------------------------------------------------------------------------------------------|---------------------------------| +| **API Deployment** | DEV | Deploys the API to [DEV environment](https://api-dev.mobilitydatabase.org/) | Manual | Build & Test, Integration Tests | | +| | QA | Deploys the API to [QA environment](https://api-qa.mobilitydatabase.org/) | Manual, Workflow Call | Build & Test | | +| | PROD | Deploys the API to [Production](https://api.mobilitydatabase.org/) | Manual, Workflow Call | Build & Test | | +| **Build & Test** | Local | Runs tests in `api` directory as well as `functions-python` | Pull Request, Workflow Call | | | +| **Dataset Batch Deployment** | DEV | Deploys the infrastructure Terraform code in `infra/batch` for the DEV environment | Merge, Manual | | ✅ | +| | QA | Deploys the infrastructure Terraform code in `infra/batch` for the QA environment | Manual, Workflow Call | | | +| | PROD | Deploys the infrastructure Terraform code in `infra/batch` for the PROD environment | Manual, Workflow Call | | | +| **Database Deployment** | QA | Deploys the database to QA, which includes the DEV instance; updates DEV secrets as part of this action | Manual, Workflow Call | | | +| | PROD | Deploys the database instance in PROD | Manual, Workflow Call | | | +| **Database Update** | DEV | Updates the schema of the DEV database and syncs content with `mobility-database-catalog` repo using `sources.csv` | Merge, Manual, Repository Dispatch | | ✅ | +| | QA | Same as DEV, but for the QA database | Workflow Call, Manual, Repository Dispatch | | ✅ | +| | PROD | Same as DEV, but for the PROD database | Workflow Call, Manual, Repository Dispatch | | ✅ | +| **Integration Tests** | N/A | Runs API integration tests on the specified link as input variable | Workflow Call | | | +| **Integration Tests - PR** | Local to GitHub runner, QA database | Runs integration tests locally while connecting to the QA database | Pull Request | | | +| **Release** | QA | Full release for QA | Merge, Manual | Database Deployment, Database Update, API Deployment, Dataset Batch Deployment, Integration Tests, Web Deployment | ✅ | +| | PROD | Full release for PROD | Release | Database Deployment, Database Update, API Deployment, Dataset Batch Deployment, Integration Tests, Web Deployment | ✅ | +| **Load Test** | QA | Runs load tests for the API | Scheduled, Manual | | ✅ | +| **TypeScript Generator Check** | Local | Verifies that TypeScript types generated from `DatabaseCatalogAPI.yaml` match existing types in the repository | Pull Request | | | +| **Validator Update** | PROD | Creates a GCP task to update all validation reports 24 hours after a new version release of the GTFS canonical validator | Repository Dispatch | | ✅ | +| **Web App Deployer** | DEV | Deploys the web app to an environment on demand; posts URL as a comment to the pull request if applicable | Manual, PR | | | +| | QA | Deploys the web app to [qa.mobilitydatabase.org](https://qa.mobilitydatabase.org/) | Manual, Workflow Call | | | +| | PROD | Deploys the web app to [mobilitydatabase.org](https://mobilitydatabase.org/) | Manual, Workflow Call | | | + +This table documents all GitHub workflows, providing an overview of the workflow groups, environments, descriptions, triggers, dependencies, and Slack notification settings for each.