Regression Tests #44719
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Regression Tests | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| tags: | |
| description: "Test scenario tags. If you want to specify multiple tags, you should separate them with a colon :" | |
| required: true | |
| type: string | |
| default: "@regression" | |
| environment: | |
| description: "Environment to run tests against" | |
| type: environment | |
| required: true | |
| default: "DEV" | |
| product: | |
| description: "The product we are testing" | |
| type: choice | |
| options: | |
| - CPTS-UI | |
| - CPTS-FHIR | |
| - EPS-FHIR | |
| - EPS-FHIR-PRESCRIBING | |
| - EPS-FHIR-DISPENSING | |
| - PFP-APIGEE | |
| - PFP-AWS | |
| - PSU | |
| - EPS-ASSIST-ME | |
| required: false | |
| default: EPS-FHIR | |
| id: | |
| description: "Unique run identifier (Do not change this)" | |
| required: false | |
| default: "Manually Triggered Run" | |
| pull_request_id: | |
| description: "The ID of the pull request. This should be in the format pr-xxxx where xxxx is the pull request id" | |
| required: false | |
| default: "" | |
| github_tag: | |
| description: "The github tag to run the test pack from" | |
| required: false | |
| default: "main" | |
| generate_report: | |
| type: boolean | |
| description: "Generate a report for successful test runs" | |
| default: false | |
| workflow_call: | |
| inputs: | |
| tags: | |
| description: "Test scenario tags. If you want to specify multiple tags, you should separate them with a colon :" | |
| required: true | |
| type: string | |
| environment: | |
| description: "Environment to run tests against" | |
| type: string | |
| default: "DEV" | |
| product: | |
| description: "The product we are testing" | |
| type: string | |
| default: EPS-FHIR | |
| id: | |
| description: "Unique run identifier (Do not change this)" | |
| required: false | |
| type: string | |
| default: "Manually Triggered Run" | |
| pull_request_id: | |
| description: "The ID of the pull request. This should be in the format pr-xxxx where xxxx is the pull request id" | |
| required: false | |
| type: string | |
| default: "" | |
| github_tag: | |
| description: "The github tag to run the test pack from" | |
| required: false | |
| type: string | |
| default: "main" | |
| generate_report: | |
| type: boolean | |
| description: "Generate a report for successful test runs" | |
| default: false | |
| jobs: | |
| regression_tests: | |
| runs-on: ubuntu-22.04 | |
| environment: ${{ inputs.environment }} | |
| steps: | |
| # we need to leave this step in as run_regression test scripts expect the third step in the job to have the name of the input id | |
| # the first step is auto generated 'set up job' step | |
| - name: show_input_parameters | |
| env: | |
| tags: ${{ inputs.tags }} | |
| environment: ${{ inputs.environment }} | |
| product: ${{ inputs.product }} | |
| id: ${{ inputs.id }} | |
| pull_request_id: ${{ inputs.pull_request_id }} | |
| github_tag: ${{ inputs.github_tag }} | |
| run: | | |
| echo "tags: ${tags}" | |
| echo "environment: ${environment}" | |
| echo "product: ${product}" | |
| echo "id: ${id}" | |
| echo "pull_request_id: ${pull_request_id}" | |
| echo "github_tag: ${github_tag}" | |
| # output to summary | |
| # shellcheck disable=SC2129 | |
| echo "tags: ${tags}" >> "$GITHUB_STEP_SUMMARY" | |
| echo "environment: ${environment}" >> "$GITHUB_STEP_SUMMARY" | |
| echo "product: ${product}" >> "$GITHUB_STEP_SUMMARY" | |
| echo "id: ${id}" >> "$GITHUB_STEP_SUMMARY" | |
| echo "pull_request_id: ${pull_request_id}" >> "$GITHUB_STEP_SUMMARY" | |
| echo "github_tag: ${github_tag}" >> "$GITHUB_STEP_SUMMARY" | |
| - name: ${{github.event.inputs.id}} | |
| env: | |
| ID: ${{github.event.inputs.id}} | |
| ENV: ${{ inputs.environment }} | |
| PRODUCT: ${{ inputs.product }} | |
| PULL_REQUEST_ID: ${{ inputs.pull_request_id }} | |
| run: | | |
| echo run identifier "$ID"-"$PRODUCT"-"$ENV"-"$PULL_REQUEST_ID" | |
| echo run identifier "$ID"-"$PRODUCT"-"$ENV"-"$PULL_REQUEST_ID" >> "$GITHUB_STEP_SUMMARY" | |
| run_regression_tests: | |
| runs-on: ubuntu-22.04 | |
| environment: ${{ inputs.environment }} | |
| needs: [regression_tests] | |
| permissions: | |
| id-token: write | |
| contents: read | |
| packages: read | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| tag: | |
| [ | |
| "multiple_access", | |
| "multiple_access_pre_selected", | |
| "single_access", | |
| "multiple_roles_single_access", | |
| "multiple_roles_no_access", | |
| "others", | |
| ] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 | |
| with: | |
| ref: ${{ inputs.github_tag }} | |
| - name: Get asdf version | |
| id: asdf-version | |
| run: echo "version=$(awk '!/^#/ && NF {print $1; exit}' .tool-versions.asdf)" >> "$GITHUB_OUTPUT" | |
| # using git commit sha for version of action to ensure we have stable version | |
| - name: Install asdf | |
| uses: asdf-vm/actions/setup@b7bcd026f18772e44fe1026d729e1611cc435d47 | |
| with: | |
| asdf_version: ${{ steps.asdf-version.outputs.version }} | |
| - name: Cache asdf | |
| uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb | |
| with: | |
| path: | | |
| ~/.asdf | |
| key: ${{ runner.os }}-asdf-${{ hashFiles('**/.tool-versions') }} | |
| - name: Install asdf dependencies in .tool-versions | |
| uses: asdf-vm/actions/install@b7bcd026f18772e44fe1026d729e1611cc435d47 | |
| with: | |
| asdf_version: ${{ steps.asdf-version.outputs.version }} | |
| env: | |
| PYTHON_CONFIGURE_OPTS: --enable-shared | |
| - name: Cache Virtualenv | |
| uses: actions/cache@v5 | |
| id: cache-venv | |
| with: | |
| path: .venv | |
| key: ${{ runner.os }}-venv-${{ hashFiles('pyproject.toml') }} | |
| - name: Install Dependencies | |
| if: steps.cache-venv.outputs.cache-hit != 'true' | |
| run: make install | |
| - name: App product to APP for PR | |
| continue-on-error: true | |
| env: | |
| PRODUCT: ${{ inputs.product }} | |
| PR_ID: ${{ inputs.pull_request_id }} | |
| APIGEE_MFA_SECRET: ${{ secrets.APIGEE_MFA_SECRET }} | |
| APIGEE_PASSWORD: ${{ secrets.APIGEE_PASSWORD }} | |
| APIGEE_USER: ${{ secrets.APIGEE_USER }} | |
| run: | | |
| export APIGEE_MFA_SECRET="$APIGEE_MFA_SECRET" | |
| export APIGEE_PASSWORD="$APIGEE_PASSWORD" | |
| export APIGEE_USER="$APIGEE_USER" | |
| poetry run python ./scripts/add_pr_product_to_app.py --product="$PRODUCT" --pr="$PR_ID" | |
| - name: Regression Tests | |
| id: tests | |
| continue-on-error: true | |
| env: | |
| PRODUCT: ${{ inputs.product }} | |
| ENVIRONMENT: ${{ inputs.environment }} | |
| ARM64: False | |
| PULL_REQUEST_ID: ${{ inputs.pull_request_id }} | |
| MATRIX_TAG: "${{ matrix.tag }}" | |
| INPUT_TAG: ${{ inputs.tags }} | |
| CPT_FHIR_CLIENT_ID: ${{ secrets.CPT_FHIR_CLIENT_ID }} | |
| CPT_FHIR_CLIENT_SECRET: ${{ secrets.CPT_FHIR_CLIENT_SECRET }} | |
| EPS_FHIR_CLIENT_ID: ${{ secrets.EPS_FHIR_CLIENT_ID }} | |
| EPS_FHIR_CLIENT_SECRET: ${{ secrets.EPS_FHIR_CLIENT_SECRET }} | |
| EPS_FHIR_SHA1_CLIENT_ID: ${{ secrets.EPS_FHIR_SHA1_CLIENT_ID }} | |
| EPS_FHIR_SHA1_CLIENT_SECRET: ${{ secrets.EPS_FHIR_SHA1_CLIENT_SECRET }} | |
| EPS_FHIR_PRESCRIBING_CLIENT_ID: ${{ secrets.EPS_FHIR_PRESCRIBING_CLIENT_ID }} | |
| EPS_FHIR_PRESCRIBING_CLIENT_SECRET: ${{ secrets.EPS_FHIR_PRESCRIBING_CLIENT_SECRET }} | |
| EPS_FHIR_PRESCRIBING_SHA1_CLIENT_ID: ${{ secrets.EPS_FHIR_PRESCRIBING_SHA1_CLIENT_ID }} | |
| EPS_FHIR_PRESCRIBING_SHA1_CLIENT_SECRET: ${{ secrets.EPS_FHIR_PRESCRIBING_SHA1_CLIENT_SECRET }} | |
| EPS_FHIR_DISPENSING_CLIENT_ID: ${{ secrets.EPS_FHIR_DISPENSING_CLIENT_ID }} | |
| EPS_FHIR_DISPENSING_CLIENT_SECRET: ${{ secrets.EPS_FHIR_DISPENSING_CLIENT_SECRET }} | |
| PFP_CLIENT_ID: ${{ secrets.PFP_CLIENT_ID }} | |
| PFP_CLIENT_SECRET: ${{ secrets.PFP_CLIENT_SECRET }} | |
| PSU_CLIENT_ID: ${{ secrets.PSU_CLIENT_ID }} | |
| PSU_CLIENT_SECRET: ${{ secrets.PSU_CLIENT_SECRET }} | |
| CERTIFICATE: ${{ secrets.CERTIFICATE }} | |
| PRIVATE_KEY: ${{ secrets.PRIVATE_KEY }} | |
| JWT_KID: ${{ secrets.JWT_KID }} | |
| JWT_PRIVATE_KEY: ${{ secrets.JWT_PRIVATE_KEY }} | |
| EPS_ASSIST_ME_ROLE_ARN: ${{ secrets.EPS_ASSIST_ME_ROLE_ARN }} | |
| run: | | |
| export CPT_FHIR_CLIENT_ID="$CPT_FHIR_CLIENT_ID" | |
| export CPT_FHIR_CLIENT_SECRET="$CPT_FHIR_CLIENT_SECRET" | |
| export EPS_FHIR_CLIENT_ID="$EPS_FHIR_CLIENT_ID" | |
| export EPS_FHIR_CLIENT_SECRET="$EPS_FHIR_CLIENT_SECRET" | |
| export EPS_FHIR_SHA1_CLIENT_ID="$EPS_FHIR_SHA1_CLIENT_ID" | |
| export EPS_FHIR_SHA1_CLIENT_SECRET="$EPS_FHIR_SHA1_CLIENT_SECRET" | |
| export EPS_FHIR_PRESCRIBING_CLIENT_ID="$EPS_FHIR_PRESCRIBING_CLIENT_ID" | |
| export EPS_FHIR_PRESCRIBING_CLIENT_SECRET="$EPS_FHIR_PRESCRIBING_CLIENT_SECRET" | |
| export EPS_FHIR_PRESCRIBING_SHA1_CLIENT_ID="$EPS_FHIR_PRESCRIBING_SHA1_CLIENT_ID" | |
| export EPS_FHIR_PRESCRIBING_SHA1_CLIENT_SECRET="$EPS_FHIR_PRESCRIBING_SHA1_CLIENT_SECRET" | |
| export EPS_FHIR_DISPENSING_CLIENT_ID="$EPS_FHIR_DISPENSING_CLIENT_ID" | |
| export EPS_FHIR_DISPENSING_CLIENT_SECRET="$EPS_FHIR_DISPENSING_CLIENT_SECRET" | |
| export PFP_CLIENT_ID="$PFP_CLIENT_ID" | |
| export PFP_CLIENT_SECRET="$PFP_CLIENT_SECRET" | |
| export PSU_CLIENT_ID="$PSU_CLIENT_ID" | |
| export PSU_CLIENT_SECRET="$PSU_CLIENT_SECRET" | |
| export CERTIFICATE="$CERTIFICATE" | |
| export PRIVATE_KEY="$PRIVATE_KEY" | |
| export JWT_KID="$JWT_KID" | |
| export JWT_PRIVATE_KEY="$JWT_PRIVATE_KEY" | |
| export PULL_REQUEST_ID="$PULL_REQUEST_ID" | |
| export EPS_ASSIST_ME_ROLE_ARN="$EPS_ASSIST_ME_ROLE_ARN" | |
| echo Pull request ID = "$PULL_REQUEST_ID" | |
| case "$MATRIX_TAG" in | |
| multiple_access) | |
| TAGS="@multiple_access:${INPUT_TAG}" | |
| ;; | |
| multiple_access_pre_selected) | |
| TAGS="@multiple_access_pre_selected:${INPUT_TAG}" | |
| ;; | |
| single_access) | |
| TAGS="@single_access:${INPUT_TAG}" | |
| ;; | |
| multiple_roles_single_access) | |
| TAGS="@multiple_roles_single_access:${INPUT_TAG}" | |
| ;; | |
| multiple_roles_no_access) | |
| TAGS="@multiple_roles_no_access:${INPUT_TAG}" | |
| ;; | |
| others) | |
| TAGS="~@multiple_access:~@multiple_access_pre_selected:~@single_access:~@multiple_roles_single_access:~@multiple_roles_no_access:${INPUT_TAG}" | |
| ;; | |
| *) | |
| echo "Unknown MATRIX_TAG value: $MATRIX_TAG" | |
| exit 1 | |
| ;; | |
| esac | |
| export TAGS | |
| echo "Going to run with TAGS ${TAGS}" | |
| poetry run python ./runner.py --env="$ENVIRONMENT" --product="$PRODUCT" --tags="$TAGS" | |
| - name: Upload Artifact | |
| uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 | |
| with: | |
| name: allure-results-${{ matrix.tag }} | |
| path: allure-results | |
| - name: force error on failure | |
| if: steps.tests.outcome != 'success' | |
| run: | | |
| exit 1 | |
| upload_results: | |
| runs-on: ubuntu-22.04 | |
| environment: ${{ inputs.environment }} | |
| needs: [run_regression_tests] | |
| if: always() | |
| steps: | |
| - name: Download matrix artifacts | |
| uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 | |
| with: | |
| path: downloaded_artifacts | |
| pattern: allure-results-* | |
| merge-multiple: true | |
| - name: Place matrix artifacts in allure results | |
| run: | | |
| mkdir -p allure-results | |
| rsync -av downloaded_artifacts/* allure-results/ | |
| rsync -av downloaded_artifacts/environment.properties allure-results/ | |
| - name: Upload Artifact | |
| uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 | |
| with: | |
| name: allure-results | |
| path: allure-results | |
| if-no-files-found: error | |
| - name: Checkout | |
| uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 | |
| with: | |
| ref: ${{ inputs.github_tag }} | |
| - name: Get asdf version | |
| id: asdf-version | |
| run: echo "version=$(awk '!/^#/ && NF {print $1; exit}' .tool-versions.asdf)" >> "$GITHUB_OUTPUT" | |
| # using git commit sha for version of action to ensure we have stable version | |
| - name: Install asdf | |
| uses: asdf-vm/actions/setup@b7bcd026f18772e44fe1026d729e1611cc435d47 | |
| with: | |
| asdf_version: ${{ steps.asdf-version.outputs.version }} | |
| - name: Cache asdf | |
| uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb | |
| with: | |
| path: | | |
| ~/.asdf | |
| key: ${{ runner.os }}-asdf-${{ hashFiles('**/.tool-versions') }} | |
| - name: Install asdf dependencies in .tool-versions | |
| uses: asdf-vm/actions/install@b7bcd026f18772e44fe1026d729e1611cc435d47 | |
| with: | |
| asdf_version: 0.16.4 | |
| env: | |
| PYTHON_CONFIGURE_OPTS: --enable-shared | |
| - name: Cache Virtualenv | |
| uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb | |
| id: cache-venv | |
| with: | |
| path: .venv | |
| key: ${{ runner.os }}-venv-${{ hashFiles('pyproject.toml') }} | |
| - name: Install Dependencies | |
| if: steps.cache-venv.outputs.cache-hit != 'true' | |
| run: | | |
| make install | |
| - name: download | |
| uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 | |
| with: | |
| name: allure-results | |
| path: allure-results | |
| - name: Generate a token | |
| id: generate-token | |
| uses: actions/create-github-app-token@7e473efe3cb98aa54f8d4bac15400b15fad77d94 | |
| if: contains(needs.*.result, 'failure') || github.event.inputs.generate_report == 'true' | |
| with: | |
| app-id: ${{ vars.REGRESSION_TESTS_APP_ID }} | |
| private-key: ${{ secrets.REGRESSION_TESTS_PEM }} | |
| owner: "NHSDigital" | |
| repositories: "electronic-prescription-service-api-regression-tests,eps-test-reports" | |
| - name: Report failure on test failure | |
| if: contains(needs.*.result, 'failure') | |
| run: | | |
| poetry run python scripts/send_test_results.py --token=${{ steps.generate-token.outputs.token }} --run_id ${{GITHUB.RUN_ID}} | |
| echo The regression tests step failed, this likely means there are test failures. | |
| echo A test report will be generated shortly and can be viewed at: https://nhsdigital.github.io/eps-test-reports | |
| exit 1 | |
| - name: Report failure on test cancelled or skipped | |
| if: contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'skipped') | |
| run: | | |
| echo The regression tests step was cancelled or skipped | |
| exit 1 | |
| - name: Generate Report | |
| if: github.event.inputs.generate_report == 'true' | |
| run: | | |
| poetry run python scripts/send_test_results.py --token=${{ steps.generate-token.outputs.token }} --run_id ${{GITHUB.RUN_ID}} | |
| echo You selected to generate a report for successful test runs. | |
| echo A test report will be generated shortly and can be viewed at: https://nhsdigital.github.io/eps-test-reports |