CCM-10980: standardise test reports

.github/actions/acceptance-tests/action.yaml

@@ -44,30 +44,9 @@ runs:
       run: |
         make test-${{ inputs.testType }}
 
-    - name: Archive ui-e2e test results
-      if: ${{ inputs.testType == 'ui-e2e' }}
+    - name: Archive test results
       uses: actions/upload-artifact@v4
+      if: always()
       with:
-        name: UI E2E test report
-        path: "tests/test-team/playwright-report"
-
-    - name: Archive ui-component test results
-      if: ${{ inputs.testType == 'ui-component' }}
-      uses: actions/upload-artifact@v4
-      with:
-        name: UI-component test report
-        path: "tests/test-team/playwright-report"
-
-    - name: Archive api test results
-      if: ${{ inputs.testType == 'api' }}
-      uses: actions/upload-artifact@v4
-      with:
-        name: API test report
-        path: "tests/test-team/playwright-report"
-
-    - name: Archive accessibility results
-      if: ${{ inputs.testType == 'accessibility' }}
-      uses: actions/upload-artifact@v4
-      with:
-        name: accessibility
-        path: "tests/accessibility/.reports/accessibility"
+        name: ${{ inputs.testType }} - test report
+        path: "tests/acceptance-test-report"

infrastructure/terraform/components/acct/README.md

@@ -22,7 +22,7 @@
 | <a name="input_group"></a> [group](#input\_group) | The group variables are being inherited from (often synonmous with account short-name) | `string` | n/a | yes |
 | <a name="input_initial_cli_secrets_provision_override"></a> [initial\_cli\_secrets\_provision\_override](#input\_initial\_cli\_secrets\_provision\_override) | A map of default value to intialise SSM secret values with. Only useful for initial setup of the account due to lifecycle rules. | `map(string)` | `{}` | no |
 | <a name="input_kms_deletion_window"></a> [kms\_deletion\_window](#input\_kms\_deletion\_window) | When a kms key is deleted, how long should it wait in the pending deletion state? | `string` | `"30"` | no |
-| <a name="input_letter_suppliers"></a> [letter\_suppliers](#input\_letter\_suppliers) | Letter suppliers enabled in the account (across all environments) | <pre>map(object({<br/>    enable_polling   = bool<br/>    default_supplier = optional(bool)<br/>  }))</pre> | `{}` | no |
+| <a name="input_letter_suppliers"></a> [letter\_suppliers](#input\_letter\_suppliers) | Letter suppliers enabled in the account (across all environments) | <pre>map(object({<br>    enable_polling   = bool<br>    default_supplier = optional(bool)<br>  }))</pre> | `{}` | no |
 | <a name="input_log_retention_in_days"></a> [log\_retention\_in\_days](#input\_log\_retention\_in\_days) | The retention period in days for the Cloudwatch Logs events to be retained, default of 0 is indefinite | `number` | `0` | no |
 | <a name="input_oam_sink_id"></a> [oam\_sink\_id](#input\_oam\_sink\_id) | The ID of the Cloudwatch OAM sink in the appropriate observability account. | `string` | `""` | no |
 | <a name="input_observability_account_id"></a> [observability\_account\_id](#input\_observability\_account\_id) | The Observability Account ID that needs access | `string` | n/a | yes |
@@ -31,7 +31,7 @@
 | <a name="input_root_domain_name"></a> [root\_domain\_name](#input\_root\_domain\_name) | The service's root DNS root nameespace, like nonprod.nhsnotify.national.nhs.uk | `string` | `"nonprod.nhsnotify.national.nhs.uk"` | no |
 | <a name="input_support_sandbox_environments"></a> [support\_sandbox\_environments](#input\_support\_sandbox\_environments) | Does this account support dev sandbox environments? | `bool` | `false` | no |
 | <a name="input_vpc_cidr"></a> [vpc\_cidr](#input\_vpc\_cidr) | n/a | `string` | `"10.0.0.0/16"` | no |
-| <a name="input_vpc_subnet_cidr_bits"></a> [vpc\_subnet\_cidr\_bits](#input\_vpc\_subnet\_cidr\_bits) | Number of additional bits to use for subnetting the VPC CIDR block. The bits are evently distributed | <pre>object({<br/>    public  = number<br/>    private = number<br/>  })</pre> | <pre>{<br/>  "private": 3,<br/>  "public": 12<br/>}</pre> | no |
+| <a name="input_vpc_subnet_cidr_bits"></a> [vpc\_subnet\_cidr\_bits](#input\_vpc\_subnet\_cidr\_bits) | Number of additional bits to use for subnetting the VPC CIDR block. The bits are evently distributed | <pre>object({<br>    public  = number<br>    private = number<br>  })</pre> | <pre>{<br>  "private": 3,<br>  "public": 12<br>}</pre> | no |
 ## Modules
 
 | Name | Source | Version |
@@ -57,6 +57,7 @@
 | <a name="output_ses_testing_config"></a> [ses\_testing\_config](#output\_ses\_testing\_config) | n/a |
 | <a name="output_vpc_nat_ips"></a> [vpc\_nat\_ips](#output\_vpc\_nat\_ips) | n/a |
 | <a name="output_vpc_subnets"></a> [vpc\_subnets](#output\_vpc\_subnets) | n/a |
+
 <!-- vale on -->
 <!-- markdownlint-enable -->
 <!-- END_TF_DOCS -->

infrastructure/terraform/components/app/README.md

@@ -37,7 +37,7 @@
 | <a name="input_external_email_domain"></a> [external\_email\_domain](#input\_external\_email\_domain) | Externally managed domain used to create an SES identity for sending emails from. Validation DNS records will need to be manually configured in the DNS provider. | `string` | `null` | no |
 | <a name="input_group"></a> [group](#input\_group) | The group variables are being inherited from (often synonmous with account short-name) | `string` | n/a | yes |
 | <a name="input_kms_deletion_window"></a> [kms\_deletion\_window](#input\_kms\_deletion\_window) | When a kms key is deleted, how long should it wait in the pending deletion state? | `string` | `"30"` | no |
-| <a name="input_letter_suppliers"></a> [letter\_suppliers](#input\_letter\_suppliers) | Letter suppliers enabled in the environment | <pre>map(object({<br/>    email_addresses  = list(string)<br/>    enable_polling   = bool<br/>    default_supplier = optional(bool)<br/>  }))</pre> | `{}` | no |
+| <a name="input_letter_suppliers"></a> [letter\_suppliers](#input\_letter\_suppliers) | Letter suppliers enabled in the environment | <pre>map(object({<br>    email_addresses  = list(string)<br>    enable_polling   = bool<br>    default_supplier = optional(bool)<br>  }))</pre> | `{}` | no |
 | <a name="input_log_retention_in_days"></a> [log\_retention\_in\_days](#input\_log\_retention\_in\_days) | The retention period in days for the Cloudwatch Logs events to be retained, default of 0 is indefinite | `number` | `0` | no |
 | <a name="input_observability_account_id"></a> [observability\_account\_id](#input\_observability\_account\_id) | The Observability Account ID that needs access | `string` | n/a | yes |
 | <a name="input_parent_acct_environment"></a> [parent\_acct\_environment](#input\_parent\_acct\_environment) | Name of the environment responsible for the acct resources used, affects things like DNS zone. Useful for named dev environments | `string` | `"main"` | no |
@@ -64,6 +64,7 @@
 |------|-------------|
 | <a name="output_amplify"></a> [amplify](#output\_amplify) | n/a |
 | <a name="output_deployment"></a> [deployment](#output\_deployment) | Deployment details used for post-deployment scripts |
+
 <!-- vale on -->
 <!-- markdownlint-enable -->
 <!-- END_TF_DOCS -->

infrastructure/terraform/components/branch/README.md

@@ -29,6 +29,7 @@
 ## Outputs
 
 No outputs.
+
 <!-- vale on -->
 <!-- markdownlint-enable -->
 <!-- END_TF_DOCS -->

infrastructure/terraform/components/sandbox/README.md

@@ -18,7 +18,7 @@
 | <a name="input_environment"></a> [environment](#input\_environment) | The name of the tfscaffold environment | `string` | n/a | yes |
 | <a name="input_group"></a> [group](#input\_group) | The group variables are being inherited from (often synonymous with account short-name) | `string` | n/a | yes |
 | <a name="input_kms_deletion_window"></a> [kms\_deletion\_window](#input\_kms\_deletion\_window) | When a kms key is deleted, how long should it wait in the pending deletion state? | `string` | `"30"` | no |
-| <a name="input_letter_suppliers"></a> [letter\_suppliers](#input\_letter\_suppliers) | Letter suppliers enabled in the environment | <pre>map(object({<br/>    email_addresses  = list(string)<br/>    enable_polling   = bool<br/>    default_supplier = optional(bool)<br/>  }))</pre> | <pre>{<br/>  "WTMMOCK": {<br/>    "default_supplier": true,<br/>    "email_addresses": [],<br/>    "enable_polling": true<br/>  }<br/>}</pre> | no |
+| <a name="input_letter_suppliers"></a> [letter\_suppliers](#input\_letter\_suppliers) | Letter suppliers enabled in the environment | <pre>map(object({<br>    email_addresses  = list(string)<br>    enable_polling   = bool<br>    default_supplier = optional(bool)<br>  }))</pre> | <pre>{<br>  "WTMMOCK": {<br>    "default_supplier": true,<br>    "email_addresses": [],<br>    "enable_polling": true<br>  }<br>}</pre> | no |
 | <a name="input_log_retention_in_days"></a> [log\_retention\_in\_days](#input\_log\_retention\_in\_days) | The retention period in days for the Cloudwatch Logs events to be retained, default of 0 is indefinite | `number` | `0` | no |
 | <a name="input_parent_acct_environment"></a> [parent\_acct\_environment](#input\_parent\_acct\_environment) | Name of the environment responsible for the acct resources used, affects things like DNS zone. Useful for named dev environments | `string` | `"main"` | no |
 | <a name="input_project"></a> [project](#input\_project) | The name of the tfscaffold project | `string` | n/a | yes |
@@ -49,6 +49,7 @@
 | <a name="output_test_email_bucket_name"></a> [test\_email\_bucket\_name](#output\_test\_email\_bucket\_name) | n/a |
 | <a name="output_test_proof_requested_email_prefix"></a> [test\_proof\_requested\_email\_prefix](#output\_test\_proof\_requested\_email\_prefix) | n/a |
 | <a name="output_test_template_submitted_email_prefix"></a> [test\_template\_submitted\_email\_prefix](#output\_test\_template\_submitted\_email\_prefix) | n/a |
+
 <!-- vale on -->
 <!-- markdownlint-enable -->
 <!-- END_TF_DOCS -->

infrastructure/terraform/modules/acct-ses-testing/README.md

@@ -30,6 +30,7 @@ No requirements.
 | <a name="output_bucket_name"></a> [bucket\_name](#output\_bucket\_name) | n/a |
 | <a name="output_iam_role_arn"></a> [iam\_role\_arn](#output\_iam\_role\_arn) | n/a |
 | <a name="output_rule_set_name"></a> [rule\_set\_name](#output\_rule\_set\_name) | n/a |
+
 <!-- vale on -->
 <!-- markdownlint-enable -->
 <!-- END_TF_DOCS -->

infrastructure/terraform/modules/backend-api/README.md

@@ -11,7 +11,7 @@ No requirements.
 |------|-------------|------|---------|:--------:|
 | <a name="input_aws_account_id"></a> [aws\_account\_id](#input\_aws\_account\_id) | The AWS Account ID (numeric) | `string` | n/a | yes |
 | <a name="input_cloudfront_distribution_arn"></a> [cloudfront\_distribution\_arn](#input\_cloudfront\_distribution\_arn) | ARN of the cloudfront distribution to serve files from | `string` | `null` | no |
-| <a name="input_cognito_config"></a> [cognito\_config](#input\_cognito\_config) | Cognito config | <pre>object({<br/>    USER_POOL_ID : string,<br/>    USER_POOL_CLIENT_ID : string<br/>  })</pre> | n/a | yes |
+| <a name="input_cognito_config"></a> [cognito\_config](#input\_cognito\_config) | Cognito config | <pre>object({<br>    USER_POOL_ID : string,<br>    USER_POOL_CLIENT_ID : string<br>  })</pre> | n/a | yes |
 | <a name="input_component"></a> [component](#input\_component) | The variable encapsulating the name of this component | `string` | n/a | yes |
 | <a name="input_csi"></a> [csi](#input\_csi) | CSI from the parent component | `string` | n/a | yes |
 | <a name="input_email_domain"></a> [email\_domain](#input\_email\_domain) | Email domain | `string` | n/a | yes |
@@ -22,7 +22,7 @@ No requirements.
 | <a name="input_function_s3_bucket"></a> [function\_s3\_bucket](#input\_function\_s3\_bucket) | Name of S3 bucket to upload lambda artefacts to | `string` | n/a | yes |
 | <a name="input_group"></a> [group](#input\_group) | The group variables are being inherited from (often synonmous with account short-name) | `string` | n/a | yes |
 | <a name="input_kms_key_arn"></a> [kms\_key\_arn](#input\_kms\_key\_arn) | KMS Key ARN | `string` | n/a | yes |
-| <a name="input_letter_suppliers"></a> [letter\_suppliers](#input\_letter\_suppliers) | Letter suppliers enabled in the environment | <pre>map(object({<br/>    email_addresses  = list(string)<br/>    enable_polling   = bool<br/>    default_supplier = optional(bool)<br/>  }))</pre> | n/a | yes |
+| <a name="input_letter_suppliers"></a> [letter\_suppliers](#input\_letter\_suppliers) | Letter suppliers enabled in the environment | <pre>map(object({<br>    email_addresses  = list(string)<br>    enable_polling   = bool<br>    default_supplier = optional(bool)<br>  }))</pre> | n/a | yes |
 | <a name="input_log_destination_arn"></a> [log\_destination\_arn](#input\_log\_destination\_arn) | Destination ARN to use for the log subscription filter | `string` | `""` | no |
 | <a name="input_log_retention_in_days"></a> [log\_retention\_in\_days](#input\_log\_retention\_in\_days) | The retention period in days for the Cloudwatch Logs events to be retained, default of 0 is indefinite | `number` | `0` | no |
 | <a name="input_log_subscription_role_arn"></a> [log\_subscription\_role\_arn](#input\_log\_subscription\_role\_arn) | The ARN of the IAM role to use for the log subscription filter | `string` | `""` | no |
@@ -78,6 +78,7 @@ No requirements.
 | <a name="output_sftp_mock_credential_path"></a> [sftp\_mock\_credential\_path](#output\_sftp\_mock\_credential\_path) | n/a |
 | <a name="output_sftp_poll_lambda_name"></a> [sftp\_poll\_lambda\_name](#output\_sftp\_poll\_lambda\_name) | n/a |
 | <a name="output_templates_table_name"></a> [templates\_table\_name](#output\_templates\_table\_name) | n/a |
+
 <!-- vale on -->
 <!-- markdownlint-enable -->
 <!-- END_TF_DOCS -->

infrastructure/terraform/modules/cognito-triggers/README.md

@@ -29,6 +29,7 @@ No requirements.
 | Name | Description |
 |------|-------------|
 | <a name="output_pre_token_generation_lambda_function_arn"></a> [pre\_token\_generation\_lambda\_function\_arn](#output\_pre\_token\_generation\_lambda\_function\_arn) | n/a |
+
 <!-- vale on -->
 <!-- markdownlint-enable -->
 <!-- END_TF_DOCS -->

infrastructure/terraform/modules/ses/README.md

@@ -25,6 +25,7 @@ No modules.
 | Name | Description |
 |------|-------------|
 | <a name="output_domain"></a> [domain](#output\_domain) | n/a |
+
 <!-- vale on -->
 <!-- markdownlint-enable -->
 <!-- END_TF_DOCS -->

scripts/tests/accessibility.sh

@@ -4,20 +4,22 @@ set -euo pipefail
 
 # feature flag
 export NEXT_PUBLIC_ENABLE_PROOFING=true
-
 export INCLUDE_AUTH_PAGES=true
 
 npm run accessibility-test-setup -w tests/accessibility
-
 npm run build
-
 mkdir -p ./frontend/public/testing
 cp ./utils/utils/src/email-templates/* ./frontend/public/testing/
 
 npm run app:start --prefix frontend
-
 npm run app:wait --prefix frontend
 
-npm run test:accessibility -w tests/accessibility
+TEST_EXIT_CODE=0
+npm run test:accessibility -w tests/accessibility || TEST_EXIT_CODE=$?
+echo "TEST_EXIT_CODE=$TEST_EXIT_CODE"
 
+mkdir -p ./tests/acceptance-test-report
+cp -r ./tests/accessibility/.reports/accessibility ./tests/acceptance-test-report
 npm run accessibility-test-teardown  -w tests/accessibility
+
+exit $TEST_EXIT_CODE

scripts/tests/api.sh

@@ -4,4 +4,12 @@ set -euo pipefail
 cd "$(git rev-parse --show-toplevel)"
 npx playwright install --with-deps > /dev/null
 cd tests/test-team
-npm run test:api
+TEST_EXIT_CODE=0
+npm run test:api || TEST_EXIT_CODE=$?
+echo "TEST_EXIT_CODE=$TEST_EXIT_CODE"
+
+mkdir -p ../acceptance-test-report
+cp -r ./playwright-report ../acceptance-test-report
+[[ -e test-results ]] && cp -r ./test-results ../acceptance-test-report
+
+exit $TEST_EXIT_CODE

scripts/tests/ui-component.sh

@@ -4,4 +4,12 @@ set -euo pipefail
 cd "$(git rev-parse --show-toplevel)"
 npx playwright install --with-deps > /dev/null
 cd tests/test-team
-npm run test:local-ui
+TEST_EXIT_CODE=0
+npm run test:local-ui || TEST_EXIT_CODE=$?
+echo "TEST_EXIT_CODE=$TEST_EXIT_CODE"
+
+mkdir -p ../acceptance-test-report
+cp -r ./playwright-report ../acceptance-test-report
+[[ -e test-results ]] && cp -r ./test-results ../acceptance-test-report
+
+exit $TEST_EXIT_CODE

scripts/tests/ui-e2e.sh

@@ -4,4 +4,12 @@ set -euo pipefail
 cd "$(git rev-parse --show-toplevel)"
 npx playwright install --with-deps > /dev/null
 cd tests/test-team
-npm run test:e2e
+TEST_EXIT_CODE=0
+npm run test:e2e || TEST_EXIT_CODE=$?
+echo "TEST_EXIT_CODE=$TEST_EXIT_CODE"
+
+mkdir -p ../acceptance-test-report
+cp -r ./playwright-report ../acceptance-test-report
+[[ -e test-results ]] && cp -r ./test-results ../acceptance-test-report
+
+exit $TEST_EXIT_CODE