Skip to content

Commit 1d9c008

Browse files
mfjarvismfjarvis
committed
Improve quality checks pipeline. (#896)
1 parent b121c96 commit 1d9c008

File tree

25 files changed

+261
-631
lines changed

25 files changed

+261
-631
lines changed

‎.github/workflows/quality-checks.yml‎

Lines changed: 23 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,9 @@ jobs:
1717
runs-on: ubuntu-latest
1818

1919
steps:
20-
- uses: actions/checkout@v5
20+
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
2121

22-
- uses: actions/setup-node@v5
22+
- uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444
2323
with:
2424
node-version: "23.11.0"
2525
cache: "npm"
@@ -30,17 +30,20 @@ jobs:
3030
- name: Lint
3131
run: make lint
3232

33+
- name: Check formatting
34+
run: make format-check
35+
3336
lint-python:
3437
name: Lint Python projects
3538
runs-on: ubuntu-latest
3639

3740
steps:
38-
- uses: actions/checkout@v5
41+
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
3942

4043
- name: Install poetry
4144
run: pip install poetry==2.1.4
4245

43-
- uses: actions/setup-python@v6
46+
- uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c
4447
with:
4548
python-version: 3.11
4649
cache: "poetry"
@@ -57,17 +60,31 @@ jobs:
5760
run: poetry run make format-check
5861
working-directory: quality_checks
5962

63+
lint-terraform:
64+
name: Lint Terraform projects
65+
runs-on: ubuntu-latest
66+
67+
steps:
68+
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
69+
70+
- uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd
71+
with:
72+
terraform_version: "1.12.2"
73+
74+
- name: Check formatting
75+
run: terraform fmt -check -recursive
76+
6077
testcoverage_and_sonarcloud:
6178
name: Test Coverage and SonarCloud
6279
runs-on: ubuntu-latest
6380

6481
steps:
65-
- uses: actions/checkout@v5
82+
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
6683

6784
- name: Install poetry
6885
run: pip install poetry==2.1.4
6986

70-
- uses: actions/setup-python@v6
87+
- uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c
7188
with:
7289
python-version: 3.11
7390
cache: "poetry"

‎Makefile‎

Lines changed: 8 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@ SHELL=/usr/bin/env bash -euo pipefail
33
PYTHON_PROJECT_DIRS_WITH_UNIT_TESTS = backend batch_processor_filter delta_backend filenameprocessor mesh_processor recordprocessor lambdas/ack_backend lambdas/redis_sync lambdas/id_sync lambdas/mns_subscription lambdas/shared
44
PYTHON_PROJECT_DIRS = e2e e2e_batch quality_checks $(PYTHON_PROJECT_DIRS_WITH_UNIT_TESTS)
55

6+
.PHONY: install-python install-node install lint format format-check clean publish build-proxy release initialise-all-python-venvs update-all-python-dependencies run-all-python-unit-tests build-all-docker-images
7+
68
#Installs dependencies using poetry.
79
install-python:
810
poetry lock --no-update
@@ -19,6 +21,12 @@ install: install-node install-python
1921
lint:
2022
npm run lint
2123

24+
format:
25+
npm run format
26+
27+
format-check:
28+
npm run format-check
29+
2230
#Removes build/ + dist/ directories
2331
clean:
2432
rm -rf build
@@ -39,7 +47,6 @@ build-proxy:
3947
# VED-811: remove everything except for proxy related files as we move to Github Actions for backend deployment
4048
_dist_include="pytest.ini poetry.lock poetry.toml pyproject.toml Makefile build/. specification sandbox terraform scripts"
4149

42-
4350
#Create /dist/ sub-directory and copy files into directory
4451
#Ensure full dir structure is preserved for Lambdas
4552
release: clean publish build-proxy
@@ -50,43 +57,6 @@ release: clean publish build-proxy
5057
cp ecs-proxies-deploy.yml dist/ecs-deploy-internal-qa-sandbox.yml
5158
cp ecs-proxies-deploy.yml dist/ecs-deploy-internal-dev-sandbox.yml
5259

53-
#################
54-
# Test commands #
55-
#################
56-
57-
TEST_CMD := @APIGEE_ACCESS_TOKEN=$(APIGEE_ACCESS_TOKEN) \
58-
poetry run pytest -v \
59-
--color=yes \
60-
--api-name=immunisation-fhir-api \
61-
--proxy-name=$(PROXY_NAME) \
62-
-s
63-
64-
PROD_TEST_CMD := $(TEST_CMD) \
65-
--apigee-app-id=$(APIGEE_APP_ID) \
66-
--status-endpoint-api-key=$(STATUS_ENDPOINT_API_KEY)
67-
68-
#Command to run end-to-end smoketests post-deployment to verify the environment is working
69-
smoketest:
70-
$(TEST_CMD) \
71-
--junitxml=smoketest-report.xml \
72-
-m smoketest
73-
74-
test:
75-
$(TEST_CMD) \
76-
--junitxml=test-report.xml \
77-
78-
smoketest-prod:
79-
$(PROD_TEST_CMD) \
80-
--junitxml=smoketest-report.xml \
81-
-m smoketest
82-
83-
test-prod:
84-
$(PROD_CMD) \
85-
--junitxml=test-report.xml \
86-
87-
setup-python-envs:
88-
scripts/setup-python-envs.sh
89-
9060
initialise-all-python-venvs:
9161
for dir in $(PYTHON_PROJECT_DIRS); do ( \
9262
cd $$dir && \

‎README.md‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -57,18 +57,18 @@ See https://nhsd-confluence.digital.nhs.uk/display/APM/Glossary.
5757
| ----------- | ------------------------------------------------------------------------------------ |
5858
| `e2e` | End-to-end tests executed during PR pipelines. |
5959
| `e2e_batch` | E2E tests specifically for batch-related functionality, also run in the PR pipeline. |
60-
| `tests` | Sample e2e test. |
6160

6261
---
6362

6463
### Utilities
6564

66-
| Folder | Description |
67-
| --------------- | ------------------------------------------------------------- |
68-
| `devtools` | Helper tools and utilities for local development |
69-
| `scripts` | Standalone or reusable scripts for development and automation |
70-
| `specification` | Specification files to document API and related definitions |
71-
| `sandbox` | Simple sandbox API |
65+
| Folder | Description |
66+
| ---------------- | ------------------------------------------------------------- |
67+
| `devtools` | Helper tools and utilities for local development |
68+
| `quality_checks` | Dependencies for linting and formatting Python code |
69+
| `scripts` | Standalone or reusable scripts for development and automation |
70+
| `specification` | Specification files to document API and related definitions |
71+
| `sandbox` | Simple sandbox API |
7272

7373
---
7474

‎devtools/dynamodb.tf‎

Lines changed: 49 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -1,63 +1,63 @@
11
terraform {
2-
required_providers {
3-
aws = {
4-
source = "hashicorp/aws"
5-
version = "~> 5"
6-
}
7-
}
8-
backend "local" {
9-
path = ".terraform/local.tfstate"
10-
}
2+
required_providers {
3+
aws = {
4+
source = "hashicorp/aws"
5+
version = "~> 5"
6+
}
7+
}
8+
backend "local" {
9+
path = ".terraform/local.tfstate"
10+
}
1111
}
1212

1313
provider "aws" {
14-
region = "us-east-1"
15-
skip_region_validation = true
14+
region = "us-east-1"
15+
skip_region_validation = true
1616

17-
endpoints {
18-
sts = "http://localhost:4566"
19-
dynamodb = "http://localhost:4566"
20-
}
17+
endpoints {
18+
sts = "http://localhost:4566"
19+
dynamodb = "http://localhost:4566"
20+
}
2121
}
2222

2323
locals {
24-
short_prefix = "imms-default"
24+
short_prefix = "imms-default"
2525
}
2626

2727
resource "aws_dynamodb_table" "events-dynamodb-table" {
28-
name = "${local.short_prefix}-imms-events"
29-
billing_mode = "PAY_PER_REQUEST"
30-
hash_key = "PK"
31-
stream_enabled = true
32-
stream_view_type = "NEW_IMAGE"
28+
name = "${local.short_prefix}-imms-events"
29+
billing_mode = "PAY_PER_REQUEST"
30+
hash_key = "PK"
31+
stream_enabled = true
32+
stream_view_type = "NEW_IMAGE"
3333

34-
attribute {
35-
name = "PK"
36-
type = "S"
37-
}
38-
attribute {
39-
name = "PatientPK"
40-
type = "S"
41-
}
42-
attribute {
43-
name = "PatientSK"
44-
type = "S"
45-
}
46-
attribute {
47-
name = "IdentifierPK"
48-
type = "S"
49-
}
34+
attribute {
35+
name = "PK"
36+
type = "S"
37+
}
38+
attribute {
39+
name = "PatientPK"
40+
type = "S"
41+
}
42+
attribute {
43+
name = "PatientSK"
44+
type = "S"
45+
}
46+
attribute {
47+
name = "IdentifierPK"
48+
type = "S"
49+
}
5050

51-
global_secondary_index {
52-
name = "PatientGSI"
53-
hash_key = "PatientPK"
54-
range_key = "PatientSK"
55-
projection_type = "ALL"
56-
}
57-
58-
global_secondary_index {
59-
name = "IdentifierGSI"
60-
hash_key = "IdentifierPK"
61-
projection_type = "ALL"
62-
}
51+
global_secondary_index {
52+
name = "PatientGSI"
53+
hash_key = "PatientPK"
54+
range_key = "PatientSK"
55+
projection_type = "ALL"
56+
}
57+
58+
global_secondary_index {
59+
name = "IdentifierGSI"
60+
hash_key = "IdentifierPK"
61+
projection_type = "ALL"
62+
}
6363
}

‎grafana/non-prod/terraform/vpce/variables.tf‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
variable "aws_region" {
2-
description = "Destination AWS region"
2+
description = "Destination AWS region"
33
}
44

55
variable "tags" {

‎grafana/non-prod/terraform/vpce/vpce.tf‎

Lines changed: 40 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -2,54 +2,54 @@
22
# VPCE alternative to NAT Gateway
33
# VPC Endpoint for ECR API
44
resource "aws_vpc_endpoint" "ecr_api" {
5-
vpc_id = var.vpc_id
6-
service_name = "com.amazonaws.${var.aws_region}.ecr.api"
7-
vpc_endpoint_type = "Interface"
8-
subnet_ids = var.private_subnet_ids
9-
security_group_ids = [aws_security_group.vpc_endpoints.id]
10-
# allow for dns resolution
11-
private_dns_enabled = true
12-
tags = merge(var.tags, {
13-
Name = "${var.prefix}-ecr-api-vpce"
14-
})
5+
vpc_id = var.vpc_id
6+
service_name = "com.amazonaws.${var.aws_region}.ecr.api"
7+
vpc_endpoint_type = "Interface"
8+
subnet_ids = var.private_subnet_ids
9+
security_group_ids = [aws_security_group.vpc_endpoints.id]
10+
# allow for dns resolution
11+
private_dns_enabled = true
12+
tags = merge(var.tags, {
13+
Name = "${var.prefix}-ecr-api-vpce"
14+
})
1515
}
1616

1717
# VPC Endpoint for ECR Docker
1818
resource "aws_vpc_endpoint" "ecr_docker" {
19-
vpc_id = var.vpc_id
20-
service_name = "com.amazonaws.${var.aws_region}.ecr.dkr"
21-
vpc_endpoint_type = "Interface"
22-
subnet_ids = var.private_subnet_ids
23-
security_group_ids = [aws_security_group.vpc_endpoints.id]
24-
# allow for dns resolution
25-
private_dns_enabled = true
26-
tags = merge(var.tags, {
27-
Name = "${var.prefix}-ecr-dkr-vpce"
28-
})
19+
vpc_id = var.vpc_id
20+
service_name = "com.amazonaws.${var.aws_region}.ecr.dkr"
21+
vpc_endpoint_type = "Interface"
22+
subnet_ids = var.private_subnet_ids
23+
security_group_ids = [aws_security_group.vpc_endpoints.id]
24+
# allow for dns resolution
25+
private_dns_enabled = true
26+
tags = merge(var.tags, {
27+
Name = "${var.prefix}-ecr-dkr-vpce"
28+
})
2929
}
3030

3131
# VPC Endpoint for CloudWatch Logs
3232
resource "aws_vpc_endpoint" "cloudwatch_logs" {
33-
vpc_id = var.vpc_id
34-
service_name = "com.amazonaws.${var.aws_region}.logs"
35-
vpc_endpoint_type = "Interface"
36-
subnet_ids = var.private_subnet_ids
37-
security_group_ids = [aws_security_group.vpc_endpoints.id]
38-
private_dns_enabled = true
39-
tags = merge(var.tags, {
40-
Name = "${var.prefix}-cloudwatch-logs-vpce"
41-
})
33+
vpc_id = var.vpc_id
34+
service_name = "com.amazonaws.${var.aws_region}.logs"
35+
vpc_endpoint_type = "Interface"
36+
subnet_ids = var.private_subnet_ids
37+
security_group_ids = [aws_security_group.vpc_endpoints.id]
38+
private_dns_enabled = true
39+
tags = merge(var.tags, {
40+
Name = "${var.prefix}-cloudwatch-logs-vpce"
41+
})
4242
}
4343

4444
# VPC Endpoint for S3 as ECR stores image layers in S3
4545
resource "aws_vpc_endpoint" "s3" {
46-
vpc_id = var.vpc_id
47-
service_name = "com.amazonaws.${var.aws_region}.s3"
48-
vpc_endpoint_type = "Gateway"
49-
route_table_ids = var.route_table_ids
50-
tags = merge(var.tags, {
51-
Name = "${var.prefix}-s3-vpce"
52-
})
46+
vpc_id = var.vpc_id
47+
service_name = "com.amazonaws.${var.aws_region}.s3"
48+
vpc_endpoint_type = "Gateway"
49+
route_table_ids = var.route_table_ids
50+
tags = merge(var.tags, {
51+
Name = "${var.prefix}-s3-vpce"
52+
})
5353
}
5454

5555
# Security group for VPC endpoints
@@ -59,10 +59,10 @@ resource "aws_security_group" "vpc_endpoints" {
5959
vpc_id = var.vpc_id
6060

6161
ingress {
62-
from_port = 443
63-
to_port = 443
64-
protocol = "tcp"
65-
security_groups = [var.ecs_sg_id]
62+
from_port = 443
63+
to_port = 443
64+
protocol = "tcp"
65+
security_groups = [var.ecs_sg_id]
6666
}
6767

6868
egress {

0 commit comments

Comments
 (0)