Skip to content

Commit d7ade1d

Browse files
eddalmond1eddalmond1
authored
Merge branch 'main' into chore/eja-adding-main-branch-scanning-for-sonarcube
2 parents 5430700 + eb10cb2 commit d7ade1d

File tree

7 files changed

+270
-25
lines changed

7 files changed

+270
-25
lines changed

.github/workflows/cicd-4-preprod-deploy.yml renamed to .github/workflows/cicd-4a-preprod-deploy.yml

File renamed without changes.

‎.github/workflows/cicd-4b-preprod-seed-users.yml‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
name: preprod - Seed DynamoDB table
2+
3+
concurrency:
4+
group: seed-preprod-dynamodb
5+
cancel-in-progress: false
6+
7+
on:
8+
workflow_run:
9+
workflows: [ "Preprod Deploy" ]
10+
types:
11+
- completed
12+
filters:
13+
conclusion:
14+
- success
15+
workflow_dispatch:
16+
inputs:
17+
environment:
18+
description: Target environment
19+
required: true
20+
type: choice
21+
options:
22+
- preprod
23+
24+
jobs:
25+
seed-dynamodb:
26+
runs-on: ubuntu-latest
27+
environment: "preprod"
28+
permissions:
29+
id-token: write
30+
contents: read
31+
env:
32+
AWS_REGION: eu-west-2
33+
DATA_FOLDER: tests/e2e/data/dynamoDB/vitaIntegrationTestData
34+
DYNAMODB_TABLE: eligibility-signposting-api-preprod-eligibility_datastore
35+
36+
steps:
37+
- name: Checkout repo
38+
uses: actions/checkout@v5
39+
40+
- name: Set up Python
41+
uses: actions/setup-python@v5
42+
with:
43+
python-version: '3.13'
44+
45+
- name: Install dependencies
46+
run: pip install boto3
47+
48+
- name: "Configure AWS Credentials"
49+
uses: aws-actions/configure-aws-credentials@v4
50+
with:
51+
role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/service-roles/github-actions-api-deployment-role
52+
aws-region: ${{ env.AWS_REGION }}
53+
54+
- name: Run seed script
55+
run: |
56+
python scripts/seed_users/seed_dynamodb.py \
57+
--table-name "${{ env.DYNAMODB_TABLE }}" \
58+
--region "${{ env.AWS_REGION }}" \
59+
--data-folder "${{ env.DATA_FOLDER }}"

‎infrastructure/stacks/iams-developer-roles/github_actions_policies.tf‎

Lines changed: 42 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -89,24 +89,42 @@ resource "aws_iam_policy" "dynamodb_management" {
8989

9090
policy = jsonencode({
9191
Version = "2012-10-17",
92-
Statement = [
93-
{
94-
Effect = "Allow",
95-
Action = [
96-
"dynamodb:DescribeTimeToLive",
97-
"dynamodb:DescribeTable",
98-
"dynamodb:DescribeContinuousBackups",
99-
"dynamodb:ListTables",
100-
"dynamodb:DeleteTable",
101-
"dynamodb:CreateTable",
102-
"dynamodb:TagResource",
103-
"dynamodb:ListTagsOfResource",
104-
],
105-
Resource = [
106-
"arn:aws:dynamodb:*:${data.aws_caller_identity.current.account_id}:table/*eligibility-signposting-api-${var.environment}-eligibility_datastore"
107-
]
108-
}
109-
]
92+
Statement = concat(
93+
[
94+
{
95+
Effect = "Allow",
96+
Action = [
97+
"dynamodb:DescribeTimeToLive",
98+
"dynamodb:DescribeTable",
99+
"dynamodb:DescribeContinuousBackups",
100+
"dynamodb:ListTables",
101+
"dynamodb:DeleteTable",
102+
"dynamodb:CreateTable",
103+
"dynamodb:TagResource",
104+
"dynamodb:ListTagsOfResource",
105+
],
106+
Resource = [
107+
"arn:aws:dynamodb:*:${data.aws_caller_identity.current.account_id}:table/*eligibility-signposting-api-${var.environment}-eligibility_datastore"
108+
]
109+
}
110+
],
111+
# to create test users in preprod
112+
var.environment == "preprod" ? [
113+
{
114+
Effect = "Allow",
115+
Action = [
116+
"dynamodb:GetItem",
117+
"dynamodb:PutItem",
118+
"dynamodb:DeleteItem",
119+
"dynamodb:Scan",
120+
"dynamodb:BatchWriteItem"
121+
],
122+
Resource = [
123+
"arn:aws:dynamodb:*:${data.aws_caller_identity.current.account_id}:table/*eligibility-signposting-api-${var.environment}-eligibility_datastore"
124+
]
125+
}
126+
] : []
127+
)
110128
})
111129

112130
tags = merge(local.tags, { Name = "dynamodb-management" })
@@ -465,8 +483,8 @@ resource "aws_iam_policy" "iam_management" {
465483
# Assume role policy document for GitHub Actions
466484
data "aws_iam_policy_document" "github_actions_assume_role" {
467485
statement {
468-
sid = "OidcAssumeRoleWithWebIdentity"
469-
effect = "Allow"
486+
sid = "OidcAssumeRoleWithWebIdentity"
487+
effect = "Allow"
470488
actions = ["sts:AssumeRoleWithWebIdentity"]
471489

472490
principals {
@@ -479,13 +497,13 @@ data "aws_iam_policy_document" "github_actions_assume_role" {
479497
condition {
480498
test = "StringLike"
481499
variable = "token.actions.githubusercontent.com:sub"
482-
values = ["repo:${var.github_org}/${var.github_repo}:*"]
500+
values = ["repo:${var.github_org}/${var.github_repo}:*"]
483501
}
484502

485503
condition {
486504
test = "StringEquals"
487505
variable = "token.actions.githubusercontent.com:aud"
488-
values = ["sts.amazonaws.com"]
506+
values = ["sts.amazonaws.com"]
489507
}
490508
}
491509
}
@@ -514,8 +532,8 @@ resource "aws_iam_policy" "firehose_readonly" {
514532
"firehose:StopDeliveryStreamEncryption"
515533
]
516534
Resource = [
517-
"arn:aws:firehose:${var.default_aws_region}:${data.aws_caller_identity.current.account_id}:deliverystream/eligibility-signposting-api*",
518-
"arn:aws:firehose:${var.default_aws_region}:${data.aws_caller_identity.current.account_id}:deliverystream/splunk-alarm-events*"
535+
"arn:aws:firehose:${var.default_aws_region}:${data.aws_caller_identity.current.account_id}:deliverystream/eligibility-signposting-api*",
536+
"arn:aws:firehose:${var.default_aws_region}:${data.aws_caller_identity.current.account_id}:deliverystream/splunk-alarm-events*"
519537
]
520538
}
521539
]

‎infrastructure/stacks/iams-developer-roles/iams_permissions_boundary.tf‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -233,7 +233,22 @@ data "aws_iam_policy_document" "permissions_boundary" {
233233
values = [var.default_aws_region]
234234
}
235235
}
236-
236+
# Environment-specific actions
237+
dynamic "statement" {
238+
for_each = var.environment == "preprod" ? [1] : []
239+
content {
240+
sid = "AllowPreprodDynamoDBItemOps"
241+
effect = "Allow"
242+
actions = [
243+
"dynamodb:GetItem",
244+
"dynamodb:PutItem",
245+
"dynamodb:DeleteItem",
246+
"dynamodb:Scan",
247+
"dynamodb:BatchWriteItem"
248+
]
249+
resources = ["*"]
250+
}
251+
}
237252
# Allow access to IAM actions for us-east-1 region only
238253
statement {
239254
sid = "AllowIamActionsInUsEast1"

‎scripts/seed_users/README.md‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
# 🧬 DynamoDB Seeder Script
2+
3+
This script deletes and inserts items into a DynamoDB table using JSON seed data. It’s designed for integration testing and local development workflows.
4+
This script is user in the Preprod seed workflow.
5+
6+
---
7+
8+
## 📦 Requirements
9+
10+
- Python 3.13
11+
- AWS credentials configured (via `~/.aws/credentials`, environment variables, or IAM role)
12+
- Required Python packages:
13+
14+
```bash
15+
pip install boto3
16+
```
17+
18+
---
19+
20+
## 🚀 Usage
21+
22+
From the project root, run:
23+
24+
```bash
25+
python scripts/seed_users/seed_dynamodb.py \
26+
--table-name <your-dynamodb-table-name> \
27+
--region <aws-region> \
28+
--data-folder <path-to-json-folder>
29+
```
30+
31+
### Example
32+
33+
```bash
34+
python scripts/seed_users/seed_dynamodb.py \
35+
--table-name eligibility-signposting-api-dev-eligibility_datastore \
36+
--region eu-west-2 \
37+
--data-folder tests/e2e/data/dynamoDB/vitaIntegrationTestData
38+
```
39+
40+
---
41+
42+
## 📁 JSON Data Format
43+
44+
Each `.json` file in the specified folder should follow this structure:
45+
46+
```json
47+
{
48+
"data": [
49+
{
50+
"NHS_NUMBER": "1234567890",
51+
"ATTRIBUTE_TYPE": "COHORTS",
52+
"otherAttribute1": "value",
53+
"otherAttribute2": "value"
54+
}
55+
]
56+
}
57+
```
58+
59+
## 🧹 What It Does
60+
61+
1. **Deletes** existing items in the table matching `NHS_NUMBER` from all JSON files.
62+
2. **Inserts** all items from the same files into the table.
63+
64+
---
65+
66+
## 🛡️ Safety Notes
67+
68+
- This script performs destructive operations — do not use this in prod environment.
69+
- Ensure your AWS credentials have appropriate permissions for `dynamodb:DeleteItem` and `dynamodb:PutItem`.
70+
71+
---

‎scripts/seed_users/__init__.py‎

Whitespace-only changes.

‎scripts/seed_users/seed_dynamodb.py‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
import argparse
2+
import glob
3+
import json
4+
import os
5+
6+
import boto3
7+
8+
9+
def parse_args():
10+
parser = argparse.ArgumentParser(description="Seed DynamoDB table with JSON data.")
11+
parser.add_argument("--table-name", required=True, help="Name of the DynamoDB table")
12+
parser.add_argument("--region", default="eu-west-2", help="AWS region")
13+
parser.add_argument("--data-folder", default="vitaIntegrationTestData/", help="Folder containing JSON seed data")
14+
return parser.parse_args()
15+
16+
17+
def resolve_data_folder(path):
18+
return os.path.abspath(path)
19+
20+
21+
def get_unique_nhs_numbers(data_folder):
22+
nhs_numbers = set()
23+
json_files = glob.glob(os.path.join(data_folder, "*.json"))
24+
for file_path in json_files:
25+
with open(file_path) as f:
26+
payload = json.load(f)
27+
items = payload.get("data", [])
28+
for item in items:
29+
nhs_number = item.get("NHS_NUMBER")
30+
if nhs_number:
31+
nhs_numbers.add(nhs_number)
32+
return list(nhs_numbers)
33+
34+
35+
def delete_all_items_for_nhs_numbers(table, nhs_numbers):
36+
for nhs_number in nhs_numbers:
37+
response = table.query(
38+
KeyConditionExpression=boto3.dynamodb.conditions.Key("NHS_NUMBER").eq(nhs_number)
39+
)
40+
items = response.get("Items", [])
41+
with table.batch_writer() as batch:
42+
for item in items:
43+
key = {
44+
"NHS_NUMBER": item["NHS_NUMBER"],
45+
"ATTRIBUTE_TYPE": item["ATTRIBUTE_TYPE"]
46+
}
47+
batch.delete_item(Key=key)
48+
49+
50+
def insert_data_from_folder(table, data_folder):
51+
json_files = glob.glob(os.path.join(data_folder, "*.json"))
52+
for file_path in json_files:
53+
with open(file_path) as f:
54+
payload = json.load(f)
55+
items = payload.get("data", [])
56+
57+
with table.batch_writer() as batch:
58+
for item in items:
59+
nhs_number = item.get("NHS_NUMBER")
60+
attr_type = item.get("ATTRIBUTE_TYPE")
61+
if nhs_number and attr_type:
62+
item["id"] = nhs_number
63+
batch.put_item(Item=item)
64+
65+
66+
def main():
67+
args = parse_args()
68+
69+
dynamodb = boto3.resource("dynamodb", region_name=args.region)
70+
table = dynamodb.Table(args.table_name)
71+
72+
data_folder = resolve_data_folder(args.data_folder)
73+
if not os.path.isdir(data_folder):
74+
raise ValueError(f"Data folder '{data_folder}' does not exist or is not a directory.")
75+
76+
nhs_numbers = get_unique_nhs_numbers(data_folder)
77+
delete_all_items_for_nhs_numbers(table, nhs_numbers)
78+
insert_data_from_folder(table, data_folder)
79+
80+
81+
if __name__ == "__main__":
82+
main()

0 commit comments

Comments
 (0)