Skip to content

Commit 47a4cb9

Browse files
MohammadIqbalAD-NHSMohammadIqbalAD-NHS
authored
[PRMT-32] Data Collection ECS (#619)
1 parent 4b17d6e commit 47a4cb9

File tree

11 files changed

+49
-267
lines changed

11 files changed

+49
-267
lines changed

.github/workflows/base-batch-update.yml renamed to .github/workflows/base-data-collection.yml

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: "Z-BASE Deploy Batch Update: Build batch update image"
1+
name: "Z-BASE Deploy Data Collection: Build data collection image"
22

33
on:
44
workflow_call:
@@ -22,7 +22,7 @@ permissions:
2222
contents: read # This is required for actions/checkout
2323

2424
jobs:
25-
batch_update_build_docker_image:
25+
data_collection_build_docker_image:
2626
runs-on: ubuntu-latest
2727
environment: ${{ inputs.environment }}
2828
defaults:
@@ -52,11 +52,11 @@ jobs:
5252
id: build-image
5353
env:
5454
ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
55-
ECR_REPOSITORY: ${{ inputs.sandbox }}-weekly-ods-update
55+
ECR_REPOSITORY: ${{ inputs.sandbox }}-data-collection
5656
IMAGE_TAG: latest
5757
IMAGE_TAG_SHA: ${{ github.sha }}
5858
run: |
59-
docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG_SHA .
59+
docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG_SHA -f ecs/data_collection/Dockerfile .
6060
docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
6161
docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG_SHA
6262
echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG_SHA" >> $GITHUB_OUTPUT
@@ -66,25 +66,25 @@ jobs:
6666
- name: Download task definition
6767
id: download-task
6868
run: |
69-
aws ecs describe-task-definition --task-definition ${{ inputs.sandbox }}-task-ods-weekly-update --query taskDefinition > task-definition.json
69+
aws ecs describe-task-definition --task-definition ${{ inputs.sandbox }}-task-data-collection --query taskDefinition > task-definition.json
7070
echo "revision=$(cat task-definition.json | jq .revision)" >> $GITHUB_OUTPUT
7171
7272
- name: Fill in the new image ID in the Amazon ECS task definition
7373
id: task-def
7474
uses: aws-actions/amazon-ecs-render-task-definition@v1
7575
with:
7676
task-definition: ./lambdas/task-definition.json
77-
container-name: ${{ inputs.sandbox }}-container-ods-weekly-update
77+
container-name: ${{ inputs.sandbox }}-container-data-collection
7878
image: ${{ steps.build-image.outputs.image }}
7979

8080
- name: Deploy Amazon ECS task definition
8181
uses: aws-actions/amazon-ecs-deploy-task-definition@v2
8282
with:
8383
task-definition: ./lambdas/task-definition.json
84-
cluster: ${{ inputs.sandbox }}-ods-weekly-update
84+
cluster: ${{ inputs.sandbox }}-data-collection
8585
wait-for-service-stability: true
8686

8787
- name: De-register previous revision
8888
run: |
8989
aws ecs deregister-task-definition \
90-
--task-definition ${{ inputs.sandbox }}-task-ods-weekly-update:${{ steps.download-task.outputs.revision }}
90+
--task-definition ${{ inputs.sandbox }}-task-data-collection:${{ steps.download-task.outputs.revision }}

.github/workflows/batch-update-deploy-to-sandbox.yml renamed to .github/workflows/data-collection-deploy-to-sandbox.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: 'SANDBOX Batch Update - Publish Batch Update Image to ECR'
1+
name: 'SANDBOX Data Collection - Publish Data Collection Image to ECR'
22

33
on:
44
workflow_dispatch:
@@ -47,7 +47,7 @@ permissions:
4747
jobs:
4848
push_image:
4949
name: Push Image
50-
uses: ./.github/workflows/base-batch-update.yml
50+
uses: ./.github/workflows/base-data-collection.yml
5151
with:
5252
build_branch: ${{ inputs.build_branch }}
5353
environment: ${{ inputs.environment }}

‎.github/workflows/full-deploy-to-pre-prod.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -116,10 +116,10 @@ jobs:
116116
secrets:
117117
AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }}
118118

119-
deploy_ods_batch_update:
120-
name: Deploy ODS Batch Update
119+
deploy_data_collection:
120+
name: Deploy ODS Data Collection
121121
needs: ["tag_and_release"]
122-
uses: ./.github/workflows/base-batch-update.yml
122+
uses: ./.github/workflows/base-data-collection.yml
123123
with:
124124
build_branch: ${{ needs.tag_and_release.outputs.version }}
125125
environment: pre-prod

‎.github/workflows/full-deploy-to-prod.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -63,9 +63,9 @@ jobs:
6363
secrets:
6464
AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }}
6565

66-
deploy_ods_batch_update:
67-
name: Deploy ODS Batch Update
68-
uses: ./.github/workflows/base-batch-update.yml
66+
deploy_data_collection:
67+
name: Deploy ODS Data Collection
68+
uses: ./.github/workflows/base-data-collection.yml
6969
with:
7070
build_branch: ${{ inputs.tag_version }}
7171
environment: prod

‎.github/workflows/lambdas-dev-to-main-ci.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -62,10 +62,10 @@ jobs:
6262
secrets:
6363
AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }}
6464

65-
deploy_ods_batch_update:
66-
name: Deploy ODS Batch Update
65+
deploy_data_collection:
66+
name: Deploy ODS Data Collection
6767
needs: [ "run_tests" ]
68-
uses: ./.github/workflows/base-batch-update.yml
68+
uses: ./.github/workflows/base-data-collection.yml
6969
if: github.ref == 'refs/heads/main'
7070
with:
7171
build_branch: ${{ github.event.pull_request.head.ref }}

‎.gitignore‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -101,8 +101,6 @@ node_modules/
101101
lambdas/tests/unit/helpers/data/pdf/tmp
102102
/lambdas/package_/
103103

104-
batch_update_progress.json
105-
106104
# jupyter notebook files
107105
*.ipynb
108106

‎lambdas/ecs/__init__.py‎

Whitespace-only changes.

lambdas/Dockerfile renamed to lambdas/ecs/data_collection/Dockerfile

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,11 @@ ENV PYTHONUNBUFFERED=1
55
ENV PYTHONPATH=/lambdas
66

77
WORKDIR /lambdas
8-
COPY requirements /lambdas/requirements
8+
COPY ../../requirements /lambdas/requirements
99

1010
RUN pip install -r requirements/layers/requirements_core_lambda_layer.txt
1111
RUN pip install -r requirements/layers/requirements_data_lambda_layer.txt
1212

13-
COPY . /lambdas
13+
COPY ../../ /lambdas
1414

15-
ENTRYPOINT ["python", "scripts/batch_update_ods_code.py"]
15+
ENTRYPOINT ["python", "ecs/data_collection/main.py"]

‎lambdas/ecs/data_collection/__init__.py‎

Whitespace-only changes.

‎lambdas/ecs/data_collection/main.py‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
import importlib
2+
import logging
3+
import sys
4+
5+
from services.data_collection_service import DataCollectionService
6+
from services.statistical_report_service import StatisticalReportService
7+
8+
9+
def setup_logging_for_local_script():
10+
importlib.reload(logging)
11+
12+
logging.basicConfig(
13+
level=logging.INFO,
14+
format="[%(asctime)s] %(levelname)s [%(name)s.%(funcName)s:%(lineno)d] %(message)s",
15+
datefmt="%d/%b/%Y %H:%M:%S",
16+
stream=sys.stdout,
17+
)
18+
19+
20+
if __name__ == "__main__":
21+
setup_logging_for_local_script()
22+
print("Starting data collection process")
23+
data_collection_service = DataCollectionService()
24+
data_collection_service.collect_all_data_and_write_to_dynamodb()
25+
print("Starting to create statistical report")
26+
statistical_report_service = StatisticalReportService()
27+
statistical_report_service.make_weekly_summary_and_output_to_bucket()

0 commit comments

Comments
 (0)