VED-738 refactoring (#893)

* -> utilities/

* utilities: package.json

* utilities: esc-proxies-containers

* utilities: esc-proxies-containers II

* utilities: azure build

* tests

* utils/resource.py pathname

* utils/resource.py pathname II

* utils/resource.py pathname III

* pipelines/azure

* Revert "pipelines/azure"

This reverts commit cebbafb.

* tests/tests -> sample_tests

* temporary_sandbox -> infrastructure/

* empty commit: see https://github.com/orgs/community/discussions/59774

* empty commit...

* grafana

* dependabot fix: sandbox

* dependabot fix: grafana

* terraform_aws_backup

* sonar-project.properties

* proxies

* terraform

* rm minified

* prettier

* terraform path fix

* fix path

* cd terraform

* mesh_infra

* infra (untested)

* sonar

* infra->global; mesh_infra->mesh

* lint

* remove temporary_sandbox

* remove temporary_sandbox II

* remove temporary_sandbox III

* ->/sandbox

* ->/sandbox II

* ->/specification

* global->account

* restored ../../../specification/...

* terraform->instance

* sonar

* working-directory

* tfplan

* paths

* paths: quality_checks

* paths: quality_checks II

* terraform fmt

* test paths

* test paths: run-e2e-tests

* test paths II

* test paths: run-e2e-tests II

* test paths III

* e2e/test_proxy

* destroy_unused_workspaces

* docker_file_path

* terraform fmt

Author: JamesW1-NHS

.github/dependabot.yml

@@ -10,11 +10,10 @@ updates:
       - "/lambdas/ack_backend"
       - "/delta_backend"
       - "/filenameprocessor"
-      - "/grafana/non-prod/docker"
+      - "/infrastructure/grafana/non-prod/docker"
       - "/mesh_processor"
       - "/recordprocessor"
       - "/sandbox"
-      - "/temporary_sandbox"
     schedule:
       interval: "daily"
     open-pull-requests-limit: 1
@@ -52,8 +51,8 @@ updates:
       - "/backend"
       - "/batch_processor_filter"
       - "/delta_backend"
-      - "/e2e"
-      - "/e2e_batch"
+      - "/tests/e2e"
+      - "/tests/e2e_batch"
       - "/filenameprocessor"
       - "/mesh_processor"
       - "/recordprocessor"
@@ -73,10 +72,10 @@ updates:
 
   - package-ecosystem: "terraform"
     directories:
-      - "/grafana/non-prod/terraform"
-      - "/infra"
-      - "/terraform"
-      - "/terraform_aws_backup/**"
+      - "/infrastructure/grafana/non-prod/terraform"
+      - "/infrastructure/account"
+      - "/infrastructure/instance"
+      - "/infrastructure/terraform_aws_backup/**"
     schedule:
       interval: "daily"
     open-pull-requests-limit: 1

.github/workflows/create-release-tag.yml

@@ -29,7 +29,7 @@ jobs:
         run: pip install semver
 
       - name: Set SPEC_VERSION env var
-        run: echo ::set-env name=SPEC_VERSION::$(python scripts/calculate_version.py)
+        run: echo ::set-env name=SPEC_VERSION::$(python utilities/scripts/calculate_version.py)
         env:
           ACTIONS_ALLOW_UNSECURE_COMMANDS: true
 

.github/workflows/deploy-backend.yml

@@ -72,18 +72,18 @@ jobs:
           terraform_version: "1.12.2"
 
       - name: Terraform Init
-        working-directory: terraform
+        working-directory: infrastructure/instance
         run: make init
 
       - name: Terraform Plan
-        working-directory: terraform
+        working-directory: infrastructure/instance
         run: make plan-ci
 
       - name: Save Terraform Plan
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
         with:
           name: ${{ env.ENVIRONMENT }}-${{ env.SUB_ENVIRONMENT }}-tfplan
-          path: terraform/tfplan
+          path: infrastructure/instance/tfplan
 
   terraform-apply:
     needs: terraform-plan
@@ -108,14 +108,14 @@ jobs:
         uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0
         with:
           name: ${{ env.ENVIRONMENT }}-${{ env.SUB_ENVIRONMENT }}-tfplan
-          path: terraform
+          path: infrastructure/instance
 
       - name: Terraform Init
-        working-directory: terraform
+        working-directory: infrastructure/instance
         run: make init
 
       - name: Terraform Apply
-        working-directory: terraform
+        working-directory: infrastructure/instance
         run: |
           make apply-ci
           echo "ID_SYNC_QUEUE_ARN=$(make -s output name=id_sync_queue_arn)" >> $GITHUB_ENV

.github/workflows/pr-teardown.yml

@@ -43,7 +43,7 @@ jobs:
           terraform_version: "1.12.2"
 
       - name: Terraform Init and extract MNS SQS QUEUE ARN
-        working-directory: terraform
+        working-directory: infrastructure/instance
         run: |
           make init apigee_environment=$APIGEE_ENVIRONMENT environment=$BACKEND_ENVIRONMENT sub_environment=$BACKEND_SUB_ENVIRONMENT
           make workspace apigee_environment=$APIGEE_ENVIRONMENT environment=$BACKEND_ENVIRONMENT sub_environment=$BACKEND_SUB_ENVIRONMENT
@@ -71,6 +71,6 @@ jobs:
           make unsubscribe
 
       - name: Terraform Destroy
-        working-directory: terraform
+        working-directory: infrastructure/instance
         run: |
           make destroy apigee_environment=$APIGEE_ENVIRONMENT environment=$BACKEND_ENVIRONMENT sub_environment=$BACKEND_SUB_ENVIRONMENT

.github/workflows/quality-checks.yml

@@ -175,7 +175,7 @@ jobs:
         working-directory: lambdas/ack_backend
         id: acklambda
         env:
-          PYTHONPATH: ${{ env.LAMBDA_PATH }}/ack_backend/src:tests:${{ env.SHARED_PATH }}/src
+          PYTHONPATH: ${{ env.LAMBDA_PATH }}/ack_backend/src:${{ env.LAMBDA_PATH }}/ack_backend/tests:${{ env.SHARED_PATH }}/src
         continue-on-error: true
         run: |
           poetry install
@@ -186,7 +186,7 @@ jobs:
         working-directory: lambdas/mns_subscription
         id: mns_subscription
         env:
-          PYTHONPATH: ${{ env.LAMBDA_PATH }}/mns_subscription/src:${{ env.SHARED_PATH }}/src
+          PYTHONPATH: ${{ env.LAMBDA_PATH }}/mns_subscription/src:${{ env.LAMBDA_PATH }}/mns_subscription/tests:${{ env.SHARED_PATH }}/src
         continue-on-error: true
         run: |
           poetry install
@@ -198,7 +198,7 @@ jobs:
         working-directory: lambdas/redis_sync
         id: redis_sync
         env:
-          PYTHONPATH: ${{ env.LAMBDA_PATH }}/redis_sync/src:${{ env.SHARED_PATH }}/src
+          PYTHONPATH: ${{ env.LAMBDA_PATH }}/redis_sync/src:${{ env.LAMBDA_PATH }}/redis_sync/tests:${{ env.SHARED_PATH }}/src
         continue-on-error: true
         run: |
           poetry install
@@ -220,7 +220,7 @@ jobs:
         working-directory: lambdas/id_sync
         id: id_sync
         env:
-          PYTHONPATH: ${{ env.LAMBDA_PATH }}/id_sync/src:${{ env.SHARED_PATH }}/src
+          PYTHONPATH: ${{ env.LAMBDA_PATH }}/id_sync/src:${{ env.LAMBDA_PATH }}/id_sync/tests:${{ env.SHARED_PATH }}/src
         continue-on-error: true
         run: |
           poetry install

.github/workflows/run-e2e-tests.yml

@@ -132,17 +132,17 @@ jobs:
 
       - name: Terraform Init
         if: ${{ env.TF_OUTPUTS_REQUIRED == 'true' }}
-        working-directory: terraform
+        working-directory: infrastructure/instance
         run: make init
 
       - name: Set Terraform workspace
         if: ${{ env.TF_OUTPUTS_REQUIRED == 'true' }}
-        working-directory: terraform
+        working-directory: infrastructure/instance
         run: make workspace
 
       - name: Read Terraform outputs
         if: ${{ env.TF_OUTPUTS_REQUIRED == 'true' }}
-        working-directory: terraform
+        working-directory: infrastructure/instance
         run: |
           echo "IMMS_DELTA_TABLE_NAME=$(make -s output name=imms_delta_table_name)" >> $GITHUB_ENV
           echo "AWS_DOMAIN_NAME=$(make -s output name=service_domain_name)" >> $GITHUB_ENV
@@ -157,15 +157,15 @@ jobs:
         with:
           python-version: 3.11
           cache: "poetry"
-          cache-dependency-path: "e2e/poetry.lock"
+          cache-dependency-path: tests/e2e/poetry.lock
 
       - name: Install e2e test dependencies
-        working-directory: e2e
+        working-directory: tests/e2e
         run: poetry install --no-root
 
       - name: Get Apigee access token
         if: ${{ vars.RUN_FULL_E2E_TESTS == 'true' }}
-        working-directory: e2e
+        working-directory: tests/e2e
         env:
           APIGEE_PASSWORD: ${{ secrets.APIGEE_PASSWORD }}
           APIGEE_BASIC_AUTH_TOKEN: ${{ secrets.APIGEE_BASIC_AUTH_TOKEN }}
@@ -187,17 +187,17 @@ jobs:
 
       - name: Run proxy e2e test suite
         if: ${{ vars.RUN_PROXY_E2E_TESTS == 'true' }}
-        working-directory: e2e
+        working-directory: tests/e2e
         run: poetry run python -m unittest test_proxy
 
       - name: Run sandbox e2e test suite
         if: ${{ vars.RUN_SANDBOX_E2E_TESTS == 'true' }}
-        working-directory: e2e
+        working-directory: tests/e2e
         run: poetry run python -m unittest test_proxy.TestProxyHealthcheck
 
       - name: Run full e2e test suite
         if: ${{ vars.RUN_FULL_E2E_TESTS == 'true' }}
-        working-directory: e2e
+        working-directory: tests/e2e
         run: poetry run python -m unittest
 
   batch-e2e-tests:
@@ -223,14 +223,14 @@ jobs:
         with:
           python-version: 3.11
           cache: "poetry"
-          cache-dependency-path: "e2e_batch/poetry.lock"
+          cache-dependency-path: tests/e2e_batch/poetry.lock
 
       - name: Install e2e test dependencies
-        working-directory: e2e_batch
+        working-directory: tests/e2e_batch
         run: poetry install --no-root
 
       - name: Run batch e2e test suite
-        working-directory: e2e_batch
+        working-directory: tests/e2e_batch
         env:
           ENVIRONMENT: ${{ inputs.sub_environment }}
         run: poetry run python -m unittest -c -v

.gitignore

@@ -17,7 +17,7 @@ __pycache__/
 .envrc
 smoketest-report.xml
 lambda_typescript/**/*.js
-terraform/zips
+instance/zips
 .dir-locals.el
 *.pyc
 .python-version

.prettierignore

@@ -3,5 +3,5 @@
 
 # We hit compile errors in Apigee if these are auto formatted
 # TODO - investigate
-/proxies/live/apiproxy/resources/jsc/
-/proxies/sandbox/apiproxy/resources/jsc/
+/infrastructure/proxies/live/apiproxy/resources/jsc/
+/infrastructure/proxies/sandbox/apiproxy/resources/jsc/

Makefile

@@ -1,7 +1,7 @@
 SHELL=/usr/bin/env bash -euo pipefail
 
 PYTHON_PROJECT_DIRS_WITH_UNIT_TESTS = backend batch_processor_filter delta_backend filenameprocessor mesh_processor recordprocessor lambdas/ack_backend lambdas/redis_sync lambdas/id_sync lambdas/mns_subscription lambdas/shared
-PYTHON_PROJECT_DIRS = e2e e2e_batch quality_checks $(PYTHON_PROJECT_DIRS_WITH_UNIT_TESTS)
+PYTHON_PROJECT_DIRS = tests/e2e tests/e2e_batch quality_checks $(PYTHON_PROJECT_DIRS_WITH_UNIT_TESTS)
 
 .PHONY: install lint format format-check clean publish build-proxy release initialise-all-python-venvs update-all-python-dependencies run-all-python-unit-tests build-all-docker-images
 
@@ -33,10 +33,10 @@ publish: clean
 
 #Runs build proxy script
 build-proxy:
-	scripts/build_proxy.sh
+	utilities/scripts/build_proxy.sh
 
 #Files to loop over in release
-_dist_include="poetry.toml Makefile build/. specification sandbox scripts"
+_dist_include="poetry.toml Makefile build/. specification sandbox utilities/scripts"
 
 #Create /dist/ sub-directory and copy files into directory
 #Ensure full dir structure is preserved for Lambdas