Merge branch 'main' into di-543-govnotify-ingestion-for-customer-services

Author: annajgibson

.github/workflows/deploy_terraform.yml

@@ -77,7 +77,7 @@ on:
 jobs:
   deploy:
     name: Terraform Apply
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     environment: ${{ inputs.environment }}
     steps:
       - name: Checkout Source
@@ -86,7 +86,7 @@ jobs:
       - name: Set Github Auth
         run: git config --global url."https://oauth2:${{ secrets.TERRAFORM_SECRET_TOKEN}}@github.com".insteadOf https://github.com
         shell: bash
-      
+
       - name: Install Terraform
         uses: hashicorp/[email protected]
         with:

.github/workflows/deploy_terraform_networking.yml

@@ -73,15 +73,15 @@ on:
 jobs:
   deploy:
     name: Terraform Apply
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     environment: ${{ inputs.environment }}
     steps:
       - name: Checkout Source
         uses: actions/checkout@v3
 
       - name: Set Github Auth
         run: git config --global url."https://oauth2:${{ secrets.TERRAFORM_SECRET_TOKEN}}@github.com".insteadOf https://github.com
-        shell: bash   
+        shell: bash
 
       - name: Install Terraform
         uses: hashicorp/[email protected]

.github/workflows/flake8_linter_python_files.yml

@@ -23,7 +23,7 @@ jobs:
 
       - name: Get Changed Files
         id: changed-files
-        uses: tj-actions/changed-files@v44
+        uses: tj-actions/changed-files@2f7c5bfce28377bc069a65ba478de0a74aa0ca32 # v46.0.1
         with:
           files: |
             **.py

.github/workflows/lint-terraform.yml

@@ -66,15 +66,15 @@ on:
 jobs:
   lint:
     name: Terraform Lint
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout
         uses: actions/checkout@v3
 
       - name: Set Github Auth
         run: git config --global url."https://oauth2:${{ secrets.TERRAFORM_SECRET_TOKEN}}@github.com".insteadOf https://github.com
         shell: bash
-      
+
       - name: Install Terraform
         uses: hashicorp/[email protected]
         with:

.github/workflows/plan-terraform.yml

@@ -66,17 +66,17 @@ on:
 jobs:
   plan:
     name: Terraform Plan
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     steps:
       - uses: terraform-compliance/github_action@main
 
       - name: Checkout
         uses: actions/checkout@v3
-      
+
       - name: Set Github Auth
         run: git config --global url."https://oauth2:${{ secrets.TERRAFORM_SECRET_TOKEN}}@github.com".insteadOf https://github.com
         shell: bash
-      
+
       - name: Install Terraform
         uses: hashicorp/[email protected]
         with:

.github/workflows/test-python-and-lambda.yml

@@ -6,7 +6,7 @@ on:
 jobs:
   tests:
     name: Test Python Jobs & Lambda Functions
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout
         uses: actions/checkout@v3

.github/workflows/unlock_terraform_state.yml

@@ -76,7 +76,7 @@ on:
 jobs:
   deploy:
     name: Terraform State Unlock
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout Source
         uses: actions/checkout@v3

.github/workflows/validate-and-lint-terraform.yml

@@ -66,7 +66,7 @@ on:
 jobs:
   validate:
     name: Terraform Validate
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -108,15 +108,15 @@ jobs:
 
   lint:
     name: Terraform Lint
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout
         uses: actions/checkout@v3
 
       - name: Set Github Auth
         run: git config --global url."https://oauth2:${{ secrets.TERRAFORM_SECRET_TOKEN}}@github.com".insteadOf https://github.com
         shell: bash
-      
+
       - name: Install Terraform
         uses: hashicorp/[email protected]
         with:

lambdas/icaseworks_api_ingestion/Makefile

@@ -5,6 +5,6 @@ install-requirements:
 	# the requirements are generated so that the packages
 	# could be downloaded and packaged up for the lambda
 
-	. venv/bin/activate && sudo pipenv lock --requirements > requirements.txt
+	. venv/bin/activate && sudo pipenv requirements > requirements.txt
 	. venv/bin/activate && sudo pip install --target ./lib -r requirements.txt
 	rm -rf venv/

lambdas/icaseworks_api_ingestion/main.py

@@ -1,31 +1,33 @@
-import sys
-
-sys.path.append('./lib/')
-
-import pybase64
-import json
+import datetime
 import hashlib
 import hmac
+import json
+import logging
 import re
-import requests
+import sys
 import time
-import boto3
-from dotenv import load_dotenv
 from os import getenv
-import datetime
-import logging
+
+
+# needs to be above imports for the additional dependencies
+# flake8 E402 would complain about this so is ignored
+sys.path.append("./lib/")
+
+import boto3  # noqa: E402
+import pybase64  # noqa: E402
+import requests  # noqa: E402
+from dotenv import load_dotenv  # noqa: E402
+
 
 logger = logging.getLogger()
 logger.setLevel(logging.INFO)
 
 
 def remove_illegal_characters(string):
     """Removes illegal characters from string"""
-    regex_list = [['=', ""], ['\/', "_"], ['+', "-"]]
+    regex_list = [["=", ""], ["\/", "_"], ["+", "-"]]  # noqa: W605
     for r in regex_list:
-        string = re.sub(string=string,
-                        pattern="[{}]".format(r[0]),
-                        repl=r[1])
+        string = re.sub(string=string, pattern="[{}]".format(r[0]), repl=r[1])
     return string
 
 
@@ -44,37 +46,39 @@ def dictionary_to_string(dictionary):
 def create_signature(header, payload, secret):
     """Encode JSON string"""
     # hashed header, hashed payload, string secret
-    unsigned_token = header + '.' + payload
-    key_bytes = bytes(secret, 'utf-8')
-    string_to_sign_bytes = bytes(unsigned_token, 'utf-8')
-    signature_hash = hmac.new(key_bytes, string_to_sign_bytes, digestmod=hashlib.sha256).digest()
+    unsigned_token = header + "." + payload
+    key_bytes = bytes(secret, "utf-8")
+    string_to_sign_bytes = bytes(unsigned_token, "utf-8")
+    signature_hash = hmac.new(
+        key_bytes, string_to_sign_bytes, digestmod=hashlib.sha256
+    ).digest()
     encoded_signature = pybase64.b64encode(signature_hash)
-    encoded_signature = encoded_signature.decode('utf-8')
+    encoded_signature = encoded_signature.decode("utf-8")
     encoded_signature = remove_illegal_characters(encoded_signature)
     return encoded_signature
 
 
 def get_token(url, encoded_header, encoded_payload, signature, headers):
     """Get token"""
     assertion = encoded_header + "." + encoded_payload + "." + signature
-    data = f'assertion={assertion}&grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Ajwt-bearer'
+    data = f"assertion={assertion}&grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Ajwt-bearer"
     response = requests.post(url, headers=headers, data=data)
     response_json = response.json()
-    auth_token = response_json.get('access_token')
+    auth_token = response_json.get("access_token")
     return auth_token
 
 
 def get_icaseworks_report_from(report_id, from_date, auth_headers, auth_payload):
     report_url = "https://hackneyreports.icasework.com/getreport?"
-    request_url = f'{report_url}ReportId={report_id}&Format=json&From={from_date}'
-    logger.info(f'Request url: {request_url}')
+    request_url = f"{report_url}ReportId={report_id}&Format=json&From={from_date}"
+    logger.info(f"Request url: {request_url}")
     response = requests.get(request_url, headers=auth_headers, data=auth_payload)
-    logger.info(f'Status Code: {response.status_code}')
+    logger.info(f"Status Code: {response.status_code}")
     return response.content
 
 
 def write_dataframe_to_s3(s3_client, data, s3_bucket, output_folder, filename):
-    filename = re.sub('[^a-zA-Z0-9]+', '-', filename).lower()
+    filename = re.sub("[^a-zA-Z0-9]+", "-", filename).lower()
     current_date = datetime.datetime.now()
     day = single_digit_to_zero_prefixed_string(current_date.day)
     month = single_digit_to_zero_prefixed_string(current_date.month)
@@ -83,7 +87,7 @@ def write_dataframe_to_s3(s3_client, data, s3_bucket, output_folder, filename):
     return s3_client.put_object(
         Bucket=s3_bucket,
         Body=data,
-        Key=f"{output_folder}/import_year={year}/import_month={month}/import_day={day}/import_date={date}/{filename}.json"
+        Key=f"{output_folder}/import_year={year}/import_month={month}/import_day={day}/import_date={date}/{filename}.json",
     )
 
 
@@ -102,14 +106,16 @@ def lambda_handler(event, lambda_context):
     url = "https://hackney.icasework.com/token"
 
     headers = {
-        'Content-Type': 'application/x-www-form-urlencoded',
+        "Content-Type": "application/x-www-form-urlencoded",
     }
 
     # Get api api credentials from secrets manager
     secret_name = getenv("SECRET_NAME")
-    secrets_manager_client = boto3.client('secretsmanager')
-    api_credentials_response = retrieve_credentials_from_secrets_manager(secrets_manager_client, secret_name)
-    api_credentials = json.loads(api_credentials_response['SecretString'])
+    secrets_manager_client = boto3.client("secretsmanager")
+    api_credentials_response = retrieve_credentials_from_secrets_manager(
+        secrets_manager_client, secret_name
+    )
+    api_credentials = json.loads(api_credentials_response["SecretString"])
     api_key = api_credentials.get("api_key")
     secret = api_credentials.get("secret")
 
@@ -122,11 +128,7 @@ def lambda_handler(event, lambda_context):
     # Create payload
     current_unix_time = int(time.time())
     str_time = str(current_unix_time)
-    payload_object = {
-        "iss": api_key,
-        "aud": url,
-        "iat": str_time
-    }
+    payload_object = {"iss": api_key, "aud": url, "iat": str_time}
 
     payload_object = dictionary_to_string(payload_object)
 
@@ -136,16 +138,21 @@ def lambda_handler(event, lambda_context):
     signature = create_signature(header, payload, secret)
 
     # Get token from response
-    auth_token = get_token(url=url, encoded_header=header, encoded_payload=payload, signature=signature,
-                           headers=headers)
+    auth_token = get_token(
+        url=url,
+        encoded_header=header,
+        encoded_payload=payload,
+        signature=signature,
+        headers=headers,
+    )
 
     # Create auth header for API Calls and auth payload
-    authorization = f'Bearer {auth_token}'
+    authorization = f"Bearer {auth_token}"
 
     auth_payload = {}
 
     auth_headers = {
-        'Authorization': authorization,
+        "Authorization": authorization,
     }
 
     report_tables = [
@@ -164,22 +171,32 @@ def lambda_handler(event, lambda_context):
     date_to_track_from = today - datetime.timedelta(days=1)
     logger.info(f"Date to track from: {date_to_track_from}")
 
-    s3_client = boto3.client('s3')
+    s3_client = boto3.client("s3")
 
     for report_details in report_tables:
-        logger.info(f'Pulling report for {report_details["name"]}')
+        logger.info(f"Pulling report for {report_details['name']}")
         case_id_report_id = report_details["id"]
-        case_id_list = get_icaseworks_report_from(case_id_report_id, date_to_track_from, auth_headers, auth_payload)
+        case_id_list = get_icaseworks_report_from(
+            case_id_report_id, date_to_track_from, auth_headers, auth_payload
+        )
         report_details["data"] = case_id_list
-        write_dataframe_to_s3(s3_client, report_details["data"], s3_bucket, output_folder_name, report_details["name"])
-        logger.info(f'Finished writing report for {report_details["name"]} to S3')
+        write_dataframe_to_s3(
+            s3_client,
+            report_details["data"],
+            s3_bucket,
+            output_folder_name,
+            report_details["name"],
+        )
+        logger.info(f"Finished writing report for {report_details['name']} to S3")
 
     # Trigger glue job to copy from landing to raw and convert to parquet
-    glue_client = boto3.client('glue')
+    glue_client = boto3.client("glue")
     start_glue_trigger(glue_client, glue_trigger_name)
 
+
 def single_digit_to_zero_prefixed_string(value):
-    return str(value) if value > 9 else '0' + str(value)
+    return str(value) if value > 9 else "0" + str(value)
+
 
 def start_glue_trigger(glue_client, trigger_name):
     trigger_details = glue_client.start_trigger(Name=trigger_name)