Merge pull request #2097 from LBHackney-IT/revert-tascomi-aws-migration

Revert tascomi aws migration

Author: LBHSBALLEY

notebook/scripts/planning/load-table-from-tascomi-API-endpoint.ipynb

@@ -58,7 +58,7 @@
     "private_key = PRIVATE_KEY.encode('utf-8')\n",
     "\n",
     "table_to_read = \"\"\n",
-    "request_uri = f'https://hackney-planning.idoxcloud.com/rest/v1/{table_to_read}'\n",
+    "request_uri = f'https://hackney-planning.tascomi.com/rest/v1/{table_to_read}'\n",
     "request_method = 'GET'"
    ]
   },

scripts/jobs/planning/tascomi_api_ingestion.py

@@ -87,7 +87,7 @@ def get_number_of_pages(resource, query=""):
 
     headers = authenticate_tascomi(headers, public_key, private_key)
 
-    url = f'https://hackney-planning.idoxcloud.com/rest/v1/{resource}{query}'
+    url = f'https://hackney-planning.tascomi.com/rest/v1/{resource}{query}'
     res = requests.get(url, data="", headers=headers)
     if res.status_code == 202:
         logger.info(f"received status code 202, whilst getting number of pages for {resource}, with query {query}")
@@ -144,7 +144,7 @@ def get_requests_since_last_import(resource, last_import_date):
         number_of_pages = number_of_pages_reponse["number_of_pages"]
         logger.info(f"Number of pages to retrieve for {day}: {number_of_pages}")
         requests_list += [RequestRow(page_number,
-                                     f'https://hackney-planning.idoxcloud.com/rest/v1/{resource}?page={page_number}&last_updated={day}',
+                                     f'https://hackney-planning.tascomi.com/rest/v1/{resource}?page={page_number}&last_updated={day}',
                                      "") for page_number in range(1, number_of_pages + 1)]
     number_of_requests = len(requests_list)
     if number_of_requests == 0:
@@ -162,7 +162,7 @@ def get_requests_for_full_load(resource):
     number_of_pages = number_of_pages_reponse["number_of_pages"]
     logger.info(f"Number of pages to retrieve: {number_of_pages}")
     requests_list = [
-        RequestRow(page_number, f'https://hackney-planning.idoxcloud.com/rest/v1/{resource}?page={page_number}', "") for
+        RequestRow(page_number, f'https://hackney-planning.tascomi.com/rest/v1/{resource}?page={page_number}', "") for
         page_number in range(1, number_of_pages + 1)]
     number_of_requests = len(requests_list)
     requests_list = sc.parallelize(requests_list)

scripts/tests/planning/test_tascomi_parse_tables.py

@@ -53,7 +53,7 @@ def test_parsed_row_data(self, spark):
             "creation_user_id": None,
             "title_id": "4",
             "page_number": 691,
-            "import_api_url_requested": "https://hackney-planning.idoxcloud.com/rest/v1/contacts?page=691",
+            "import_api_url_requested": "https://hackney-planning.tascomi.com/rest/v1/contacts?page=691",
             "import_api_status_code": 200,
             "import_exception_thrown": "",
             "import_datetime": datetime(2021, 9, 16, 13, 10),
@@ -69,7 +69,7 @@ def parse_json_into_dataframe(self, spark, column, data):
         data_with_imports = [
             {
                 "page_number": 691,
-                "import_api_url_requested": "https://hackney-planning.idoxcloud.com/rest/v1/contacts?page=691",
+                "import_api_url_requested": "https://hackney-planning.tascomi.com/rest/v1/contacts?page=691",
                 "import_api_status_code": 200,
                 "import_exception_thrown": "",
                 "import_datetime": datetime(2021, 9, 16, 13, 10),

terraform/core/15-unrestricted-geolive-database-ingestion.tf

@@ -280,6 +280,7 @@ module "llpg_geolive_ingestion_job" {
   jdbc_connections           = [module.llpg_geolive_database_ingestion[0].jdbc_connection_name]
   triggered_by_crawler       = module.llpg_geolive_database_ingestion[0].crawler_name
   workflow_name              = module.llpg_geolive_database_ingestion[0].workflow_name
+  max_retries = 1
   job_parameters = {
     "--s3_ingestion_bucket_target"  = "s3://${module.raw_zone.bucket_id}/unrestricted/geolive/llpg/"
     "--s3_ingestion_details_target" = "s3://${module.raw_zone.bucket_id}/unrestricted/geolive/llpg/ingestion-details/"

terraform/etl/24-aws-glue-tascomi-data.tf

@@ -142,7 +142,7 @@ resource "aws_glue_trigger" "tascomi_tables_weekly_ingestion_triggers" {
 
   name     = "${local.short_identifier_prefix}Tascomi ${title(replace(each.value, "_", " "))} Ingestion Trigger"
   type     = "SCHEDULED"
-  schedule = "cron(0 21 ? * MON *)"
+  schedule = "cron(0 16 ? * SUN *)"
   enabled  = local.is_production_environment
 
   actions {