feat: set a schedule for executing export csv (#926)

Author: jcpitre

functions-python/export_csv/function_config.json

@@ -2,8 +2,8 @@
   "name": "export-csv",
   "description": "Export the DB feed data as a csv file",
   "entry_point": "export_and_upload_csv",
-  "timeout": 20,
-  "memory": "1Gi",
+  "timeout": 600,
+  "memory": "2Gi",
   "trigger_http": true,
   "include_folders": ["helpers", "dataset_service"],
   "include_api_folders": ["utils", "database", "feed_filters", "common", "database_gen"],

infra/functions-python/main.tf

@@ -891,9 +891,31 @@ resource "google_cloudfunctions2_function" "export_csv" {
       }
     }
   }
+}
 
+resource "google_cloud_scheduler_job" "export_csv_scheduler" {
+  name = "export-csv-scheduler-${var.environment}"
+  description = "Schedule the export_csv function"
+  time_zone = "Etc/UTC"
+  schedule = var.export_csv_schedule
+  region = var.gcp_region
+  paused = var.environment == "prod" ? false : true
+  depends_on = [google_cloudfunctions2_function.export_csv, google_cloudfunctions2_function_iam_member.export_csv_invoker]
+  http_target {
+    http_method = "POST"
+    uri = google_cloudfunctions2_function.export_csv.url
+    oidc_token {
+      service_account_email = google_service_account.functions_service_account.email
+    }
+    headers = {
+      "Content-Type" = "application/json"
+    }
+  }
+  # Export CSV can take several minutes to run (5?) so we need to give it a longer deadline
+  attempt_deadline = "600s"
 }
 
+
 # IAM entry for all users to invoke the function
 resource "google_cloudfunctions2_function_iam_member" "tokens_invoker" {
   project        = var.project_id
@@ -1050,4 +1072,12 @@ resource "google_project_iam_member" "datastore_owner" {
   project = var.project_id
   role    = "roles/datastore.owner"
   member  = "serviceAccount:${google_service_account.functions_service_account.email}"
+}
+
+resource "google_cloudfunctions2_function_iam_member" "export_csv_invoker" {
+  project        = var.project_id
+  location       = var.gcp_region
+  cloud_function = google_cloudfunctions2_function.export_csv.name
+  role           = "roles/cloudfunctions.invoker"
+  member         = "serviceAccount:${google_service_account.functions_service_account.email}"
 }

infra/functions-python/vars.tf

@@ -67,7 +67,7 @@ variable "gbfs_scheduler_schedule" {
 
 variable "transitland_scraping_schedule" {
     type        = string
-    description = "Schedule for the GBFS scheduler job"
+    description = "Schedule Transitland scraping job"
     default     = "0 0 3 * *" # every month on the 3rd day at 00:00
 }
 
@@ -79,4 +79,10 @@ variable "transitland_api_key" {
 variable "operations_oauth2_client_id" {
   type = string
   description = "value of the OAuth2 client id for the Operations API"
+}
+
+variable "export_csv_schedule" {
+    type        = string
+    description = "Schedule the export_csv function"
+    default = "0 4 * * 2,5" # At 4am every Tuesday and Friday.
 }