fix(satellite): use latest version of satellite (read-only db)

Author: luabida

alertflow/airflow.cfg

@@ -585,7 +585,7 @@ reload_on_plugin_change = False
 # The token generated using the secret key has a short expiry time though - make sure that time on
 # ALL the machines that you run airflow components on is synchronized (for example using ntpd)
 # otherwise you might get "forbidden" errors when the logs are accessed.
-# secret_key =
+secret_key = $AIRFLOW__WEBSERVER__SECRET_KEY
 
 # Number of workers to run the Gunicorn web server
 workers = 4

alertflow/dags/satellite-weather/brasil.py

@@ -45,7 +45,7 @@
     dag_id="COPERNICUS_BRASIL",
     description="ETL of weather data for Brazil",
     tags=["Brasil", "Copernicus"],
-    schedule="@monthly",
+    schedule="@daily",
     default_args=DEFAULT_ARGS,
     start_date=pendulum.datetime(2000, 1, 1),
     catchup=True,
@@ -83,12 +83,13 @@ def fetch_ds(locale, dt, uri, api_key):
             locale=locale,
         ) as ds:
             for geocode in geocodes:
-                adm = ADM2.get(code=geocode):
+                adm = ADM2.get(code=geocode, adm0=locale)
                 with engine.connect() as conn:
                     ds.cope.to_sql(adm, conn, tablename, "weather")
-            file = Path(f"{basename}.zip")
-            if file.exists():
-                file.unlink()
-                print(f"{file} removed")
+
+        file = Path(f"{basename}.zip")
+        if file.exists():
+            file.unlink()
+            print(f"{file} removed")
 
     fetch_ds("BRA", DATE, URI["PSQL_MAIN_URI"], KEY["CDSAPI_KEY"])

conda/env.yaml

@@ -7,3 +7,6 @@ dependencies:
   - pip
   - pre-commit
   - poetry >= 1.7.1
+  - docker-compose
+  - pip:
+      - git+https://github.com/luabida/satellite-weather-downloader@set-read-only

docker/compose.yaml

@@ -28,10 +28,12 @@ x-airflow-common:
   environment:
     &airflow-common-env
     AIRFLOW__CORE__FERNET_KEY: ${AIRFLOW__CORE__FERNET_KEY}
+    AIRFLOW__WEBSERVER__SECRET_KEY: ${AIRFLOW__WEBSERVER__SECRET_KEY}
     AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK: 'true'
     AIRFLOW_HOME: ${AIRFLOW_HOME:-/opt/airflow}
     AIRFLOW_VAR_PSQL_MAIN_URI: '{"PSQL_MAIN_URI":"${PSQL_URI_MAIN}"}'
     AIRFLOW_VAR_CDSAPI_KEY: '{"CDSAPI_KEY":"${CDSAPI_KEY}"}'
+
     HOST_UID: ${HOST_UID}
     HOST_GID: ${HOST_GID}
   volumes:
@@ -170,17 +172,17 @@ services:
       timeout: 10s
       retries: 5
       start_period: 30s
-    environment:
-      AIRFLOW__CORE__FERNET_KEY: ${AIRFLOW__CORE__FERNET_KEY}
-      AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK: 'true'
-      AIRFLOW_HOME: ${AIRFLOW_HOME:-/opt/airflow}
-      AIRFLOW_VAR_PSQL_MAIN_URI: '{"PSQL_MAIN_URI":"${PSQL_URI_MAIN}"}'
-      AIRFLOW_VAR_CDSAPI_KEY: '{"CDSAPI_KEY":"${CDSAPI_KEY}"}'
-      HOST_UID: ${HOST_UID}
-      HOST_GID: ${HOST_GID}
-      # Required to handle warm shutdown of the celery workers properly
-      # See https://airflow.apache.org/docs/docker-stack/entrypoint.html#signal-propagation
-      DUMB_INIT_SETSID: "0"
+    # environment:
+    #   AIRFLOW__CORE__FERNET_KEY: ${AIRFLOW__CORE__FERNET_KEY}
+    #   AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK: 'true'
+    #   AIRFLOW_HOME: ${AIRFLOW_HOME:-/opt/airflow}
+    #   AIRFLOW_VAR_PSQL_MAIN_URI: '{"PSQL_MAIN_URI":"${PSQL_URI_MAIN}"}'
+    #   AIRFLOW_VAR_CDSAPI_KEY: '{"CDSAPI_KEY":"${CDSAPI_KEY}"}'
+    #   HOST_UID: ${HOST_UID}
+    #   HOST_GID: ${HOST_GID}
+    #   # Required to handle warm shutdown of the celery workers properly
+    #   # See https://airflow.apache.org/docs/docker-stack/entrypoint.html#signal-propagation
+    #   DUMB_INIT_SETSID: "0"
     restart: always
     depends_on:
       redis: