Merge pull request #773 from bcgov/upgrade-airflow-version

Upgrade airflow version

Author: LiamF-FS

.github/workflows/.unit-tests.yml

@@ -60,12 +60,14 @@ jobs:
       run:
         working-directory: ./client
     steps:
-      - name: Checkout
+      - name: Checkoutup
         uses: actions/checkout@v4
       - name: Setup Node
         uses: actions/setup-node@v4
         with:
           node-version: '20.18.0'
+      - name: Install Chrome
+        uses: browser-actions/setup-chrome@v1
       - name: Cache NPM
         uses: actions/cache@v4
         with:
@@ -82,7 +84,7 @@ jobs:
         with:
           working-directory: ./client
           component: true
-          browser: electron
+          browser: chrome
         env:
             VITE_APP_MAPBOX_TOKEN: ${{ secrets.MAPBOX_TOKEN }}
             VITE_BASE_API_URL: http://localhost:8000

.github/workflows/dev-pr-run-unit-tests.yml

@@ -10,4 +10,4 @@ jobs:
   unit_tests:
     uses: ./.github/workflows/.unit-tests.yml
     secrets:
-      mapbox_token: ${{ secrets.MAPBOX_TOKEN }}
+      mapbox_token: ${{ secrets.CYPRESS_MAPBOX_TOKEN }}

.github/workflows/foundry.okd.on-push.application.yml

@@ -191,6 +191,8 @@ jobs:
             --set-string secrets.bcwatFlowworks.password="${{ secrets.FOUNDRY_BCWAT_FLOWWORKS_PASSWORD }}" \
             --set-string secrets.bcwatFlowworks.username="${{ secrets.FOUNDRY_BCWAT_FLOWWORKS_USERNAME }}" \
             --set-string secrets.fernet.key="${{ secrets.FOUNDRY_AIRFLOW_FERNET_KEY }}" \
+            --set-string secrets.api.secretKey="${{ secrets.FOUNDRY_AIRFLOW_API_SECRET_KEY_}}" \
+            --set-string secrets.api.jwtKey="${{ secrets.FOUNDRY_AIRFLOW_API_JWT_KEY_}}" \
             --set-string secrets.sendgridApiKey="${{ secrets.FOUNDRY_SENDGRID_API_KEY }}" \
             --set migrations.image=${{ secrets.ACR_REGISTRY }}/bcwat/flyway:${{ needs.bump_version.outputs.version }} \
             --set-string webserver.podAnnotations."rollout-timestamp"="$(date +%s)" \

airflow/Dockerfile

@@ -1,4 +1,4 @@
-FROM apache/airflow:2.10.5-python3.12
+FROM apache/airflow:3.1.7-python3.12
 
 USER root
 
@@ -14,7 +14,15 @@ ENV PYTHONPATH = "/opt/airflow/etl_pipelines:${PYTHONPATH}"
 
 COPY requirements.txt .
 
+# Switch to airflow user for pip installations
 USER airflow
 
+# PIN AIRFLOW VERSION when installing packages
+RUN pip install --no-cache-dir "apache-airflow==${AIRFLOW_VERSION}" \
+    --constraint ${HOME}/constraints.txt \
+    apache-airflow-providers-cncf-kubernetes \
+    apache-airflow-providers-sendgrid
+
+RUN pip install --no-cache-dir apache-airflow-providers-fab==3.1.2
+
 RUN pip install --no-cache-dir -r requirements.txt
-RUN pip install apache-airflow-providers-cncf-kubernetes apache-airflow-providers-sendgrid

airflow/Dockerfile_OKD

@@ -1,4 +1,4 @@
-FROM apache/airflow:2.10.5-python3.12
+FROM apache/airflow:3.1.7-python3.12
 
 USER root
 
@@ -10,12 +10,18 @@ COPY etl_pipelines/ /opt/airflow/etl_pipelines/
 
 RUN chown -R airflow: /opt/airflow/dags /opt/airflow/etl_pipelines /opt/airflow/shared
 
-
 ENV PYTHONPATH = "/opt/airflow/etl_pipelines:${PYTHONPATH}"
 
 COPY requirements.txt .
 
 USER airflow
 
+# PIN AIRFLOW VERSION when installing packages
+RUN pip install --no-cache-dir "apache-airflow==${AIRFLOW_VERSION}" \
+    --constraint ${HOME}/constraints.txt \
+    apache-airflow-providers-cncf-kubernetes \
+    apache-airflow-providers-sendgrid
+
+RUN pip install --no-cache-dir apache-airflow-providers-fab==3.1.2
+
 RUN pip install --no-cache-dir -r requirements.txt
-RUN pip install apache-airflow-providers-cncf-kubernetes apache-airflow-providers-sendgrid

airflow/README.md

@@ -29,7 +29,7 @@ In production, we are using the `KubernetesExecutor`. This does not impact runni
 ```bash
 import os
 from datetime import datetime
-from airflow.decorators import dag, task
+from airflow.sdk import dag, task
 from shared.constants import default_args
 from shared.functions import generate_executor_config_template
 from dotenv import load_dotenv, find_dotenv
@@ -41,7 +41,7 @@ ENVIRONMENT = os.getenv('ENVIRONMENT', 'no-env-found')
 # Does not prevent running locally
 # pod_template_file handles worker pod config
 @dag(
-    schedule_interval="@daily",
+    schedule="@daily",
     start_date=datetime(2024, 1, 1),
     catchup=False,
     tags=["example"],

airflow/dags/asp_dag.py

@@ -1,6 +1,6 @@
 import os
-import pendulum
-from airflow.decorators import dag, task
+from datetime import datetime
+from airflow.sdk import dag, task
 from shared.constants import default_args
 from shared.functions import generate_executor_config_template
 from dotenv import load_dotenv, find_dotenv
@@ -10,8 +10,8 @@
 
 @dag(
     dag_id="asp_dag",
-    schedule_interval="5 8 * * *",
-    start_date=pendulum.datetime(2025, 5, 7, tz="UTC"),
+    schedule="5 8 * * *",
+    start_date=datetime(2025, 5, 7),
     catchup=False,
     tags=["water","climate", "station_observations", "daily"],
     default_args=default_args

airflow/dags/ec_xml_dag.py

@@ -1,6 +1,6 @@
 import os
-import pendulum
-from airflow.decorators import dag, task
+from datetime import datetime
+from airflow.sdk import dag, task
 from shared.constants import default_args
 from shared.functions import generate_executor_config_template
 from dotenv import load_dotenv, find_dotenv
@@ -10,8 +10,8 @@
 
 @dag(
     dag_id="ec_xml_dag",
-    schedule_interval="0 8 * * *",
-    start_date=pendulum.datetime(2025, 5, 7, tz="UTC"),
+    schedule="0 8 * * *",
+    start_date=datetime(2025, 5, 7),
     catchup=False,
     tags=["climate", "station_observations", "daily"],
     default_args=default_args

airflow/dags/env_aqn_dag.py

@@ -1,6 +1,6 @@
 import os
-import pendulum
-from airflow.decorators import dag, task
+from datetime import datetime
+from airflow.sdk import dag, task
 from shared.constants import default_args
 from shared.functions import generate_executor_config_template
 from dotenv import load_dotenv, find_dotenv
@@ -10,8 +10,8 @@
 
 @dag(
     dag_id="env_aqn_dag",
-    schedule_interval="0 8 * * *",
-    start_date=pendulum.datetime(2025, 5, 15, tz="UTC"),
+    schedule="0 8 * * *",
+    start_date=datetime(2025, 5, 15),
     catchup=False,
     tags=["climate", "station_observations", "daily"],
     default_args=default_args

airflow/dags/env_hydro_dag.py

@@ -1,6 +1,6 @@
 import os
-import pendulum
-from airflow.decorators import dag, task
+from datetime import datetime
+from airflow.sdk import dag, task
 from shared.constants import default_args
 from shared.functions import generate_executor_config_template
 from dotenv import load_dotenv, find_dotenv
@@ -10,8 +10,8 @@
 
 @dag(
     dag_id="env_hydro_dag",
-    schedule_interval="10 8 * * *",
-    start_date=pendulum.datetime(2025, 4, 17, tz="UTC"),
+    schedule="10 8 * * *",
+    start_date=datetime(2025, 4, 17),
     catchup=False,
     tags=["water", "station_observations", "daily"],
     default_args=default_args