airflow-scheduled-job: adapt to listener changes

demos/airflow-scheduled-job/03-enable-and-run-spark-dag.yaml

@@ -13,14 +13,18 @@ spec:
           # restarted. Additionally, the db-init job takes a few minutes to complete before the cluster is deployed. The wait/watch steps
           # below are not "water-tight" but add a layer of stability by at least ensuring that the db is initialized and ready and that
           # all pods are reachable (albeit independent of each other).
-          command: ["bash", "-c", "
-            kubectl rollout status --watch statefulset/airflow-webserver-default
-            && kubectl rollout status --watch statefulset/airflow-scheduler-default
-            && export AIRFLOW_ADMIN_PASSWORD=$(cat /airflow-credentials/adminUser.password)
-            && export ACCESS_TOKEN=$(curl -XPOST http://airflow-webserver-default:8080/auth/token -H 'Content-Type: application/json' -d '{\"username\": \"admin\", \"password\": \"'$AIRFLOW_ADMIN_PASSWORD'\"}' | jq '.access_token' | tr -d '\"')
-            && curl -H \"Authorization: Bearer $ACCESS_TOKEN\" -H 'Content-Type: application/json' -XPATCH http://airflow-webserver-default:8080/api/v2/dags/sparkapp_dag -d '{\"is_paused\": false}' | jq
-            && curl -H \"Authorization: Bearer $ACCESS_TOKEN\" -H 'Content-Type: application/json' -XPOST http://airflow-webserver-default:8080/api/v2/dags/sparkapp_dag/dagRuns -d '{\"logical_date\": null}' | jq
-          "]
+          command: [
+              "bash",
+              "-c",
+              '
+              kubectl rollout status --watch statefulset/airflow-webserver-default
+              && kubectl rollout status --watch statefulset/airflow-scheduler-default
+              && export AIRFLOW_ADMIN_PASSWORD=$(cat /airflow-credentials/adminUser.password)
+              && export ACCESS_TOKEN=$(curl -XPOST http://airflow-webserver-default-headless:8080/auth/token -H ''Content-Type: application/json'' -d ''{"username": "admin", "password": "''$AIRFLOW_ADMIN_PASSWORD''"}'' | jq ''.access_token'' | tr -d ''"'')
+              && curl -H "Authorization: Bearer $ACCESS_TOKEN" -H ''Content-Type: application/json'' -XPATCH http://airflow-webserver-default-headless:8080/api/v2/dags/sparkapp_dag -d ''{"is_paused": false}'' | jq
+              && curl -H "Authorization: Bearer $ACCESS_TOKEN" -H ''Content-Type: application/json'' -XPOST http://airflow-webserver-default-headless:8080/api/v2/dags/sparkapp_dag/dagRuns -d ''{"logical_date": null}'' | jq
+              ',
+            ]
           volumeMounts:
             - name: airflow-credentials
               mountPath: /airflow-credentials

demos/airflow-scheduled-job/04-enable-and-run-date-dag.yaml

@@ -13,14 +13,18 @@ spec:
           # restarted. Additionally, the db-init job takes a few minutes to complete before the cluster is deployed. The wait/watch steps
           # below are not "water-tight" but add a layer of stability by at least ensuring that the db is initialized and ready and that
           # all pods are reachable (albeit independent of each other).
-          command: ["bash", "-c", "
-            kubectl rollout status --watch statefulset/airflow-webserver-default
-            && kubectl rollout status --watch statefulset/airflow-scheduler-default
-            && export AIRFLOW_ADMIN_PASSWORD=$(cat /airflow-credentials/adminUser.password)
-            && export ACCESS_TOKEN=$(curl -XPOST http://airflow-webserver-default:8080/auth/token -H 'Content-Type: application/json' -d '{\"username\": \"admin\", \"password\": \"'$AIRFLOW_ADMIN_PASSWORD'\"}' | jq '.access_token' | tr -d '\"')
-            && curl -H \"Authorization: Bearer $ACCESS_TOKEN\" -H 'Content-Type: application/json' -XPATCH http://airflow-webserver-default:8080/api/v2/dags/date_demo -d '{\"is_paused\": false}' | jq
-            && curl -H \"Authorization: Bearer $ACCESS_TOKEN\" -H 'Content-Type: application/json' -XPOST http://airflow-webserver-default:8080/api/v2/dags/date_demo/dagRuns -d '{\"logical_date\": null}' | jq
-          "]
+          command: [
+              "bash",
+              "-c",
+              '
+              kubectl rollout status --watch statefulset/airflow-webserver-default
+              && kubectl rollout status --watch statefulset/airflow-scheduler-default
+              && export AIRFLOW_ADMIN_PASSWORD=$(cat /airflow-credentials/adminUser.password)
+              && export ACCESS_TOKEN=$(curl -XPOST http://airflow-webserver-default-headless:8080/auth/token -H ''Content-Type: application/json'' -d ''{"username": "admin", "password": "''$AIRFLOW_ADMIN_PASSWORD''"}'' | jq ''.access_token'' | tr -d ''"'')
+              && curl -H "Authorization: Bearer $ACCESS_TOKEN" -H ''Content-Type: application/json'' -XPATCH http://airflow-webserver-default-headless:8080/api/v2/dags/date_demo -d ''{"is_paused": false}'' | jq
+              && curl -H "Authorization: Bearer $ACCESS_TOKEN" -H ''Content-Type: application/json'' -XPOST http://airflow-webserver-default-headless:8080/api/v2/dags/date_demo/dagRuns -d ''{"logical_date": null}'' | jq
+              ',
+            ]
           volumeMounts:
             - name: airflow-credentials
               mountPath: /airflow-credentials

stacks/airflow/airflow.yaml

@@ -26,6 +26,8 @@ spec:
         mountPath: /dags/pyspark_pi.yaml
         subPath: pyspark_pi.yaml
   webservers:
+    roleConfig:
+      listenerClass: external-unstable
     config:
       resources:
         cpu:
@@ -34,7 +36,6 @@ spec:
         memory:
           limit: 2Gi
       gracefulShutdownTimeout: 30s
-      listenerClass: external-unstable
     roleGroups:
       default:
         envOverrides:
@@ -302,6 +303,8 @@ data:
             memory:
               limit: 1024Mi
         replicas: 3
+
+
 # {% endraw %}
 ---
 apiVersion: v1