fix: revert dag launcher role changes

.github/workflows/pr.yml

@@ -111,10 +111,10 @@ jobs:
           ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           COMMENT_ID: ${{ steps.init-comment.outputs.COMMENT_ID }}
-          KEYCLOAK_STAGING_SM2A_FAB_CLIENT_SECRET: ${{ secrets.KEYCLOAK_STAGING_SM2A_FAB_CLIENT_SECRET }}
+          STAGING_SM2A_ADMIN_USERNAME: ${{ secrets.STAGING_SM2A_ADMIN_USERNAME }}
+          STAGING_SM2A_ADMIN_PASSWORD: ${{ secrets.STAGING_SM2A_ADMIN_PASSWORD }}
           STAGING_SM2A_API_URL: ${{ vars.STAGING_SM2A_API_URL }}
           DATASET_DAG_NAME: ${{ vars.DATASET_DAG_NAME }}
-          KEYCLOAK_STAGING_URL: ${{ vars.KEYCLOAK_STAGING_URL }}
         run: |
           pip install -r ./scripts/requirements.txt
 

.github/workflows/promote.yml

@@ -61,10 +61,10 @@ jobs:
 
       - name: Publish to production on PR merge
         env:
+          SM2A_ADMIN_USERNAME: ${{ secrets.SM2A_ADMIN_USERNAME }}
+          SM2A_ADMIN_PASSWORD: ${{ secrets.SM2A_ADMIN_PASSWORD }}
           SM2A_API_URL: ${{ vars.SM2A_API_URL }}
           PROMOTION_DAG: ${{ vars.PROMOTION_DAG_NAME }}
-          KEYCLOAK_PROD_URL: ${{ vars.KEYCLOAK_PROD_URL }}
-          KEYCLOAK_PROD_SM2A_FAB_CLIENT_SECRET: ${{ secrets.KEYCLOAK_PROD_SM2A_FAB_CLIENT_SECRET }}
 
         run: |
           pip install -r ./scripts/requirements.txt

scripts/promote_collection.py

@@ -5,7 +5,7 @@
 import sys
 import os
 import uuid
-import requests
+from base64 import b64encode
 
 
 def trigger_collection_dag(payload: Dict[str, Any], stage: str):
@@ -16,41 +16,34 @@ def trigger_collection_dag(payload: Dict[str, Any], stage: str):
 
     if stage == "staging":
         api_url_env = "STAGING_SM2A_API_URL"
-        token_url = f"https://{os.getenv('KEYCLOAK_STAGING_URL')}/realms/veda/protocol/openid-connect/token"
-        client_id = "airflow-webserver-fab"
-        client_secret = os.getenv("KEYCLOAK_STAGING_SM2A_FAB_CLIENT_SECRET")
+        username_env = "STAGING_SM2A_ADMIN_USERNAME"
+        password_env = "STAGING_SM2A_ADMIN_PASSWORD"
     elif stage == "production":
         api_url_env = "SM2A_API_URL"
-        token_url = f"https://{os.getenv('KEYCLOAK_PROD_URL')}/realms/veda/protocol/openid-connect/token"
-        client_id = "airflow-webserver-fab"
-        client_secret = os.getenv("KEYCLOAK_PROD_SM2A_FAB_CLIENT_SECRET")
+        username_env = "SM2A_ADMIN_USERNAME"
+        password_env = "SM2A_ADMIN_PASSWORD"
     else:
         raise ValueError(
             f"Invalid stage provided: {stage}. Must be 'staging' or 'production'."
         )
 
     base_api_url = os.getenv(api_url_env)
+    username = os.getenv(username_env)
+    password = os.getenv(password_env)
 
-    response = requests.post(
-        token_url,
-        data={
-            "client_id": client_id,
-            "client_secret": client_secret,
-            "grant_type": "client_credentials",
-        },
-    )
-    access_token = response.json()["access_token"]
-
-    if not all([base_api_url, access_token]):
+    if not all([base_api_url, username, password]):
         raise ValueError(
             f"Missing one or more environment variables: "
             f"stage is None={stage is None}, "
-            f"access_token is None={access_token is None}"
+            f"username is None={username_env is None}, "
+            f"password is None={password_env is None}"
         )
 
+    api_token = b64encode(f"{username}:{password}".encode()).decode()
+
     headers = {
         "Content-Type": "application/json",
-        "Authorization": "Bearer " + access_token,
+        "Authorization": "Basic " + api_token,
     }
 
     body = {

scripts/promote_dataset.py

@@ -5,7 +5,7 @@
 import sys
 import os
 import uuid
-import requests
+from base64 import b64encode
 
 
 class MissingFieldError(Exception):
@@ -35,30 +35,21 @@ def validate_discovery_item_config(item: Dict[str, Any]) -> Dict[str, Any]:
 def publish_to_staging(payload):
     base_api_url = os.getenv("STAGING_SM2A_API_URL")
     dataset_pipeline_dag = os.getenv("DATASET_DAG_NAME", "veda_dataset_pipeline")
+    username = os.getenv("STAGING_SM2A_ADMIN_USERNAME")
+    password = os.getenv("STAGING_SM2A_ADMIN_PASSWORD")
 
-    token_url = f"https://{os.getenv('KEYCLOAK_STAGING_URL')}/realms/veda/protocol/openid-connect/token"
-    client_id = "airflow-webserver-fab"
-    client_secret = os.getenv("KEYCLOAK_STAGING_SM2A_FAB_CLIENT_SECRET")
-
-    response = requests.post(
-        token_url,
-        data={
-            "client_id": client_id,
-            "client_secret": client_secret,
-            "grant_type": "client_credentials",
-        },
-    )
-    access_token = response.json()["access_token"]
+    api_token = b64encode(f"{username}:{password}".encode()).decode()
 
-    if not base_api_url or not access_token:
+    if not base_api_url or not api_token:
         raise ValueError(
-            "STAGING_SM2A_API_URL or KEYCLOAK_STAGING_SM2A_FAB_CLIENT_SECRET is not"
+            "STAGING_SM2A_API_URL or STAGING_SM2A_ADMIN_USERNAME"
+            + " or STAGING_SM2A_ADMIN_PASSWORD is not"
             + " set in the environment variables."
         )
 
     headers = {
         "Content-Type": "application/json",
-        "Authorization": "Bearer " + access_token,
+        "Authorization": "Basic " + api_token,
     }
 
     body = {
@@ -85,30 +76,20 @@ def publish_to_staging(payload):
 def promote_to_production(payload):
     base_api_url = os.getenv("SM2A_API_URL")
     promotion_dag = os.getenv("PROMOTION_DAG_NAME", "veda_promotion_pipeline")
+    username = os.getenv("SM2A_ADMIN_USERNAME")
+    password = os.getenv("SM2A_ADMIN_PASSWORD")
 
-    token_url = f"https://{os.getenv('KEYCLOAK_PROD_URL')}/realms/veda/protocol/openid-connect/token"
-    client_id = "airflow-webserver-fab"
-    client_secret = os.getenv("KEYCLOAK_PROD_SM2A_FAB_CLIENT_SECRET")
-
-    response = requests.post(
-        token_url,
-        data={
-            "client_id": client_id,
-            "client_secret": client_secret,
-            "grant_type": "client_credentials",
-        },
-    )
-    access_token = response.json()["access_token"]
+    api_token = b64encode(f"{username}:{password}".encode()).decode()
 
-    if not base_api_url or not access_token:
+    if not base_api_url or not api_token:
         raise ValueError(
-            "SM2A_API_URL or KEYCLOAK_PRODUCTION_SM2A_FAB_CLIENT_SECRET is not"
+            "SM2A_API_URL or SM2A_ADMIN_USERNAME or SM2A_ADMIN_PASSWORD is not"
             + " set in the environment variables."
         )
 
     headers = {
         "Content-Type": "application/json",
-        "Authorization": "Bearer " + access_token,
+        "Authorization": "Basic " + api_token,
     }
 
     payload["conf"]["transfer"] = True

scripts/requirements.txt

@@ -1,2 +1 @@
-pyyaml
-requests
+pyyaml