UID2-5092 AKS E2E test (#157)

cYKatherine · web-flow · commit dfde20c83b29 · 2025-03-13T16:41:36.000+11:00
* Add steps to prepare for azure metadata

* Rename azure to aks

* Add steps to start AKS private operator

* Always reply yes with the prompt

* Pass in AKS operator key

* Comment out setting up AKS and stuff

* Fix yq issue

* Add debug message

* Hardcode MANAGED_IDENTITY_ID

* Use python file to add env var

* Update other env names

* Add commands to setup proper kube config

* Replace before starting

* Update ROOT path

* Remove unnecessary comments

* Remove aks_operator_key

* Move bore urls

* Wait for deployment to be ready instead of svc

* Remove wait command

* Update port

* Add cleanup step for AKS operator

* Remove unnecessary outputs

* Update azure-aks path

* Cleanup aks cluster only when input is aks

* Make python command one line

* Always cleanup private operator

* Throw when error adding env

* Update comments

* Add meaningful comments

* Remove unnecessary env variables

* Replace kcc-UID2-5092-aks-e2e with v3
diff --git a/.github/workflows/shared-run-e2e-tests.yaml b/.github/workflows/shared-run-e2e-tests.yaml
@@ -54,7 +54,7 @@ on:
         default: main
 
       operator_type:
-        description: The operator type [public, gcp, azure, aws]
+        description: The operator type [public, gcp, azure, aws, aks]
         type: string
         default: public
       uid2_e2e_identity_scope:
@@ -169,7 +169,7 @@ jobs:
 
       - name: Bring up bore
         id: bore
-        if: ${{ inputs.operator_type == 'gcp' || inputs.operator_type == 'azure' || inputs.operator_type == 'aws' || inputs.operator_type == 'eks'}}
+        if: ${{ inputs.operator_type == 'gcp' || inputs.operator_type == 'azure' || inputs.operator_type == 'aws' || inputs.operator_type == 'eks' || inputs.operator_type == 'aks'}}
         env:
           BORE_URL: ${{ secrets.BORE_URL }}
           BORE_SECRET: ${{ secrets.BORE_SECRET }}
@@ -211,6 +211,16 @@ jobs:
           admin_root: ${{ inputs.admin_root }}
           eks_pcr0: ${{ inputs.eks_pcr0 }}
 
+      - name: Prepare AKS metadata
+        id: prepare_aks_metadata
+        if: ${{ inputs.operator_type == 'aks' }}
+        uses: IABTechLab/uid2-shared-actions/actions/prepare_aks_metadata@v3
+        with:
+          operator_image_version: ${{ inputs.operator_image_version }}
+          admin_root: ${{ inputs.admin_root }}
+          bore_url_core: ${{ steps.bore.outputs.BORE_URL_CORE }}
+          bore_url_optout: ${{ steps.bore.outputs.BORE_URL_OPTOUT }}
+
       - name: Bring up Docker Compose
         id: docker-compose
         env:
@@ -281,6 +291,14 @@ jobs:
           eks_test_cluster: ${{ inputs.eks_test_cluster }}
           eks_test_cluster_region: ${{ inputs.eks_test_cluster_region }}
 
+      - name: Start AKS private operator
+        id: start_aks_private_operator
+        if: ${{ inputs.operator_type == 'aks' }}
+        uses: IABTechLab/uid2-shared-actions/actions/start_aks_private_operator@v3
+        with:
+          output_template_file: ${{ steps.prepare_aks_metadata.outputs.output_template_file }}
+          azure_credentials: ${{ secrets.AZURE_CREDENTIALS }}
+
       - name: Decide E2E test environment variables
         id: decide_env_var
         shell: bash
@@ -294,6 +312,7 @@ jobs:
           AZURE_OPERATOR_URL: ${{ steps.start_azure_private_operator.outputs.uid2_e2e_pipeline_operator_url }}
           AWS_OPERATOR_URL: ${{ steps.start_aws_private_operator.outputs.uid2_e2e_pipeline_operator_url }}
           EKS_OPERATOR_URL: ${{ steps.start_EKS_operator.outputs.uid2_e2e_pipeline_operator_url }}
+          AKS_OPERATOR_URL: ${{ steps.start_aks_private_operator.outputs.uid2_e2e_pipeline_operator_url }}
         run: |
           bash uid2-shared-actions/scripts/decide_e2e_env.sh
 
@@ -312,28 +331,34 @@ jobs:
           uid2_e2e_pipeline_optout_url: ${{ steps.decide_env_var.outputs.uid2_e2e_pipeline_optout_url }}
 
       - name: Stop GCP private operator
-        if: ${{ inputs.operator_type == 'gcp' }}
+        if: always()
         env:
           GCP_PROJECT: ${{ inputs.gcp_project }}
           SERVICE_ACCOUNT: ${{ inputs.gcp_service_account }}
           GCP_INSTANCE_NAME: ${{ steps.start_gcp_private_operator.outputs.gcp_instance_name }}
         run: |
-          bash uid2-shared-actions/scripts/gcp/stop_gcp_enclave.sh
+          if [[ ${{ inputs.operator_type }} == 'gcp' ]]; then
+            bash uid2-shared-actions/scripts/gcp/stop_gcp_enclave.sh
+          fi
 
       - name: Stop Azure private operator
-        if: ${{ inputs.operator_type == 'azure' }}
+        if: always()
         env:
           AZURE_CONTAINER_GROUP_NAME: ${{ steps.start_azure_private_operator.outputs.azure_container_group_name }}
         run: |
-          bash uid2-shared-actions/scripts/azure/stop_azure_enclave.sh
+          if [[ ${{ inputs.operator_type }} == 'azure' ]]; then
+            bash uid2-shared-actions/scripts/azure/stop_azure_enclave.sh
+          fi
 
       - name: Stop AWS private operator
-        if: ${{ inputs.operator_type == 'aws' }}
+        if: always()
         env:
           AWS_STACK_NAME: ${{ steps.start_aws_private_operator.outputs.aws_stack_name }}
           AWS_REGION: ${{ inputs.aws_region }}
         run: |
-          bash uid2-shared-actions/scripts/aws/stop_aws_enclave.sh
+          if [[ ${{ inputs.operator_type }} == 'aws' ]]; then
+            bash uid2-shared-actions/scripts/aws/stop_aws_enclave.sh
+          fi
 
       - name: Stop EKS operator
         if: ${{ inputs.operator_type == 'eks' }}
@@ -342,3 +367,10 @@ jobs:
           eks_test_cluster: ${{ inputs.eks_test_cluster }}
           eks_test_cluster_region: ${{ inputs.eks_test_cluster_region }}
           identity_scope: ${{ inputs.uid2_e2e_identity_scope }}
+
+      - name: Stop AKS operator
+        if: always() 
+        run: |
+          if [[ ${{ inputs.operator_type }} == 'aks' ]]; then
+            bash uid2-shared-actions/scripts/aks/stop_aks_enclave.sh
+          fi
diff --git a/actions/prepare_aks_metadata/action.yaml b/actions/prepare_aks_metadata/action.yaml
@@ -0,0 +1,61 @@
+name: Prepare AKS Metadata
+description: Prepares the AKS CC artifacts and enclave metadata
+
+inputs:
+  operator_image_version:
+    description: The uid2-operator image version
+    default: latest
+  admin_root:
+    description: The root path for uid2-admin folder
+    default: uid2-admin
+  operator_root:
+    description: The root path for uid2-operator folder
+    default: uid2-operator
+  bore_url_core:
+    description: The bore URL for core service
+    required: true
+  bore_url_optout:
+    description: The bore URL for optout service
+    required: true
+outputs:
+  output_template_file:
+    description: The output template file
+    value: ${{ steps.enclave_artifacts.outputs.OUTPUT_TEMPLATE_FILE }}
+
+runs:
+  using: "composite"
+
+  steps:
+    - name: check azure-cli version
+      shell: bash
+      run: |
+        az --version
+    
+    - name: Generate AKS enclave deployment artifacts
+      id: enclave_artifacts
+      shell: bash
+      env:
+        IMAGE_VERSION: ${{ inputs.operator_image_version }}
+        OPERATOR_ROOT: ${{ inputs.operator_root }}
+        BORE_URL_CORE: ${{ inputs.bore_url_core }}
+        BORE_URL_OPTOUT: ${{ inputs.bore_url_optout }}
+      run: |
+        bash uid2-shared-actions/scripts/aks/prepare_aks_artifacts.sh
+
+    - name: Prepare AKS enclave ID
+      id: enclave_id
+      shell: bash
+      env:
+        OUTPUT_POLICY_DIGEST_FILE: ${{ steps.enclave_artifacts.outputs.OUTPUT_POLICY_DIGEST_FILE }}
+      run: |
+        bash uid2-shared-actions/scripts/aks/prepare_aks_enclave_id.sh
+
+    - name: Prepare AKS enclave metadata
+      id: enclave_metadata
+      shell: bash
+      env:
+        ADMIN_ROOT: ${{ inputs.admin_root }}
+        ENCLAVE_ID: ${{ steps.enclave_id.outputs.ENCLAVE_ID }}
+        ENCLAVE_PROTOCOL: azure-cc
+      run: |
+        bash uid2-shared-actions/scripts/save_enclave_id_to_admin.sh
diff --git a/actions/start_aks_private_operator/action.yaml b/actions/start_aks_private_operator/action.yaml
@@ -0,0 +1,33 @@
+name: Start AKS Private Operator
+description: Spins up an AKS private operator
+
+inputs:
+  output_template_file:
+    description: The output template file
+    required: true
+  azure_credentials:
+    description: The Azure credentials
+    required: true
+
+outputs:
+  uid2_e2e_pipeline_operator_url:
+    description: The AKS operator URL
+    value: ${{ steps.start_aks.outputs.uid2_e2e_pipeline_operator_url }}
+
+runs:
+  using: "composite"
+
+  steps:
+    - name: Log in to Azure
+      uses: azure/login@v2
+      with:
+        creds: ${{ inputs.azure_credentials }}
+        enable-AzPSSession: true
+
+    - name: Start AKS private operator
+      id: start_aks
+      shell: bash
+      env:
+        OUTPUT_TEMPLATE_FILE: ${{ inputs.output_template_file }}
+      run: |
+        bash uid2-shared-actions/scripts/aks/start_aks_enclave.sh
diff --git a/scripts/aks/add_env.py b/scripts/aks/add_env.py
@@ -0,0 +1,58 @@
+import yaml
+import argparse
+
+def add_env_variables(yaml_file, container_name, env_vars):
+    """
+    Adds multiple environment variables to a specified container in a Kubernetes YAML file.
+    Args:
+        yaml_file (str): Path to the YAML file.
+        container_name (str): Name of the container to modify.
+        env_vars (list): List of dictionaries, each containing 'name' and 'value' of an env var.
+    """
+    try:
+        with open(yaml_file, 'r') as file:
+            data = list(yaml.safe_load_all(file))
+
+        modified = False
+
+        for doc in data:
+            if doc and doc.get('kind') == 'Deployment':
+                containers = doc['spec']['template']['spec']['containers']
+                for container in containers:
+                    if container['name'] == container_name:
+                        if 'env' not in container:
+                            container['env'] = []
+                        for env_var in env_vars:
+                            container['env'].append(env_var)
+                        modified = True
+                        break
+
+        if modified:
+            with open(yaml_file, 'w') as file:
+                yaml.dump_all(data, file, default_flow_style=False)
+            print(f"Environment variables added to container '{container_name}' in '{yaml_file}'.")
+        else:
+            print(f"Container '{container_name}' not found in any Deployment document.")
+
+    except FileNotFoundError:
+        raise FileNotFoundError(f"Error: File '{yaml_file}' not found.")
+    except Exception as e:
+        raise RuntimeError(f"An error occurred: {e}") from e #Reraise with context.
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Add multiple environment variables to a Kubernetes YAML file.")
+    parser.add_argument("yaml_file", help="Path to the YAML file.")
+    parser.add_argument("container_name", help="Name of the container.")
+    parser.add_argument("env_vars", nargs="+", help="Environment variables in 'name value' pairs.")
+
+    args = parser.parse_args()
+
+    if len(args.env_vars) % 2 != 0:
+        print("Error: Environment variables must be provided in 'name value' pairs.")
+        exit(1)
+
+    env_vars = []
+    for i in range(0, len(args.env_vars), 2):
+        env_vars.append({'name': args.env_vars[i], 'value': args.env_vars[i + 1]})
+
+    add_env_variables(args.yaml_file, args.container_name, env_vars)
diff --git a/scripts/aks/prepare_aks_artifacts.sh b/scripts/aks/prepare_aks_artifacts.sh
@@ -0,0 +1,104 @@
+#!/usr/bin/env bash
+set -ex
+
+# Below resources should be prepared ahead of running the E2E test.
+# See https://github.com/UnifiedID2/aks-demo/tree/master/vn-aks#setup-aks--node-pool
+export RESOURCE_GROUP="pipeline-vn-aks"
+export LOCATION="eastus"
+export VNET_NAME="pipeline-vnet"
+export PUBLIC_IP_ADDRESS_NAME="pipeline-public-ip"
+export NAT_GATEWAY_NAME="pipeline-nat-gateway"
+export AKS_CLUSTER_NAME="pipelinevncluster"
+export KEYVAULT_NAME="pipeline-vn-aks-vault"
+export KEYVAULT_SECRET_NAME="pipeline-vn-aks-opr-key-name"
+export MANAGED_IDENTITY="pipeline-vn-aks-opr-id"
+export AKS_NODE_RESOURCE_GROUP="MC_${RESOURCE_GROUP}_${AKS_CLUSTER_NAME}_${LOCATION}"
+export SUBSCRIPTION_ID="$(az account show --query id --output tsv)"
+export DEPLOYMENT_ENV="integ"
+export MANAGED_IDENTITY_ID="/subscriptions/001a3882-eb1c-42ac-9edc-5e2872a07783/resourcegroups/pipeline-vn-aks/providers/Microsoft.ManagedIdentity/userAssignedIdentities/pipeline-vn-aks-opr-id"
+
+if [ -z "${IMAGE_VERSION}" ]; then
+  echo "IMAGE_VERSION can not be empty"
+  exit 1
+fi
+
+if [ -z "${OPERATOR_ROOT}" ]; then
+  echo "OPERATOR_ROOT can not be empty"
+  exit 1
+fi
+
+if [ -z "${BORE_URL_CORE}" ]; then
+  echo "BORE_URL_CORE can not be empty"
+  exit 1
+fi
+
+if [ -z "${BORE_URL_OPTOUT}" ]; then
+  echo "BORE_URL_OPTOUT can not be empty"
+  exit 1
+fi
+
+ROOT="uid2-shared-actions/scripts/aks"
+OUTPUT_DIR="${ROOT}/azure-aks-artifacts"
+
+IMAGE="ghcr.io/iabtechlab/uid2-operator:${IMAGE_VERSION}"
+
+if [ -d "${OUTPUT_DIR}" ]; then
+  echo "${OUTPUT_DIR} exists"
+fi
+
+INPUT_TEMPLATE_FILE="${OPERATOR_ROOT}/scripts/azure-aks/deployment/operator.yaml"
+OUTPUT_TEMPLATE_FILE="${OUTPUT_DIR}/operator.yaml"
+OUTPUT_POLICY_DIGEST_FILE="${OUTPUT_DIR}/aks-digest.txt"
+
+if [[ -d ${OUTPUT_DIR} ]]; then
+  echo "${OUTPUT_DIR} exists, skipping - this only happens during local testing"
+else
+  mkdir -p ${OUTPUT_DIR}
+
+  # Install confcom extension, az is originally available in GitHub workflow environment
+  az extension add --name confcom
+  if [[ $? -ne 0 ]]; then
+    echo "Failed to install Azure confcom extension"
+    exit 1
+  fi
+
+  # Required by az confcom
+  sudo usermod -aG docker ${USER}
+  if [[ $? -ne 0 ]]; then
+    echo "Failed to add current user to Docker group"
+    exit 1
+  fi
+
+  # Generate deployment template
+  cp ${INPUT_TEMPLATE_FILE} ${OUTPUT_TEMPLATE_FILE}
+  sed -i "s#IMAGE_PLACEHOLDER#${IMAGE}#g" ${OUTPUT_TEMPLATE_FILE}
+  sed -i "s#IDENTITY_PLACEHOLDER#$MANAGED_IDENTITY_ID#g" "${OUTPUT_TEMPLATE_FILE}"
+  sed -i "s#VAULT_NAME_PLACEHOLDER#$KEYVAULT_NAME#g" "${OUTPUT_TEMPLATE_FILE}"
+  sed -i "s#OPERATOR_KEY_SECRET_NAME_PLACEHOLDER#$KEYVAULT_SECRET_NAME#g" "${OUTPUT_TEMPLATE_FILE}"
+  sed -i "s#DEPLOYMENT_ENVIRONMENT_PLACEHOLDER#integ#g" "${OUTPUT_TEMPLATE_FILE}"
+  cat ${OUTPUT_TEMPLATE_FILE}
+
+  python3 ${ROOT}/add_env.py ${OUTPUT_TEMPLATE_FILE} uid2-operator CORE_BASE_URL http://$BORE_URL_CORE OPTOUT_BASE_URL http://$BORE_URL_OPTOUT SKIP_VALIDATIONS true
+  cat ${OUTPUT_TEMPLATE_FILE}
+  # --- Finished updating yaml file with resources ---
+  if [[ $? -ne 0 ]]; then
+    echo "Failed to pre-process template file"
+    exit 1
+  fi
+
+  # Generate policy using debug mode as we will need to override environment variables
+  yes | az confcom acipolicygen --virtual-node-yaml ${OUTPUT_TEMPLATE_FILE} --debug-mode > ${OUTPUT_POLICY_DIGEST_FILE}
+  if [[ $? -ne 0 ]]; then
+    echo "Failed to generate template file"
+    exit 1
+  fi
+  # The previous pipe will be stored in ${OUTPUT_POLICY_DIGEST_FILE} as well. The below command is to remove the prompt and only extract the enclave id.
+  sed -i 's/.*(y\/n) //g' "${OUTPUT_POLICY_DIGEST_FILE}"
+fi
+
+if [ -z "${GITHUB_OUTPUT}" ]; then
+  echo "Not in GitHub action"
+else
+  echo "OUTPUT_TEMPLATE_FILE=${OUTPUT_TEMPLATE_FILE}" >> ${GITHUB_OUTPUT}
+  echo "OUTPUT_POLICY_DIGEST_FILE=${OUTPUT_POLICY_DIGEST_FILE}" >> ${GITHUB_OUTPUT}
+fi
diff --git a/scripts/aks/prepare_aks_enclave_id.sh b/scripts/aks/prepare_aks_enclave_id.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -ex
+
+if [[ ! -f ${OUTPUT_POLICY_DIGEST_FILE} ]]; then
+  echo "OUTPUT_POLICY_DIGEST_FILE does not exist"
+  exit 1
+fi
+
+AZURE_POLICY_DIGEST="$(cat ${OUTPUT_POLICY_DIGEST_FILE})"
+echo "AZURE_POLICY_DIGEST=${AZURE_POLICY_DIGEST}"
+
+ENCLAVE_ID=${AZURE_POLICY_DIGEST}
+echo "ENCLAVE_ID=${ENCLAVE_ID}"
+
+# export to Github output
+if [ -z "${GITHUB_OUTPUT}" ]; then
+  echo "Not in GitHub action"
+else
+  echo "ENCLAVE_ID=${ENCLAVE_ID}" >> ${GITHUB_OUTPUT}
+fi
diff --git a/scripts/aks/start_aks_enclave.sh b/scripts/aks/start_aks_enclave.sh
diff --git a/scripts/aks/stop_aks_enclave.sh b/scripts/aks/stop_aks_enclave.sh
diff --git a/scripts/decide_e2e_env.sh b/scripts/decide_e2e_env.sh
