diff --git a/.github/actions/install_az_cli/action.yaml b/.github/actions/install_az_cli/action.yaml new file mode 100644 index 000000000..e61f02458 --- /dev/null +++ b/.github/actions/install_az_cli/action.yaml @@ -0,0 +1,36 @@ +name: 'Install Azure CLI' +description: 'Install Azure CLI' +runs: + using: 'composite' + steps: + - name: uninstall azure-cli + shell: bash + run: | + sudo apt-get remove -y azure-cli + + - name: install azure-cli 2.61.0 + shell: bash + run: | + sudo apt-get update + sudo apt-get install apt-transport-https ca-certificates curl gnupg lsb-release + sudo mkdir -p /etc/apt/keyrings + curl -sLS https://packages.microsoft.com/keys/microsoft.asc | + gpg --dearmor | sudo tee /etc/apt/keyrings/microsoft.gpg > /dev/null + sudo chmod go+r /etc/apt/keyrings/microsoft.gpg + AZ_DIST=$(lsb_release -cs) + echo "Types: deb + URIs: https://packages.microsoft.com/repos/azure-cli/ + Suites: ${AZ_DIST} + Components: main + Architectures: $(dpkg --print-architecture) + Signed-by: /etc/apt/keyrings/microsoft.gpg" | sudo tee /etc/apt/sources.list.d/azure-cli.sources + sudo apt-get update + sudo apt-get install azure-cli + + apt-cache policy azure-cli + # Obtain the currently installed distribution + AZ_DIST=$(lsb_release -cs) + # Store an Azure CLI version of choice + AZ_VER=2.61.0 + # Install a specific version + sudo apt-get install azure-cli=${AZ_VER}-1~${AZ_DIST} --allow-downgrades diff --git a/.github/workflows/publish-azure-cc-enclave-docker.yaml b/.github/workflows/publish-azure-cc-enclave-docker.yaml index 84e74f5cb..2f90fac3b 100644 --- a/.github/workflows/publish-azure-cc-enclave-docker.yaml +++ b/.github/workflows/publish-azure-cc-enclave-docker.yaml @@ -68,7 +68,10 @@ jobs: pull-requests: write outputs: jar_version: ${{ steps.update_version.outputs.new_version }} + docker_version: ${{ steps.meta.outputs.version }} image_tag: ${{ steps.update_version.outputs.image_tag }} + tags: ${{ steps.meta.outputs.tags }} + is_release: ${{ steps.update_version.outputs.is_release }} steps: - name: Update Operator Version id: update_version @@ -159,35 +162,19 @@ jobs: JAR_VERSION=${{ steps.update_version.outputs.new_version }} IMAGE_VERSION=${{ steps.update_version.outputs.new_version }} - - name: uninstall azure-cli - run: | - sudo apt-get remove -y azure-cli - - - name: install azure-cli 2.61.0 - run: | - sudo apt-get update - sudo apt-get install apt-transport-https ca-certificates curl gnupg lsb-release - sudo mkdir -p /etc/apt/keyrings - curl -sLS https://packages.microsoft.com/keys/microsoft.asc | - gpg --dearmor | sudo tee /etc/apt/keyrings/microsoft.gpg > /dev/null - sudo chmod go+r /etc/apt/keyrings/microsoft.gpg - AZ_DIST=$(lsb_release -cs) - echo "Types: deb - URIs: https://packages.microsoft.com/repos/azure-cli/ - Suites: ${AZ_DIST} - Components: main - Architectures: $(dpkg --print-architecture) - Signed-by: /etc/apt/keyrings/microsoft.gpg" | sudo tee /etc/apt/sources.list.d/azure-cli.sources - sudo apt-get update - sudo apt-get install azure-cli - - apt-cache policy azure-cli - # Obtain the currently installed distribution - AZ_DIST=$(lsb_release -cs) - # Store an Azure CLI version of choice - AZ_VER=2.61.0 - # Install a specific version - sudo apt-get install azure-cli=${AZ_VER}-1~${AZ_DIST} --allow-downgrades + azureCc: + name: Azure CC + runs-on: ubuntu-latest + permissions: {} + needs: buildImage + env: + ARTIFACT_PREFIX: azure-cc- + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Azure CLI + uses: ./.github/actions/install_az_cli - name: check azure-cli version run: | @@ -195,61 +182,140 @@ jobs: - name: Generate Azure deployment artifacts env: - IMAGE: ${{ steps.meta.outputs.tags }} + IMAGE: ${{ needs.buildImage.outputs.tags }} OUTPUT_DIR: ${{ env.ARTIFACTS_OUTPUT_DIR }} MANIFEST_DIR: ${{ env.MANIFEST_OUTPUT_DIR }} - VERSION_NUMBER: ${{ steps.update_version.outputs.new_version }} + VERSION_NUMBER: ${{ needs.buildImage.outputs.jar_version }} run: | bash ./scripts/azure-cc/deployment/generate-deployment-artifacts.sh - name: Upload deployment artifacts uses: actions/upload-artifact@v4 with: - name: azure-cc-deployment-files-${{ steps.update_version.outputs.new_version }} + name: ${{ env.ARTIFACT_PREFIX }}deployment-files-${{ needs.buildImage.outputs.jar_version }} path: ${{ env.ARTIFACTS_OUTPUT_DIR }} if-no-files-found: error - name: Upload manifest uses: actions/upload-artifact@v4 with: - name: azure-cc-enclave-id-${{ steps.update_version.outputs.new_version }} + name: ${{ env.ARTIFACT_PREFIX }}enclave-id-${{ needs.buildImage.outputs.jar_version }} path: ${{ env.MANIFEST_OUTPUT_DIR }} if-no-files-found: error - name: Generate release archive - if: ${{ inputs.version_number_input == '' && steps.update_version.outputs.is_release == 'true' }} + if: ${{ inputs.version_number_input == '' && needs.buildImage.outputs.is_release == 'true' }} run: | - zip -j ${{ env.ARTIFACTS_OUTPUT_DIR }}/uid2-operator-deployment-artifacts-${{ steps.meta.outputs.version }}.zip ${{ env.ARTIFACTS_OUTPUT_DIR }}/* + zip -j ${{ env.ARTIFACTS_OUTPUT_DIR }}/uid2-operator-deployment-artifacts-${{ needs.buildImage.outputs.docker_version }}.zip ${{ env.ARTIFACTS_OUTPUT_DIR }}/* - name: Build changelog id: github_release - if: ${{ inputs.version_number_input == '' && steps.update_version.outputs.is_release == 'true' }} + if: ${{ inputs.version_number_input == '' && needs.buildImage.outputs.is_release == 'true' }} uses: mikepenz/release-changelog-builder-action@v4 with: configurationJson: | { - "template": "#{{CHANGELOG}}\n## Installation\n```\ndocker pull ${{ steps.meta.outputs.tags }}\n```\n\n## Image reference to deploy: \n```\n${{ steps.update_version.outputs.image_tag }}\n```\n\n## Changelog\n#{{UNCATEGORIZED}}", + "template": "#{{CHANGELOG}}\n## Installation\n```\ndocker pull ${{ needs.buildImage.outputs.tags }}\n```\n\n## Image reference to deploy: \n```\n${{ needs.buildImage.outputs.image_tag }}\n```\n\n## Changelog\n#{{UNCATEGORIZED}}", "pr_template": " - #{{TITLE}} - ( PR: ##{{NUMBER}} )" } env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Create release - if: ${{ inputs.version_number_input == '' && steps.update_version.outputs.is_release == 'true' }} + if: ${{ inputs.version_number_input == '' && needs.buildImage.outputs.is_release == 'true' }} uses: softprops/action-gh-release@v2 with: - name: ${{ steps.update_version.outputs.new_version }} + name: ${{ needs.buildImage.outputs.jar_version }} body: ${{ steps.github_release.outputs.changelog }} draft: true files: | - ${{ env.ARTIFACTS_OUTPUT_DIR }}/uid2-operator-deployment-artifacts-${{ steps.update_version.outputs.new_version }}.zip - ${{ env.MANIFEST_OUTPUT_DIR }}/azure-cc-operator-digest-${{ steps.update_version.outputs.new_version }}.txt + ${{ env.ARTIFACTS_OUTPUT_DIR }}/uid2-operator-deployment-artifacts-${{ needs.buildImage.outputs.jar_version }}.zip + ${{ env.MANIFEST_OUTPUT_DIR }}/azure-cc-operator-digest-${{ needs.buildImage.outputs.jar_version }}.txt - e2e: - name: E2E + e2eAzureCc: + name: E2E Azure CC uses: ./.github/workflows/run-e2e-tests-on-operator.yaml - needs: buildImage + needs: [buildImage, azureCc] with: operator_type: azure operator_image_version: ${{ needs.buildImage.outputs.image_tag }} secrets: inherit + + azureVn: + name: Azure VN + runs-on: ubuntu-latest + permissions: {} + needs: buildImage + env: + ARTIFACT_PREFIX: azure-vn- + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Azure CLI + uses: ./.github/actions/install_az_cli + + - name: check azure-cli version + run: | + az --version + + - name: Generate Azure deployment artifacts + env: + IMAGE: ${{ needs.buildImage.outputs.tags }} + OUTPUT_DIR: ${{ env.ARTIFACTS_OUTPUT_DIR }} + MANIFEST_DIR: ${{ env.MANIFEST_OUTPUT_DIR }} + VERSION_NUMBER: ${{ needs.buildImage.outputs.jar_version }} + run: | + bash ./scripts/azure-vn/deployment/generate-deployment-artifacts.sh + + - name: Upload deployment artifacts + uses: actions/upload-artifact@v4 + with: + name: ${{ env.ARTIFACT_PREFIX }}deployment-files-${{ needs.buildImage.outputs.jar_version }} + path: ${{ env.ARTIFACTS_OUTPUT_DIR }} + if-no-files-found: error + + - name: Upload manifest + uses: actions/upload-artifact@v4 + with: + name: ${{ env.ARTIFACT_PREFIX }}enclave-id-${{ needs.buildImage.outputs.jar_version }} + path: ${{ env.MANIFEST_OUTPUT_DIR }} + if-no-files-found: error + + - name: Generate release archive + if: ${{ inputs.version_number_input == '' && needs.buildImage.outputs.is_release == 'true' }} + run: | + zip -j ${{ env.ARTIFACTS_OUTPUT_DIR }}/uid2-operator-deployment-artifacts-${{ needs.buildImage.outputs.docker_version }}.zip ${{ env.ARTIFACTS_OUTPUT_DIR }}/* + + - name: Build changelog + id: github_release + if: ${{ inputs.version_number_input == '' && needs.buildImage.outputs.is_release == 'true' }} + uses: mikepenz/release-changelog-builder-action@v4 + with: + configurationJson: | + { + "template": "#{{CHANGELOG}}\n## Installation\n```\ndocker pull ${{ needs.buildImage.outputs.tags }}\n```\n\n## Image reference to deploy: \n```\n${{ needs.buildImage.outputs.image_tag }}\n```\n\n## Changelog\n#{{UNCATEGORIZED}}", + "pr_template": " - #{{TITLE}} - ( PR: ##{{NUMBER}} )" + } + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Create release + if: ${{ inputs.version_number_input == '' && needs.buildImage.outputs.is_release == 'true' }} + uses: softprops/action-gh-release@v2 + with: + name: ${{ needs.buildImage.outputs.jar_version }} + body: ${{ steps.github_release.outputs.changelog }} + draft: true + files: | + ${{ env.ARTIFACTS_OUTPUT_DIR }}/uid2-operator-deployment-artifacts-${{ needs.buildImage.outputs.jar_version }}.zip + ${{ env.MANIFEST_OUTPUT_DIR }}/${{ env.ARTIFACT_PREFIX }}operator-digest-${{ needs.buildImage.outputs.jar_version }}.txt + + e2eAzureCc: + name: E2E Azure VN + uses: ./.github/workflows/run-e2e-tests-on-operator.yaml + needs: [buildImage, azureCc] + with: + operator_type: azureVn + operator_image_version: ${{ needs.buildImage.outputs.image_tag }} + secrets: inherit diff --git a/.github/workflows/run-e2e-tests-on-operator.yaml b/.github/workflows/run-e2e-tests-on-operator.yaml index e57756c1b..40e05411b 100644 --- a/.github/workflows/run-e2e-tests-on-operator.yaml +++ b/.github/workflows/run-e2e-tests-on-operator.yaml @@ -4,7 +4,7 @@ on: workflow_dispatch: inputs: operator_type: - description: The operator type [public, gcp, azure, aws, eks] + description: The operator type [public, gcp, azure, aws, eks, aks] required: true type: choice options: @@ -13,6 +13,7 @@ on: - azure - aws - eks + - aks identity_scope: description: The identity scope [UID2, EUID] required: true @@ -106,7 +107,7 @@ on: jobs: e2e-test: name: E2E Test - uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-run-e2e-tests.yaml@v3 + uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-run-e2e-tests.yaml@kcc-UID2-5092-aks-e2e with: operator_type: ${{ inputs.operator_type }} operator_image_version: ${{ inputs.operator_image_version }} diff --git a/scripts/azure-aks/deployment/generate-deployment-artifacts.sh b/scripts/azure-aks/deployment/generate-deployment-artifacts.sh new file mode 100644 index 000000000..7d9c5cc4c --- /dev/null +++ b/scripts/azure-aks/deployment/generate-deployment-artifacts.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +set -x + +# Following environment variables must be set +# - IMAGE: uid2-operator image +# - OUTPUT_DIR: output directory to store the artifacts +# - MANIFEST_DIR: output directory to store the manifest for the enclave Id +# - VERSION_NUMBER: the version number of the build + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +INPUT_DIR=${SCRIPT_DIR} + +if [[ -z ${IMAGE} ]]; then + echo "IMAGE cannot be empty" + exit 1 +fi +IMAGE_VERSION=$(echo $IMAGE | awk -F':' '{print $2}') +if [[ -z ${IMAGE_VERSION} ]]; then + echo "Failed to extract image version from ${IMAGE}" + exit 1 +fi + +if [[ -z ${OUTPUT_DIR} ]]; then + echo "OUTPUT_DIR cannot be empty" + exit 1 +fi + +mkdir -p ${OUTPUT_DIR} +if [[ $? -ne 0 ]]; then + echo "Failed to create ${OUTPUT_DIR}" + exit 1 +fi + +mkdir -p ${MANIFEST_DIR} +if [[ $? -ne 0 ]]; then + echo "Failed to create ${MANIFEST_DIR}" + exit 1 +fi + +# Input files +INPUT_FILES=( + operator.yaml +) + +# Copy input files to output dir +for f in ${INPUT_FILES[@]}; do + cp ${INPUT_DIR}/${f} ${OUTPUT_DIR}/${f} + if [[ $? -ne 0 ]]; then + echo "Failed to copy ${INPUT_DIR}/${f} to ${OUTPUT_DIR}" + exit 1 + fi +done + +az version +# Install confcom extension, az is originally available in GitHub workflow environment +az extension add --name confcom +if [[ $? -ne 0 ]]; then + echo "Failed to install Azure confcom extension" + exit 1 +fi + +# Required by az confcom +sudo usermod -aG docker ${USER} +if [[ $? -ne 0 ]]; then + echo "Failed to add current user to docker group" + exit 1 +fi + +# Generate operator template +sed -i "s#IMAGE_PLACEHOLDER#${IMAGE}#g" ${OUTPUT_DIR}/operator.yaml +# && \ +# sed -i "s#IMAGE_VERSION_PLACEHOLDER#${IMAGE_VERSION}#g" ${OUTPUT_DIR}/operator.yaml +if [[ $? -ne 0 ]]; then + echo "Failed to pre-process operator template file" + exit 1 +fi + +# Export the policy, update it to turn off allow_environment_variable_dropping, and then insert it into the template +# note that the EnclaveId is generated by generate.py on the raw policy, not the base64 version +POLICY_DIGEST_FILE=azure-vn-operator-digest-$VERSION_NUMBER.txt +az confcom acipolicygen --virtual-node-yaml ${OUTPUT_DIR}/operator.yaml --print-policy > ${INPUT_DIR}/policy.base64 +if [[ $? -ne 0 ]]; then + echo "Failed to generate ACI policy" + exit 1 +fi + +base64 -di < ${INPUT_DIR}/policy.base64 > ${INPUT_DIR}/generated.rego +sed -i "s#allow_environment_variable_dropping := true#allow_environment_variable_dropping := false#g" ${INPUT_DIR}/generated.rego +sed -i 's#{"pattern":"DEPLOYMENT_ENVIRONMENT=DEPLOYMENT_ENVIRONMENT_PLACEHOLDER","required":false,"strategy":"string"}#{"pattern":"DEPLOYMENT_ENVIRONMENT=.+","required":false,"strategy":"re2"}#g' generated.rego +sed -i 's#{"pattern":"VAULT_NAME=VAULT_NAME_PLACEHOLDER","required":false,"strategy":"string"}#{"pattern":"VAULT_NAME=.+","required":false,"strategy":"re2"}#g' generated.rego +sed -i 's#{"pattern":"OPERATOR_KEY_SECRET_NAME=OPERATOR_KEY_SECRET_NAME_PLACEHOLDER","required":false,"strategy":"string"}#{"pattern":"OPERATOR_KEY_SECRET_NAME=.+","required":false,"strategy":"re2"}#g' generated.rego +base64 -w0 < ${INPUT_DIR}/generated.rego > ${INPUT_DIR}/generated.rego.base64 +python3 ${SCRIPT_DIR}/generate.py ${INPUT_DIR}/generated.rego > ${MANIFEST_DIR}/${POLICY_DIGEST_FILE} + +sed -i "s#CCE_POLICY_PLACEHOLDER#$(cat ${INPUT_DIR}/generated.rego.base64)#g" ${OUTPUT_DIR}/operator.yaml diff --git a/scripts/azure-aks/deployment/generate.py b/scripts/azure-aks/deployment/generate.py new file mode 100644 index 000000000..07845beac --- /dev/null +++ b/scripts/azure-aks/deployment/generate.py @@ -0,0 +1,20 @@ +import sys +from hashlib import sha256 + +def str_to_sha256(x: str) -> str: + return sha256(x.encode('utf-8')).hexdigest() + +def print_data_sha256(data: str) -> str: + print(str_to_sha256(data)) + +def print_data_sha256_stripped(data: str) -> str: + print(str_to_sha256(data.strip())) + +def main(): + with open(sys.argv[1], 'r') as file: + data = file.read() + + print_data_sha256(data) + +if __name__ == '__main__': + main() diff --git a/scripts/azure-aks/deployment/operator.yaml b/scripts/azure-aks/deployment/operator.yaml new file mode 100644 index 000000000..80fd144cd --- /dev/null +++ b/scripts/azure-aks/deployment/operator.yaml @@ -0,0 +1,91 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: operator-deployment +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: operator + template: + metadata: + labels: + app.kubernetes.io/name: operator + annotations: + microsoft.containerinstance.virtualnode.ccepolicy: CCE_POLICY_PLACEHOLDER + microsoft.containerinstance.virtualnode.identity: IDENTITY_PLACEHOLDER + microsoft.containerinstance.virtualnode.injectdns: "false" + spec: + containers: + - image: "mcr.microsoft.com/aci/skr:2.7" + imagePullPolicy: Always + name: skr + resources: + limits: + cpu: 2250m + memory: 2256Mi + requests: + cpu: 100m + memory: 512Mi + env: + - name: Port + value: "9000" + volumeMounts: + - mountPath: /opt/confidential-containers/share/kata-containers/reference-info-base64 + name: endorsement-location + command: + - /skr.sh + - name: uid2-operator + image: IMAGE_PLACEHOLDER + resources: + limits: + memory: "8Gi" + imagePullPolicy: Always + securityContext: + runAsUser: 1000 + env: + - name: VAULT_NAME + value: VAULT_NAME_PLACEHOLDER + - name: OPERATOR_KEY_SECRET_NAME + value: OPERATOR_KEY_SECRET_NAME_PLACEHOLDER + - name: DEPLOYMENT_ENVIRONMENT + value: DEPLOYMENT_ENVIRONMENT_PLACEHOLDER + ports: + - containerPort: 8080 + protocol: TCP + - name: prometheus + containerPort: 9080 + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /ops/healthcheck + port: 8080 + scheme: HTTP + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + volumes: + - name: endorsement-location + hostPath: + path: /opt/confidential-containers/share/kata-containers/reference-info-base64 + nodeSelector: + virtualization: virtualnode2 + tolerations: + - effect: NoSchedule + key: virtual-kubelet.io/provider + operator: Exists +--- +apiVersion: v1 +kind: Service +metadata: + name: operator-svc +spec: + type: LoadBalancer + selector: + app.kubernetes.io/name: operator + ports: + - protocol: TCP + port: 80 + targetPort: 8080