From 2dd9fd5b38870af16f8f66ff89cd2255b4081e6c Mon Sep 17 00:00:00 2001 From: Colin Beeby Date: Fri, 28 Mar 2025 16:25:49 +0000 Subject: [PATCH] Bringing Prod CI files into line with CI/RC --- .../learninghub-moodle_Deploy_prod.yml | 92 ++++++++++++++++--- Terraform/prod/backend-prod.tfvars | 3 +- Terraform/prod/main.tf | 52 ++++++++++- kubectl/deployment-prod.yml | 1 + kubectl/pv-definition-prod.yml | 2 +- kubectl/pv-definition-theme-prod.yml | 3 +- kubectl/pvc-definition-prod.yml | 1 + kubectl/pvc-definition-theme-prod.yml | 1 + kubectl/service-prod.yml | 1 + 9 files changed, 135 insertions(+), 21 deletions(-) diff --git a/.github/workflows/learninghub-moodle_Deploy_prod.yml b/.github/workflows/learninghub-moodle_Deploy_prod.yml index 284fc4e9615..176182d3c98 100644 --- a/.github/workflows/learninghub-moodle_Deploy_prod.yml +++ b/.github/workflows/learninghub-moodle_Deploy_prod.yml @@ -92,8 +92,8 @@ jobs: # -var="SQLAdministratorLoginPassword=${{ secrets.SQL_ADMINISTRATOR_LOGIN_PASSWORD }}" \ # -var="Environment=${{ vars.AZURE_ENVIRONMENT }}" \ # -var="StorageQuota=${{ vars.AZURE_STORAGE_QUOTA }}" \ - # -var="ClusterNodeCount=${{ vars.AZURE_CLUSTER_NODE_COUNT }}" \ - # -var="ClusterNodeSize=${{ vars.AZURE_CLUSTER_NODE_SIZE }}" \ + # -var="ClusterNodeCount=${{ steps.evaluate_cluster_node_count.outputs.clusternodecount }}" \ + # -var="ClusterNodeSize=${{ steps.evaluate_cluster_node_size.outputs.clusternodesize }}" \ # -var="SQLSkuName=${{ vars.SQL_SKU_NAME }}" \ # -var="SQLStorageSize=${{ vars.SQL_STORAGE_SIZE }}" \ # -var="SQLVcores=${{ vars.SQL_VCORES }}" \ @@ -145,16 +145,16 @@ jobs: run: az aks get-credentials --resource-group ${{ vars.AZURE_RESOURCE_GROUP_NAME }} --name ${{ vars.AZURE_CLUSTER_NAME }} - name: Create PersistentVolume - run: kubectl apply -f kubectl/pv-definition-test.yml + run: kubectl apply -f kubectl/pv-definition-prod.yml - name: Create PersistentVolume for theme - run: kubectl apply -f kubectl/pv-definition-theme-test.yml + run: kubectl apply -f kubectl/pv-definition-theme-prod.yml - name: Create PersistentVolumeClaim - run: kubectl apply -f kubectl/pvc-definition-test.yml + run: kubectl apply -f kubectl/pvc-definition-prod.yml - name: Create PersistentVolumeClaim for theme - run: kubectl apply -f kubectl/pvc-definition-theme-test.yml + run: kubectl apply -f kubectl/pvc-definition-theme-prod.yml - name: Attach ACR to cluster run: az aks update -n ${{ vars.AZURE_CLUSTER_NAME }} -g ${{ vars.AZURE_RESOURCE_GROUP_NAME }} --attach-acr ${{ vars.AZURE_CONTAINER_REGISTRY_NAME }} @@ -235,7 +235,9 @@ jobs: - name: Build and push Docker image run: | - docker build -t ${{ vars.AZURE_CONTAINER_REGISTRY_NAME }}.azurecr.io/${{ vars.DOCKER_IMAGE_NAME }}:latest . + docker build \ + --build-arg PLUGIN_SET="${{ vars.PLUGIN_SET }}" \ + -t ${{ vars.AZURE_CONTAINER_REGISTRY_NAME }}.azurecr.io/${{ vars.DOCKER_IMAGE_NAME }}:latest . docker push ${{ vars.AZURE_CONTAINER_REGISTRY_NAME }}.azurecr.io/${{ vars.DOCKER_IMAGE_NAME }}:latest deploy_learninghubmoodle_to_cluster: @@ -268,12 +270,12 @@ jobs: STORAGE_KEY=$(az storage account keys list --resource-group ${{ vars.AZURE_RESOURCE_GROUP_NAME }} --account-name ${{ vars.AZURE_STORAGE_ACCOUNT_NAME }} --query [0].value -o tsv) echo "storage_key=$STORAGE_KEY" >> $GITHUB_OUTPUT - - name: Create the default namespace secret - run: kubectl create secret generic azure-secret --from-literal=azurestorageaccountname=${{ vars.AZURE_STORAGE_ACCOUNT_NAME }} --from-literal=azurestorageaccountkey=${{ steps.get-storage-key.outputs.storage_key }} -n default + - name: Create the namespace secret + run: kubectl create secret generic azure-secret --from-literal=azurestorageaccountname=${{ vars.AZURE_STORAGE_ACCOUNT_NAME }} --from-literal=azurestorageaccountkey=${{ steps.get-storage-key.outputs.storage_key }} -n learninghubmoodle continue-on-error: true - name: Deploy application image to AKS cluster - run: kubectl apply -f kubectl/deployment-prod.yml + run: kubectl apply -f kubectl/deployment-prod.yml --validate=false - name: Deploy the Horizontal Pod Autoscaler to the AKS cluster run: kubectl apply -f kubectl/hpa-prod.yml @@ -283,7 +285,7 @@ jobs: az aks nodepool update \ --resource-group ${{ vars.AZURE_RESOURCE_GROUP_NAME }} \ --cluster-name ${{ vars.AZURE_CLUSTER_NAME }} \ - --name default \ + --name userpool \ --enable-cluster-autoscaler \ --min-count 1 \ --max-count 10 @@ -292,4 +294,70 @@ jobs: run: kubectl apply -f kubectl/service-prod.yml - name: Restart the pods to reload the image - run: kubectl rollout restart deployment/learninghubmoodleprod \ No newline at end of file + run: kubectl rollout restart deployment/learninghubmoodleprod -n learninghubmoodle + + fix_recommendations: + needs: deploy_learninghubmoodle_to_cluster + runs-on: ubuntu-latest + environment: Prod + name: 'Ensure Azure recommendations are fixed' + env: + AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} + + steps: + - uses: actions/checkout@v2 + + - name: 'Az CLI login' + uses: azure/login@v1 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + - name: Get AKS credentials + run: az aks get-credentials --resource-group ${{ vars.AZURE_RESOURCE_GROUP_NAME }} --name ${{ vars.AZURE_CLUSTER_NAME }} + + - name: Enable Defender for AKS cluster + run: az aks update --resource-group ${{ vars.AZURE_RESOURCE_GROUP_NAME }} --name ${{ vars.AZURE_CLUSTER_NAME }} --enable-defender + + - name: Disable automounting API credentials + run: | + kubectl patch deployment learninghubmoodledev -p "{\"spec\": {\"template\": {\"spec\": {\"automountServiceAccountToken\": false}}}}" -n learninghubmoodle + kubectl patch serviceaccount default -p "{\"automountServiceAccountToken\": false}" -n learninghubmoodle + + - name: Get node resource group + id: get_node_resource_group + run: | + node_resource_group=$(az aks show --resource-group ${{ vars.AZURE_RESOURCE_GROUP_NAME }} --name ${{ vars.AZURE_CLUSTER_NAME }} --query "nodeResourceGroup" -o tsv) + echo "node_resource_group=$node_resource_group" >> $GITHUB_OUTPUT + + - name: Add Azure Policy add-on for AKS + run: az aks enable-addons --addon azure-policy --resource-group ${{ vars.AZURE_RESOURCE_GROUP_NAME }} --name ${{ vars.AZURE_CLUSTER_NAME }} + + - name: Enable Container insights + run: az aks enable-addons --addon monitoring --resource-group ${{ vars.AZURE_RESOURCE_GROUP_NAME }} --name ${{ vars.AZURE_CLUSTER_NAME }} + continue-on-error: true + + - name: Get cluster id + id: get_cluster_id + run: | + CLUSTER_ID=$(az aks show --resource-group ${{ vars.AZURE_RESOURCE_GROUP_NAME }} --name ${{ vars.AZURE_CLUSTER_NAME }} --query id -o tsv) + echo "cluster_id=$CLUSTER_ID" >> $GITHUB_OUTPUT + + - name: Create analytics workspace + id: create_analytics_workspace + run: | + az monitor log-analytics workspace create --resource-group ${{ vars.AZURE_RESOURCE_GROUP_NAME }} --workspace-name LearningHubWorkspace --location ${{ vars.AZURE_RESOURCE_GROUP_LOCATION }} + WORKSPACE_ID=$(az monitor log-analytics workspace show --resource-group ${{ vars.AZURE_RESOURCE_GROUP_NAME }} --workspace-name LearningHubWorkspace --query id -o tsv) + echo "workspace_id=$WORKSPACE_ID" >> $GITHUB_OUTPUT + + - name: Create diagnostic settings + run: | + az monitor diagnostic-settings create --name "AKS-Diagnostic-Logs" --resource-group ${{ vars.AZURE_RESOURCE_GROUP_NAME }} --resource ${{ steps.get_cluster_id.outputs.cluster_id }} --workspace ${{ steps.create_analytics_workspace.outputs.workspace_id }} --logs "[{\"category\": \"kube-apiserver\", \"enabled\": true}, {\"category\": \"kube-controller-manager\", \"enabled\": true}, {\"category\": \"kube-scheduler\", \"enabled\": true}, {\"category\": \"cluster-autoscaler\", \"enabled\": true}]" + + - name: Define trusted container registries + run: | + az policy assignment create --resource-group ${{ vars.AZURE_RESOURCE_GROUP_NAME }} --name "EnforceTrustedRegistries" --policy "febd0533-8e55-448f-b837-bd0e06f16469" --params "{\"allowedContainerImagesRegex\":{\"value\":\"learninghubmoodlecrprod.azurecr.io\"}}" --location northeurope \ No newline at end of file diff --git a/Terraform/prod/backend-prod.tfvars b/Terraform/prod/backend-prod.tfvars index 3678bcb5aea..c63150c4a9a 100644 --- a/Terraform/prod/backend-prod.tfvars +++ b/Terraform/prod/backend-prod.tfvars @@ -1,3 +1,2 @@ key="learninghub-moodle.prod.terraform.tfstate" -storage_account_name="learninghubmoodleprod" -resource_group_name = "LearningHub-Prod-Stor-RG" \ No newline at end of file +storage_account_name="userprofilesa11" \ No newline at end of file diff --git a/Terraform/prod/main.tf b/Terraform/prod/main.tf index 1e4dc223de9..36a0d1d0a88 100644 --- a/Terraform/prod/main.tf +++ b/Terraform/prod/main.tf @@ -23,16 +23,25 @@ resource "azurerm_storage_share" "storage_share_theme" { quota = var.StorageQuota } +resource "azurerm_storage_container" "assessment_container" { + name = "assessmentstoragecontainer" + storage_account_name = azurerm_storage_account.storage_account.name + container_access_type = "private" +} + resource "azurerm_kubernetes_cluster" "aks" { name = var.ClusterName location = azurerm_resource_group.learningHubMoodleResourceGroup.location resource_group_name = azurerm_resource_group.learningHubMoodleResourceGroup.name dns_prefix = var.ClusterName default_node_pool { - name = "default" - node_count = var.ClusterNodeCount - vm_size = var.ClusterNodeSize - temporary_name_for_rotation = "tmpnodepool1" + name = "default" + vm_size = var.ClusterNodeSize + temporary_name_for_rotation = "tmpnodepool1" + auto_scaling_enabled = true + min_count = 2 + max_count = 3 + only_critical_addons_enabled = true } identity { type = "SystemAssigned" @@ -45,6 +54,33 @@ resource "azurerm_kubernetes_cluster" "aks" { } } +provider "kubernetes" { + host = azurerm_kubernetes_cluster.aks.kube_config.0.host + client_certificate = base64decode(azurerm_kubernetes_cluster.aks.kube_config.0.client_certificate) + client_key = base64decode(azurerm_kubernetes_cluster.aks.kube_config.0.client_key) + cluster_ca_certificate = base64decode(azurerm_kubernetes_cluster.aks.kube_config.0.cluster_ca_certificate) +} + +resource "kubernetes_namespace" "custom_namespace" { + metadata { + name = "learninghubmoodle" + } +} + +resource "azurerm_kubernetes_cluster_node_pool" "user_node_pool" { + name = "userpool" + kubernetes_cluster_id = azurerm_kubernetes_cluster.aks.id + vm_size = var.ClusterNodeSize + auto_scaling_enabled = true + min_count = 1 + max_count = 20 + node_count = var.ClusterNodeCount + mode = "User" + tags = { + Environment = var.Environment + } +} + resource "azurerm_container_registry" "containerRegistry" { name = var.ContainerRegistryName resource_group_name = azurerm_resource_group.learningHubMoodleResourceGroup.name @@ -259,6 +295,12 @@ resource "azurerm_mssql_managed_instance" "sqlmi" { tags = { environment = var.Environment } + identity { + type = "SystemAssigned" + } + lifecycle { + prevent_destroy = true + } } resource "azurerm_mssql_managed_database" "sqldb" { @@ -273,7 +315,7 @@ resource "azurerm_redis_cache" "moodle_cache" { capacity = 2 family = "C" sku_name = "Standard" - non_ssl_port_enabled = true + non_ssl_port_enabled = false minimum_tls_version = "1.2" } diff --git a/kubectl/deployment-prod.yml b/kubectl/deployment-prod.yml index 6ce6140b99f..0ca88ed94ef 100644 --- a/kubectl/deployment-prod.yml +++ b/kubectl/deployment-prod.yml @@ -2,6 +2,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: learninghubmoodleprod + namespace: learninghubmoodle spec: replicas: 5 selector: diff --git a/kubectl/pv-definition-prod.yml b/kubectl/pv-definition-prod.yml index e2d171ae9d0..b93608f45be 100644 --- a/kubectl/pv-definition-prod.yml +++ b/kubectl/pv-definition-prod.yml @@ -18,7 +18,7 @@ spec: shareName: moodledata nodeStageSecretRef: name: azure-secret - namespace: default + namespace: learninghubmoodle mountOptions: - dir_mode=0777 - file_mode=0777 diff --git a/kubectl/pv-definition-theme-prod.yml b/kubectl/pv-definition-theme-prod.yml index baf5bcd6b41..2872e6ce9f4 100644 --- a/kubectl/pv-definition-theme-prod.yml +++ b/kubectl/pv-definition-theme-prod.yml @@ -16,9 +16,10 @@ spec: volumeHandle: "moodleCluster#learninghubmoodleprod#moodletheme" # make sure this volumeid is unique for every identical share in the cluster volumeAttributes: shareName: moodletheme + server: learninghubmoodletest.privatelink.file.core.windows.net nodeStageSecretRef: name: azure-secret - namespace: default + namespace: learninghubmoodle mountOptions: - dir_mode=0777 - file_mode=0777 diff --git a/kubectl/pvc-definition-prod.yml b/kubectl/pvc-definition-prod.yml index c3421991a82..99631c40a98 100644 --- a/kubectl/pvc-definition-prod.yml +++ b/kubectl/pvc-definition-prod.yml @@ -2,6 +2,7 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: moodledataclaim + namespace: learninghubmoodle spec: accessModes: - ReadWriteMany diff --git a/kubectl/pvc-definition-theme-prod.yml b/kubectl/pvc-definition-theme-prod.yml index 3fae7d24702..51189c3fd48 100644 --- a/kubectl/pvc-definition-theme-prod.yml +++ b/kubectl/pvc-definition-theme-prod.yml @@ -2,6 +2,7 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: moodlethemeclaim + namespace: learninghubmoodle spec: accessModes: - ReadWriteMany diff --git a/kubectl/service-prod.yml b/kubectl/service-prod.yml index 27ec59d0e6f..ddbace79512 100644 --- a/kubectl/service-prod.yml +++ b/kubectl/service-prod.yml @@ -2,6 +2,7 @@ apiVersion: v1 kind: Service metadata: name: learninghubmoodle-service + namespace: learninghubmoodle spec: selector: app: learninghubmoodleprod