TechnologyEnhancedLearning · ColinBeebyHEE · Mar 28, 2025 · Mar 28, 2025 · Mar 28, 2025
diff --git a/.github/workflows/learninghub-moodle_Deploy_prod.yml b/.github/workflows/learninghub-moodle_Deploy_prod.yml
@@ -92,8 +92,8 @@ jobs:
       #       -var="SQLAdministratorLoginPassword=${{ secrets.SQL_ADMINISTRATOR_LOGIN_PASSWORD }}" \
       #       -var="Environment=${{ vars.AZURE_ENVIRONMENT }}" \
       #       -var="StorageQuota=${{ vars.AZURE_STORAGE_QUOTA }}" \
-      #       -var="ClusterNodeCount=${{ vars.AZURE_CLUSTER_NODE_COUNT }}" \
-      #       -var="ClusterNodeSize=${{ vars.AZURE_CLUSTER_NODE_SIZE }}" \
+      #       -var="ClusterNodeCount=${{ steps.evaluate_cluster_node_count.outputs.clusternodecount }}" \
+      #       -var="ClusterNodeSize=${{ steps.evaluate_cluster_node_size.outputs.clusternodesize }}" \
       #       -var="SQLSkuName=${{ vars.SQL_SKU_NAME }}" \
       #       -var="SQLStorageSize=${{ vars.SQL_STORAGE_SIZE }}" \
       #       -var="SQLVcores=${{ vars.SQL_VCORES }}" \
@@ -145,16 +145,16 @@ jobs:
         run: az aks get-credentials --resource-group ${{ vars.AZURE_RESOURCE_GROUP_NAME }} --name ${{ vars.AZURE_CLUSTER_NAME }}
 
       - name: Create PersistentVolume
-        run: kubectl apply -f kubectl/pv-definition-test.yml
+        run: kubectl apply -f kubectl/pv-definition-prod.yml
 
       - name: Create PersistentVolume for theme
-        run: kubectl apply -f kubectl/pv-definition-theme-test.yml
+        run: kubectl apply -f kubectl/pv-definition-theme-prod.yml
 
       - name: Create PersistentVolumeClaim
-        run: kubectl apply -f kubectl/pvc-definition-test.yml
+        run: kubectl apply -f kubectl/pvc-definition-prod.yml
 
       - name: Create PersistentVolumeClaim for theme
-        run: kubectl apply -f kubectl/pvc-definition-theme-test.yml
+        run: kubectl apply -f kubectl/pvc-definition-theme-prod.yml
 
       - name: Attach ACR to cluster
         run: az aks update -n ${{ vars.AZURE_CLUSTER_NAME }} -g ${{ vars.AZURE_RESOURCE_GROUP_NAME }} --attach-acr ${{ vars.AZURE_CONTAINER_REGISTRY_NAME }}
@@ -235,7 +235,9 @@ jobs:
 
       - name: Build and push Docker image
         run: |
-          docker build -t ${{ vars.AZURE_CONTAINER_REGISTRY_NAME }}.azurecr.io/${{ vars.DOCKER_IMAGE_NAME }}:latest . 
+          docker build \
+            --build-arg PLUGIN_SET="${{ vars.PLUGIN_SET }}" \
+            -t ${{ vars.AZURE_CONTAINER_REGISTRY_NAME }}.azurecr.io/${{ vars.DOCKER_IMAGE_NAME }}:latest . 
           docker push ${{ vars.AZURE_CONTAINER_REGISTRY_NAME }}.azurecr.io/${{ vars.DOCKER_IMAGE_NAME }}:latest
 
   deploy_learninghubmoodle_to_cluster:
@@ -268,12 +270,12 @@ jobs:
           STORAGE_KEY=$(az storage account keys list --resource-group ${{ vars.AZURE_RESOURCE_GROUP_NAME }} --account-name ${{ vars.AZURE_STORAGE_ACCOUNT_NAME }} --query [0].value -o tsv)
           echo "storage_key=$STORAGE_KEY" >> $GITHUB_OUTPUT
 
-      - name: Create the default namespace secret
-        run: kubectl create secret generic azure-secret --from-literal=azurestorageaccountname=${{ vars.AZURE_STORAGE_ACCOUNT_NAME }} --from-literal=azurestorageaccountkey=${{ steps.get-storage-key.outputs.storage_key }} -n default
+      - name: Create the namespace secret
+        run: kubectl create secret generic azure-secret --from-literal=azurestorageaccountname=${{ vars.AZURE_STORAGE_ACCOUNT_NAME }} --from-literal=azurestorageaccountkey=${{ steps.get-storage-key.outputs.storage_key }} -n learninghubmoodle
         continue-on-error: true
 
       - name: Deploy application image to AKS cluster
-        run: kubectl apply -f kubectl/deployment-prod.yml
+        run: kubectl apply -f kubectl/deployment-prod.yml --validate=false
 
       - name: Deploy the Horizontal Pod Autoscaler to the AKS cluster
         run: kubectl apply -f kubectl/hpa-prod.yml
@@ -283,7 +285,7 @@ jobs:
           az aks nodepool update \
           --resource-group ${{ vars.AZURE_RESOURCE_GROUP_NAME }} \
           --cluster-name ${{ vars.AZURE_CLUSTER_NAME }} \
-          --name default \
+          --name userpool \
           --enable-cluster-autoscaler \
           --min-count 1 \
           --max-count 10
@@ -292,4 +294,70 @@ jobs:
         run: kubectl apply -f kubectl/service-prod.yml
 
       - name: Restart the pods to reload the image
-        run: kubectl rollout restart deployment/learninghubmoodleprod
+        run: kubectl rollout restart deployment/learninghubmoodleprod -n learninghubmoodle
+
+  fix_recommendations:
+    needs: deploy_learninghubmoodle_to_cluster
+    runs-on: ubuntu-latest
+    environment: Prod
+    name: 'Ensure Azure recommendations are fixed'
+    env:
+      AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+      AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
+      AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
+      AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: 'Az CLI login'
+        uses: azure/login@v1
+        with:
+          client-id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+
+      - name: Get AKS credentials
+        run: az aks get-credentials --resource-group ${{ vars.AZURE_RESOURCE_GROUP_NAME }} --name ${{ vars.AZURE_CLUSTER_NAME }}
+
+      - name: Enable Defender for AKS cluster
+        run: az aks update --resource-group ${{ vars.AZURE_RESOURCE_GROUP_NAME }} --name ${{ vars.AZURE_CLUSTER_NAME }} --enable-defender
+
+      - name: Disable automounting API credentials
+        run: |
+          kubectl patch deployment learninghubmoodledev -p "{\"spec\": {\"template\": {\"spec\": {\"automountServiceAccountToken\": false}}}}" -n learninghubmoodle
+          kubectl patch serviceaccount default -p "{\"automountServiceAccountToken\": false}" -n learninghubmoodle
+
+      - name: Get node resource group
+        id: get_node_resource_group
+        run: |
+          node_resource_group=$(az aks show --resource-group ${{ vars.AZURE_RESOURCE_GROUP_NAME }} --name ${{ vars.AZURE_CLUSTER_NAME }} --query "nodeResourceGroup" -o tsv)
+          echo "node_resource_group=$node_resource_group" >> $GITHUB_OUTPUT
+
+      - name: Add Azure Policy add-on for AKS
+        run: az aks enable-addons --addon azure-policy --resource-group ${{ vars.AZURE_RESOURCE_GROUP_NAME }} --name ${{ vars.AZURE_CLUSTER_NAME }} 
+
+      - name: Enable Container insights
+        run: az aks enable-addons --addon monitoring --resource-group ${{ vars.AZURE_RESOURCE_GROUP_NAME }} --name ${{ vars.AZURE_CLUSTER_NAME }} 
+        continue-on-error: true
+
+      - name: Get cluster id
+        id: get_cluster_id
+        run: |
+          CLUSTER_ID=$(az aks show --resource-group ${{ vars.AZURE_RESOURCE_GROUP_NAME }} --name ${{ vars.AZURE_CLUSTER_NAME }} --query id -o tsv)
+          echo "cluster_id=$CLUSTER_ID" >> $GITHUB_OUTPUT
+
+      - name: Create analytics workspace
+        id: create_analytics_workspace
+        run: |
+          az monitor log-analytics workspace create --resource-group ${{ vars.AZURE_RESOURCE_GROUP_NAME }} --workspace-name LearningHubWorkspace --location ${{ vars.AZURE_RESOURCE_GROUP_LOCATION }}
+          WORKSPACE_ID=$(az monitor log-analytics workspace show --resource-group ${{ vars.AZURE_RESOURCE_GROUP_NAME }} --workspace-name LearningHubWorkspace --query id -o tsv)
+          echo "workspace_id=$WORKSPACE_ID" >> $GITHUB_OUTPUT
+
+      - name: Create diagnostic settings
+        run: |
+          az monitor diagnostic-settings create --name "AKS-Diagnostic-Logs" --resource-group ${{ vars.AZURE_RESOURCE_GROUP_NAME }} --resource ${{ steps.get_cluster_id.outputs.cluster_id }} --workspace ${{ steps.create_analytics_workspace.outputs.workspace_id }} --logs "[{\"category\": \"kube-apiserver\", \"enabled\": true}, {\"category\": \"kube-controller-manager\", \"enabled\": true}, {\"category\": \"kube-scheduler\", \"enabled\": true}, {\"category\": \"cluster-autoscaler\", \"enabled\": true}]"
+
+      - name: Define trusted container registries
+        run: |
+          az policy assignment create --resource-group ${{ vars.AZURE_RESOURCE_GROUP_NAME }} --name "EnforceTrustedRegistries" --policy "febd0533-8e55-448f-b837-bd0e06f16469" --params "{\"allowedContainerImagesRegex\":{\"value\":\"learninghubmoodlecrprod.azurecr.io\"}}" --location northeurope
diff --git a/Terraform/prod/backend-prod.tfvars b/Terraform/prod/backend-prod.tfvars
@@ -1,3 +1,2 @@
 key="learninghub-moodle.prod.terraform.tfstate"
-storage_account_name="learninghubmoodleprod"
-resource_group_name     = "LearningHub-Prod-Stor-RG"
+storage_account_name="userprofilesa11"
diff --git a/Terraform/prod/main.tf b/Terraform/prod/main.tf
@@ -23,16 +23,25 @@ resource "azurerm_storage_share" "storage_share_theme" {
   quota                = var.StorageQuota
 }
 
+resource "azurerm_storage_container" "assessment_container" {
+  name                  = "assessmentstoragecontainer"
+  storage_account_name  = azurerm_storage_account.storage_account.name
+  container_access_type = "private"
+}
+
 resource "azurerm_kubernetes_cluster" "aks" {
   name                = var.ClusterName
   location            = azurerm_resource_group.learningHubMoodleResourceGroup.location
   resource_group_name = azurerm_resource_group.learningHubMoodleResourceGroup.name
   dns_prefix          = var.ClusterName
   default_node_pool {
-    name       = "default"
-    node_count = var.ClusterNodeCount
-    vm_size    = var.ClusterNodeSize
-    temporary_name_for_rotation = "tmpnodepool1"
+    name                         = "default"
+    vm_size                      = var.ClusterNodeSize
+    temporary_name_for_rotation  = "tmpnodepool1"
+    auto_scaling_enabled         = true
+    min_count                    = 2
+    max_count                    = 3
+    only_critical_addons_enabled = true
   }
   identity {
     type = "SystemAssigned"
@@ -45,6 +54,33 @@ resource "azurerm_kubernetes_cluster" "aks" {
   }
 }
 
+provider "kubernetes" {
+  host                   = azurerm_kubernetes_cluster.aks.kube_config.0.host
+  client_certificate     = base64decode(azurerm_kubernetes_cluster.aks.kube_config.0.client_certificate)
+  client_key             = base64decode(azurerm_kubernetes_cluster.aks.kube_config.0.client_key)
+  cluster_ca_certificate = base64decode(azurerm_kubernetes_cluster.aks.kube_config.0.cluster_ca_certificate)
+}
+
+resource "kubernetes_namespace" "custom_namespace" {
+  metadata {
+    name = "learninghubmoodle"
+  }
+}
+
+resource "azurerm_kubernetes_cluster_node_pool" "user_node_pool" {
+  name                  = "userpool"
+  kubernetes_cluster_id = azurerm_kubernetes_cluster.aks.id
+  vm_size               = var.ClusterNodeSize
+  auto_scaling_enabled  = true
+  min_count             = 1
+  max_count             = 20
+  node_count            = var.ClusterNodeCount
+  mode                  = "User"
+  tags = {
+    Environment = var.Environment
+  }
+}
+
 resource "azurerm_container_registry" "containerRegistry" {
   name                = var.ContainerRegistryName
   resource_group_name = azurerm_resource_group.learningHubMoodleResourceGroup.name
@@ -259,6 +295,12 @@ resource "azurerm_mssql_managed_instance" "sqlmi" {
   tags = {
     environment = var.Environment
   }
+  identity {
+    type = "SystemAssigned"
+  }
+  lifecycle {
+    prevent_destroy = true
+  }
 }
 
 resource "azurerm_mssql_managed_database" "sqldb" {
@@ -273,7 +315,7 @@ resource "azurerm_redis_cache" "moodle_cache" {
   capacity            = 2
   family              = "C"
   sku_name            = "Standard"
-  non_ssl_port_enabled = true
+  non_ssl_port_enabled = false
   minimum_tls_version = "1.2"
 }
 

diff --git a/kubectl/deployment-prod.yml b/kubectl/deployment-prod.yml
@@ -2,6 +2,7 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: learninghubmoodleprod
+  namespace: learninghubmoodle
 spec:
   replicas: 5
   selector:

diff --git a/kubectl/pv-definition-prod.yml b/kubectl/pv-definition-prod.yml
@@ -18,7 +18,7 @@ spec:
       shareName: moodledata
     nodeStageSecretRef:
       name: azure-secret
-      namespace: default
+      namespace: learninghubmoodle
   mountOptions:
     - dir_mode=0777
     - file_mode=0777

diff --git a/kubectl/pv-definition-theme-prod.yml b/kubectl/pv-definition-theme-prod.yml
@@ -16,9 +16,10 @@ spec:
     volumeHandle: "moodleCluster#learninghubmoodleprod#moodletheme"  # make sure this volumeid is unique for every identical share in the cluster
     volumeAttributes:
       shareName: moodletheme
+      server: learninghubmoodletest.privatelink.file.core.windows.net
     nodeStageSecretRef:
       name: azure-secret
-      namespace: default
+      namespace: learninghubmoodle
   mountOptions:
     - dir_mode=0777
     - file_mode=0777

diff --git a/kubectl/pvc-definition-prod.yml b/kubectl/pvc-definition-prod.yml
@@ -2,6 +2,7 @@ apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
   name: moodledataclaim
+  namespace: learninghubmoodle
 spec:
   accessModes:
     - ReadWriteMany

diff --git a/kubectl/pvc-definition-theme-prod.yml b/kubectl/pvc-definition-theme-prod.yml
@@ -2,6 +2,7 @@ apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
   name: moodlethemeclaim
+  namespace: learninghubmoodle
 spec:
   accessModes:
     - ReadWriteMany

diff --git a/kubectl/service-prod.yml b/kubectl/service-prod.yml
@@ -2,6 +2,7 @@ apiVersion: v1
 kind: Service
 metadata:
   name: learninghubmoodle-service
+  namespace: learninghubmoodle
 spec:
   selector:
     app: learninghubmoodleprod