feat: windows cse ensure oras and reserve some exit code for network isolated cluster (#7995)

Author: fseldow

parts/windows/kuberneteswindowssetup.ps1

@@ -221,8 +221,13 @@ $global:WindowsCiliumNetworkingPath = Join-Path -Path $global:cacheDir -ChildPat
 $global:WindowsCiliumInstallPath = Join-Path -Path $global:WindowsCiliumNetworkingPath -ChildPath 'install'
 
 # Network isolated cluster
-$global:BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER="{{GetBootstrapProfileContainerRegistryServer}}"
-$global:MCR_REPOSITORY_BASE="{{GetMCRRepositoryBase}}"
+$global:BootstrapProfileContainerRegistryServer="{{GetBootstrapProfileContainerRegistryServer}}"
+$global:MCRRepositoryBase="{{GetMCRRepositoryBase}}"
+
+$global:OrasCacheDir="c:\aks-tools\oras\" # refer to components.json
+$global:OrasPath="c:\aks-tools\oras\oras.exe"
+$global:OrasOutput="c:\aks-tools\oras\oras_verbose.out"
+$global:OrasRegistryConfigFile="c:\aks-tools\oras\config.yaml" # oras registry auth config file, not used, but have to define to avoid error "Error: failed to get user home directory: $HOME is not defined"
 
 # Extract cse helper script from ZIP
 [io.file]::WriteAllBytes("scripts.zip", [System.Convert]::FromBase64String($zippedFiles))
@@ -291,6 +296,12 @@ if (Test-Path -Path 'c:\AzureData\windows\windowsciliumnetworkingfunc.ps1') {
     Write-Log "Windows Cilium Networking function script not found, skipping dot-source"
 }
 
+if (Test-Path -Path 'c:\AzureData\windows\networkisolatedclusterfunc.ps1') {
+    . c:\AzureData\windows\networkisolatedclusterfunc.ps1
+} else {
+    Write-Log "Network Isolated Cluster function script not found, skipping dot-source"
+}
+
 # ====== BASE PREP: BASE IMAGE PREPARATION ======
 # All operations that prepare the base VHD image
 function BasePrep {
@@ -333,6 +344,18 @@ function BasePrep {
 
     Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp
 
+    # oras initialization, including install and login, must be in front of Install-CredentialProvider, Get-KubePackage and Install-Containerd-Based-On-Kubernetes-Version
+    if ((Test-Path variable:global:BootstrapProfileContainerRegistryServer) -and
+    -not [string]::IsNullOrWhiteSpace($global:BootstrapProfileContainerRegistryServer)) {
+    # variable exists and is not empty/whitespace
+        if (Get-Command -Name Initialize-Oras -ErrorAction SilentlyContinue) {
+            Logs-To-Event -TaskName "AKS.WindowsCSE.InitializeOras" -TaskMessage "Ensure oras is initialized for network isolated cluster"
+            Initialize-Oras
+        } else {
+            Write-Log "Initialize-Oras is not a recognized function, will skip oras initialization for network isolated cluster"
+        }
+    }
+
     # to ensure we don't introduce any incompatibility between base CSE + CSE package versions
     if (Get-Command -Name Install-SecureTLSBootstrapClient -ErrorAction SilentlyContinue) {
         Install-SecureTLSBootstrapClient -KubeDir $global:KubeDir -CustomSecureTLSBootstrapClientDownloadUrl $global:CustomSecureTLSBootstrappingClientDownloadURL

parts/windows/windowscsehelper.ps1

@@ -79,9 +79,16 @@ $global:WINDOWS_CSE_ERROR_WINDOWS_CILIUM_NETWORKING_INSTALL_FAILED=72
 $global:WINDOWS_CSE_ERROR_EXTRACT_ZIP=73
 $global:WINDOWS_CSE_ERROR_LOAD_METADATA=74
 $global:WINDOWS_CSE_ERROR_PARSE_METADATA=75
+$global:WINDOWS_CSE_ERROR_ORAS_NOT_FOUND=76 # exit code for not finding oras in the expected path, which is a prerequisite for pulling packages from registry for network isolated cluster
+$global:WINDOWS_CSE_ERROR_ORAS_IMDS_TIMEOUT=77 # exit code for timeout waiting for IMDS response
+$global:WINDOWS_CSE_ERROR_ORAS_PULL_NETWORK_TIMEOUT=78 # exit code for error pulling oras when login
+$global:WINDOWS_CSE_ERROR_ORAS_PULL_UNAUTHORIZED=79 # exit code for error pulling artifact with oras from registry with authorization issue
+$global:WINDOWS_CSE_ERROR_ORAS_PULL_WINDOWSZIP_FAIL=80 # exit code for error pulling kubelet kubectl artifact with oras from registry
+$global:WINDOWS_CSE_ERROR_ORAS_PULL_CREDENTIAL_PROVIDER=81 # exit code for error pulling credential provider artifact with oras from registry
+$global:WINDOWS_CSE_ERROR_ORAS_PULL_POD_INFRA_CONTAINER=82 # exit code for error pulling pause image with oras from registry
 # WINDOWS_CSE_ERROR_MAX_CODE is only used in unit tests to verify whether new error code name is added in $global:ErrorCodeNames
 # Please use the current value of WINDOWS_CSE_ERROR_MAX_CODE as the value of the new error code and increment it by 1
-$global:WINDOWS_CSE_ERROR_MAX_CODE=76
+$global:WINDOWS_CSE_ERROR_MAX_CODE=83
 
 # Please add new error code for downloading new packages in RP code too
 $global:ErrorCodeNames = @(
@@ -160,7 +167,14 @@ $global:ErrorCodeNames = @(
     "WINDOWS_CSE_ERROR_WINDOWS_CILIUM_NETWORKING_INSTALL_FAILED",
     "WINDOWS_CSE_ERROR_EXTRACT_ZIP",
     "WINDOWS_CSE_ERROR_LOAD_METADATA",
-    "WINDOWS_CSE_ERROR_PARSE_METADATA"
+    "WINDOWS_CSE_ERROR_PARSE_METADATA",
+    "WINDOWS_CSE_ERROR_ORAS_NOT_FOUND",
+    "WINDOWS_CSE_ERROR_ORAS_IMDS_TIMEOUT",
+    "WINDOWS_CSE_ERROR_ORAS_PULL_NETWORK_TIMEOUT",
+    "WINDOWS_CSE_ERROR_ORAS_PULL_UNAUTHORIZED",
+    "WINDOWS_CSE_ERROR_ORAS_PULL_WINDOWSZIP_FAIL",
+    "WINDOWS_CSE_ERROR_ORAS_PULL_CREDENTIAL_PROVIDER",
+    "WINDOWS_CSE_ERROR_ORAS_PULL_POD_INFRA_CONTAINER"
 )
 
 # The package domain to be used
@@ -635,3 +649,4 @@ function Resolve-Error ($ErrorRecord=$Error[0])
        $Exception |Format-List * -Force
    }
 }
+