diff --git a/.pipelines/e2e-gpu.yaml b/.pipelines/e2e-gpu.yaml new file mode 100644 index 00000000000..aa35693fcf9 --- /dev/null +++ b/.pipelines/e2e-gpu.yaml @@ -0,0 +1,40 @@ +name: $(Date:yyyyMMdd)$(Rev:.r) +variables: + TAGS_TO_RUN: "gpu=true" + TAGS_TO_SKIP: "os=windows" + SKIP_E2E_TESTS: false +trigger: + branches: + include: + - main +pr: + branches: + include: + - official/* + - windows/* + - main + paths: + include: + - .pipelines/e2e.yaml + - .pipelines/templates/e2e-template.yaml + - .pipelines/scripts/e2e_run.sh + - e2e + - parts/linux + - parts/common/components.json + - pkg/agent + - go.mod + - go.sum + exclude: + - pkg/agent/datamodel/sig_config*.go # SIG config changes + - pkg/agent/datamodel/*.json # SIG version changes + - pkg/agent/testdata/AKSWindows* # Windows test data + - parts/common/components.json # centralized components management file + - staging/cse/windows/README + - /**/*.md + - e2e/scenario_win_test.go + +jobs: + - template: ./templates/e2e-template.yaml + parameters: + name: Linux GPU Tests + IgnoreScenariosWithMissingVhd: false diff --git a/.pipelines/e2e.yaml b/.pipelines/e2e.yaml index ba4a24dd1dd..cefa2e00142 100644 --- a/.pipelines/e2e.yaml +++ b/.pipelines/e2e.yaml @@ -1,6 +1,6 @@ name: $(Date:yyyyMMdd)$(Rev:.r) variables: - TAGS_TO_SKIP: "os=windows" + TAGS_TO_SKIP: "os=windows,gpu=true" SKIP_E2E_TESTS: false trigger: branches: diff --git a/.pipelines/scripts/e2e_run.sh b/.pipelines/scripts/e2e_run.sh index bf19ba1bf31..249a7cd860f 100644 --- a/.pipelines/scripts/e2e_run.sh +++ b/.pipelines/scripts/e2e_run.sh @@ -64,16 +64,37 @@ if [ -n "${SIG_GALLERY_NAME}" ]; then export GALLERY_NAME=$SIG_GALLERY_NAME fi +az extension add --name bastion + # this software is used to take the output of "go test" and produce a junit report that we can upload to the pipeline # and see fancy test results. cd e2e mkdir -p bin -GOBIN=`pwd`/bin/ go install gotest.tools/gotestsum@latest +architecture=$(uname -m) + +case "$architecture" in + x86_64 | amd64) architecture="amd64" ;; + aarch64 | arm64) architecture="arm64" ;; + *) + echo "Unsupported architecture: $architecture" + exit 1 + ;; +esac + +gotestsum_version="1.13.0" +gotestsum_archive="gotestsum_${gotestsum_version}_linux_${architecture}.tar.gz" +gotestsum_url="https://github.com/gotestyourself/gotestsum/releases/download/v${gotestsum_version}/${gotestsum_archive}" + +temp_file="$(mktemp)" +curl -fsSL "$gotestsum_url" -o "$temp_file" +tar -xzf "$temp_file" -C bin +chmod +x bin/gotestsum +rm -f "$temp_file" # gotestsum configure to only show logs for failed tests, json file for detailed logs # Run the tests! Yey! test_exit_code=0 -./bin/gotestsum --format testdox --junitfile "${BUILD_SRC_DIR}/e2e/report.xml" --jsonfile "${BUILD_SRC_DIR}/e2e/test-log.json" -- -parallel 100 -timeout 90m || test_exit_code=$? +./bin/gotestsum --format testdox --junitfile "${BUILD_SRC_DIR}/e2e/report.xml" --jsonfile "${BUILD_SRC_DIR}/e2e/test-log.json" -- -parallel 150 -timeout 90m || test_exit_code=$? # Upload test results as Azure DevOps artifacts echo "##vso[artifact.upload containerfolder=test-results;artifactname=e2e-test-log]${BUILD_SRC_DIR}/e2e/test-log.json" diff --git a/.pipelines/templates/e2e-template.yaml b/.pipelines/templates/e2e-template.yaml index 21231672f04..08cebe39538 100644 --- a/.pipelines/templates/e2e-template.yaml +++ b/.pipelines/templates/e2e-template.yaml @@ -34,6 +34,8 @@ jobs: displayName: Run AgentBaker E2E env: E2E_SUBSCRIPTION_ID: $(E2E_SUBSCRIPTION_ID) + SYS_SSH_PUBLIC_KEY: $(SYS_SSH_PUBLIC_KEY) + SYS_SSH_PRIVATE_KEY_B64: $(SYS_SSH_PRIVATE_KEY_B64) BUILD_SRC_DIR: $(System.DefaultWorkingDirectory) DefaultWorkingDirectory: $(Build.SourcesDirectory) VHD_BUILD_ID: $(VHD_BUILD_ID) diff --git a/aks-node-controller/README.md b/aks-node-controller/README.md index f528b5e3820..49d0fa4ef9f 100644 --- a/aks-node-controller/README.md +++ b/aks-node-controller/README.md @@ -109,15 +109,21 @@ Clients need to provide CSE and Custom Data. [nodeconfigutils](pkg/nodeconfiguti ```mermaid sequenceDiagram participant Client as Client - participant ARM as Azure Resource Manager (ARM) - participant VM as Virtual Machine (VM) + participant AgentBaker as Versioned AgentBaker Services
(Deprecated) + participant ARM as Azure Resource Manager
(ARM) + participant VM as Virtual Machine
(VM) - Client->>ARM: Request to create VM
with CustomData & CSE + Client -x AgentBaker: ~~Request artifacts for
node provisioning~~ (deprecated) + note over Client, AgentBaker: Scriptless no longer needs the 26+ absvc pods.
Instead it uses one AgentBaker service that keeps
providing the latest SIG images list (not shown). + + AgentBaker-->>Client: ~~Provide "CSE command
& provisioning scripts"~~ (deprecated) + + Client->>ARM: Request to create VM
with CustomData & CSE
(using AgentBaker artifacts) ARM->>VM: Deploy config.json
(CustomData) note over VM: cloud-init handles
config.json deployment note over VM: cloud-init completes processing - note over VM: Start aks-node-controller.service (systemd service)
after cloud-init + note over VM: Start aks-node-controller.service (systemd service)
after cloud-init VM->>VM: Run aks-node-controller
(Go binary) in provision mode
using config.json ARM->>VM: Initiate aks-node-controller (Go binary)
in provision-wait mode via CSE @@ -126,7 +132,7 @@ sequenceDiagram VM->>VM: Check /opt/azure/containers/provision.complete end - VM->>Client: Return CSE status with
/var/log/azure/aks/provision.json content + VM-->>Client: Return CSE status with
/var/log/azure/aks/provision.json content ``` Key components: diff --git a/aks-node-controller/app.go b/aks-node-controller/app.go index 940d45dd103..22fdae3a334 100644 --- a/aks-node-controller/app.go +++ b/aks-node-controller/app.go @@ -43,11 +43,11 @@ type ProvisionStatusFiles struct { } func (a *App) Run(ctx context.Context, args []string) int { - slog.Info("aks-node-controller started") + slog.Info("aks-node-controller started", "args", args) err := a.run(ctx, args) exitCode := errToExitCode(err) if exitCode == 0 { - slog.Info("aks-node-controller finished successfully") + slog.Info("aks-node-controller finished successfully.") } else { slog.Error("aks-node-controller failed", "error", err) } diff --git a/aks-node-controller/go.mod b/aks-node-controller/go.mod index c8a2098da8c..b2789565991 100644 --- a/aks-node-controller/go.mod +++ b/aks-node-controller/go.mod @@ -1,6 +1,6 @@ module github.com/Azure/agentbaker/aks-node-controller -go 1.23.7 +go 1.24.0 require ( github.com/Azure/agentbaker v0.20240503.0 @@ -28,7 +28,7 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/vincent-petithory/dataurl v1.0.0 // indirect - golang.org/x/sys v0.35.0 // indirect + golang.org/x/sys v0.39.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/aks-node-controller/go.sum b/aks-node-controller/go.sum index c053db067b9..516e50f3e17 100644 --- a/aks-node-controller/go.sum +++ b/aks-node-controller/go.sum @@ -55,12 +55,12 @@ github.com/vincent-petithory/dataurl v1.0.0 h1:cXw+kPto8NLuJtlMsI152irrVw9fRDX8A github.com/vincent-petithory/dataurl v1.0.0/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= -golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= -golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= -golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= -golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= +golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= +golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/e2e/aks_model.go b/e2e/aks_model.go index b90ebc026b4..3cf335e232a 100644 --- a/e2e/aks_model.go +++ b/e2e/aks_model.go @@ -13,9 +13,9 @@ import ( "github.com/Azure/agentbaker/pkg/agent" "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v6" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry/v2" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v8" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v7" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns" ) @@ -175,6 +175,16 @@ func getBaseClusterModel(clusterName, location, k8sSystemPoolSKU string) *armcon Enabled: to.Ptr(false), }, }, + LinuxProfile: &armcontainerservice.LinuxProfile{ + AdminUsername: to.Ptr("azureuser"), + SSH: &armcontainerservice.SSHConfiguration{ + PublicKeys: []*armcontainerservice.SSHPublicKey{ + { + KeyData: to.Ptr(string(config.SysSSHPublicKey)), + }, + }, + }, + }, }, Identity: &armcontainerservice.ManagedClusterIdentity{ Type: to.Ptr(armcontainerservice.ResourceIdentityTypeSystemAssigned), @@ -275,6 +285,17 @@ func addFirewallRules( ctx context.Context, clusterModel *armcontainerservice.ManagedCluster, location string, ) error { + routeTableName := "abe2e-fw-rt" + rtGetResp, err := config.Azure.RouteTables.Get( + ctx, + *clusterModel.Properties.NodeResourceGroup, + routeTableName, + nil, + ) + if err == nil && len(rtGetResp.Properties.Subnets) != 0 { + // already associated with aks subnet + return nil + } vnet, err := getClusterVNet(ctx, *clusterModel.Properties.NodeResourceGroup) if err != nil { @@ -366,7 +387,6 @@ func addFirewallRules( return fmt.Errorf("failed to get firewall private IP address") } - routeTableName := "abe2e-fw-rt" routeTableParams := armnetwork.RouteTable{ Location: to.Ptr(location), Properties: &armnetwork.RouteTablePropertiesFormat{ @@ -535,26 +555,16 @@ func airGapSecurityGroup(location, clusterFQDN string) (armnetwork.SecurityGroup func addPrivateEndpointForACR(ctx context.Context, nodeResourceGroup, privateACRName string, vnet VNet, location string) error { logf(ctx, "Checking if private endpoint for private container registry is in rg %s", nodeResourceGroup) - var err error - var exists bool + var privateEndpoint *armnetwork.PrivateEndpoint privateEndpointName := "PE-for-ABE2ETests" - if exists, err = privateEndpointExists(ctx, nodeResourceGroup, privateEndpointName); err != nil { - return err - } - if exists { - logf(ctx, "Private Endpoint already exists, skipping creation") - return nil - } - - var peResp armnetwork.PrivateEndpointsClientCreateOrUpdateResponse - if peResp, err = createPrivateEndpoint(ctx, nodeResourceGroup, privateEndpointName, privateACRName, vnet, location); err != nil { + if privateEndpoint, err = createPrivateEndpoint(ctx, nodeResourceGroup, privateEndpointName, privateACRName, vnet, location); err != nil { return err } privateZoneName := "privatelink.azurecr.io" - var pzResp armprivatedns.PrivateZonesClientCreateOrUpdateResponse - if pzResp, err = createPrivateZone(ctx, nodeResourceGroup, privateZoneName); err != nil { + var privateZone *armprivatedns.PrivateZone + if privateZone, err = createPrivateZone(ctx, nodeResourceGroup, privateZoneName); err != nil { return err } @@ -562,28 +572,16 @@ func addPrivateEndpointForACR(ctx context.Context, nodeResourceGroup, privateACR return err } - if err = addRecordSetToPrivateDNSZone(ctx, peResp, nodeResourceGroup, privateZoneName); err != nil { + if err = addRecordSetToPrivateDNSZone(ctx, privateEndpoint, nodeResourceGroup, privateZoneName); err != nil { return err } - if err = addDNSZoneGroup(ctx, pzResp, nodeResourceGroup, privateZoneName, *peResp.Name); err != nil { + if err = addDNSZoneGroup(ctx, privateZone, nodeResourceGroup, privateZoneName, *privateEndpoint.Name); err != nil { return err } return nil } -func privateEndpointExists(ctx context.Context, nodeResourceGroup, privateEndpointName string) (bool, error) { - existingPE, err := config.Azure.PrivateEndpointClient.Get(ctx, nodeResourceGroup, privateEndpointName, nil) - if err == nil && existingPE.ID != nil { - logf(ctx, "Private Endpoint already exists with ID: %s", *existingPE.ID) - return true, nil - } - if err != nil && !strings.Contains(err.Error(), "ResourceNotFound") { - return false, fmt.Errorf("failed to get private endpoint: %w", err) - } - return false, nil -} - func createPrivateAzureContainerRegistryPullSecret(ctx context.Context, cluster *armcontainerservice.ManagedCluster, kubeconfig *Kubeclient, resourceGroup string, isNonAnonymousPull bool) error { privateACRName := config.GetPrivateACRName(isNonAnonymousPull, *cluster.Location) if isNonAnonymousPull { @@ -768,7 +766,15 @@ func addCacheRulesToPrivateAzureContainerRegistry(ctx context.Context, resourceG return nil } -func createPrivateEndpoint(ctx context.Context, nodeResourceGroup, privateEndpointName, privateACRName string, vnet VNet, location string) (armnetwork.PrivateEndpointsClientCreateOrUpdateResponse, error) { +func createPrivateEndpoint(ctx context.Context, nodeResourceGroup, privateEndpointName, privateACRName string, vnet VNet, location string) (*armnetwork.PrivateEndpoint, error) { + existingPE, err := config.Azure.PrivateEndpointClient.Get(ctx, nodeResourceGroup, privateEndpointName, nil) + if err == nil && existingPE.ID != nil { + logf(ctx, "Private Endpoint already exists with ID: %s", *existingPE.ID) + return &existingPE.PrivateEndpoint, nil + } + if err != nil && !strings.Contains(err.Error(), "ResourceNotFound") { + return nil, fmt.Errorf("failed to get private endpoint: %w", err) + } logf(ctx, "Creating Private Endpoint in rg %s", nodeResourceGroup) acrID := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.ContainerRegistry/registries/%s", config.Config.SubscriptionID, config.ResourceGroupName(location), privateACRName) @@ -798,18 +804,27 @@ func createPrivateEndpoint(ctx context.Context, nodeResourceGroup, privateEndpoi nil, ) if err != nil { - return armnetwork.PrivateEndpointsClientCreateOrUpdateResponse{}, fmt.Errorf("failed to create private endpoint in BeginCreateOrUpdate: %w", err) + return nil, fmt.Errorf("failed to create private endpoint in BeginCreateOrUpdate: %w", err) } resp, err := poller.PollUntilDone(ctx, nil) if err != nil { - return armnetwork.PrivateEndpointsClientCreateOrUpdateResponse{}, fmt.Errorf("failed to create private endpoint in polling: %w", err) + return nil, fmt.Errorf("failed to create private endpoint in polling: %w", err) } logf(ctx, "Private Endpoint created or updated with ID: %s", *resp.ID) - return resp, nil + return &resp.PrivateEndpoint, nil } -func createPrivateZone(ctx context.Context, nodeResourceGroup, privateZoneName string) (armprivatedns.PrivateZonesClientCreateOrUpdateResponse, error) { +func createPrivateZone(ctx context.Context, nodeResourceGroup, privateZoneName string) (*armprivatedns.PrivateZone, error) { + pzResp, err := config.Azure.PrivateZonesClient.Get( + ctx, + nodeResourceGroup, + privateZoneName, + nil, + ) + if err == nil { + return &pzResp.PrivateZone, nil + } dnsZoneParams := armprivatedns.PrivateZone{ Location: to.Ptr("global"), } @@ -821,23 +836,36 @@ func createPrivateZone(ctx context.Context, nodeResourceGroup, privateZoneName s nil, ) if err != nil { - return armprivatedns.PrivateZonesClientCreateOrUpdateResponse{}, fmt.Errorf("failed to create private dns zone in BeginCreateOrUpdate: %w", err) + return nil, fmt.Errorf("failed to create private dns zone in BeginCreateOrUpdate: %w", err) } resp, err := poller.PollUntilDone(ctx, nil) if err != nil { - return armprivatedns.PrivateZonesClientCreateOrUpdateResponse{}, fmt.Errorf("failed to create private dns zone in polling: %w", err) + return nil, fmt.Errorf("failed to create private dns zone in polling: %w", err) } logf(ctx, "Private DNS Zone created or updated with ID: %s", *resp.ID) - return resp, nil + return &resp.PrivateZone, nil } func createPrivateDNSLink(ctx context.Context, vnet VNet, nodeResourceGroup, privateZoneName string) error { + networkLinkName := "link-ABE2ETests" + _, err := config.Azure.VirutalNetworkLinksClient.Get( + ctx, + nodeResourceGroup, + privateZoneName, + networkLinkName, + nil, + ) + + if err == nil { + // private dns link already created + return nil + } + vnetForId, err := config.Azure.VNet.Get(ctx, nodeResourceGroup, vnet.name, nil) if err != nil { return fmt.Errorf("failed to get vnet: %w", err) } - networkLinkName := "link-ABE2ETests" linkParams := armprivatedns.VirtualNetworkLink{ Location: to.Ptr("global"), Properties: &armprivatedns.VirtualNetworkLinkProperties{ @@ -867,8 +895,8 @@ func createPrivateDNSLink(ctx context.Context, vnet VNet, nodeResourceGroup, pri return nil } -func addRecordSetToPrivateDNSZone(ctx context.Context, peResp armnetwork.PrivateEndpointsClientCreateOrUpdateResponse, nodeResourceGroup, privateZoneName string) error { - for i, dnsConfigPtr := range peResp.Properties.CustomDNSConfigs { +func addRecordSetToPrivateDNSZone(ctx context.Context, privateEndpoint *armnetwork.PrivateEndpoint, nodeResourceGroup, privateZoneName string) error { + for i, dnsConfigPtr := range privateEndpoint.Properties.CustomDNSConfigs { var ipAddresses []string if dnsConfigPtr == nil { return fmt.Errorf("CustomDNSConfigs[%d] is nil", i) @@ -876,7 +904,7 @@ func addRecordSetToPrivateDNSZone(ctx context.Context, peResp armnetwork.Private // get the ip addresses dnsConfig := *dnsConfigPtr - if dnsConfig.IPAddresses == nil || len(dnsConfig.IPAddresses) == 0 { + if len(dnsConfig.IPAddresses) == 0 { return fmt.Errorf("CustomDNSConfigs[%d].IPAddresses is nil or empty", i) } for _, ipPtr := range dnsConfig.IPAddresses { @@ -907,15 +935,19 @@ func addRecordSetToPrivateDNSZone(ctx context.Context, peResp armnetwork.Private return nil } -func addDNSZoneGroup(ctx context.Context, pzResp armprivatedns.PrivateZonesClientCreateOrUpdateResponse, nodeResourceGroup, privateZoneName, endpointName string) error { +func addDNSZoneGroup(ctx context.Context, privateZone *armprivatedns.PrivateZone, nodeResourceGroup, privateZoneName, endpointName string) error { groupName := strings.Replace(privateZoneName, ".", "-", -1) // replace . with - + _, err := config.Azure.PrivateDNSZoneGroup.Get(ctx, nodeResourceGroup, endpointName, groupName, nil) + if err == nil { + return nil + } dnsZonegroup := armnetwork.PrivateDNSZoneGroup{ Name: to.Ptr(fmt.Sprintf("%s/default", privateZoneName)), Properties: &armnetwork.PrivateDNSZoneGroupPropertiesFormat{ PrivateDNSZoneConfigs: []*armnetwork.PrivateDNSZoneConfig{{ Name: to.Ptr(groupName), Properties: &armnetwork.PrivateDNSZonePropertiesFormat{ - PrivateDNSZoneID: pzResp.ID, + PrivateDNSZoneID: privateZone.ID, }, }}, }, diff --git a/e2e/bastionssh.go b/e2e/bastionssh.go new file mode 100644 index 00000000000..e7e11735bfb --- /dev/null +++ b/e2e/bastionssh.go @@ -0,0 +1,277 @@ +package e2e + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/url" + "slices" + "strings" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/coder/websocket" + "golang.org/x/crypto/ssh" +) + +var AllowedSSHPrefixes = []string{ssh.KeyAlgoED25519, ssh.KeyAlgoRSA} + +type Bastion struct { + credential *azidentity.AzureCLICredential + subscriptionID, resourceGroupName, dnsName string + httpClient *http.Client + httpTransport *http.Transport +} + +func NewBastion(credential *azidentity.AzureCLICredential, subscriptionID, resourceGroupName, dnsName string) *Bastion { + transport := &http.Transport{ + MaxIdleConns: 100, + MaxIdleConnsPerHost: 100, + IdleConnTimeout: 30 * time.Second, + } + + return &Bastion{ + credential: credential, + subscriptionID: subscriptionID, + resourceGroupName: resourceGroupName, + dnsName: dnsName, + httpTransport: transport, + httpClient: &http.Client{ + Transport: transport, + Timeout: 30 * time.Second, + }, + } +} + +type tunnelSession struct { + bastion *Bastion + ws *websocket.Conn + session *sessionToken +} + +func (b *Bastion) NewTunnelSession(targetHost string, port uint16) (*tunnelSession, error) { + session, err := b.newSessionToken(targetHost, port) + if err != nil { + return nil, err + } + + wsUrl := fmt.Sprintf("wss://%v/webtunnelv2/%v?X-Node-Id=%v", b.dnsName, session.WebsocketToken, session.NodeID) + + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + ws, _, err := websocket.Dial(ctx, wsUrl, &websocket.DialOptions{ + CompressionMode: websocket.CompressionDisabled, + }) + cancel() + if err != nil { + return nil, err + } + + ws.SetReadLimit(32 * 1024 * 1024) + + return &tunnelSession{ + bastion: b, + ws: ws, + session: session, + }, nil +} + +type sessionToken struct { + AuthToken string `json:"authToken"` + Username string `json:"username"` + DataSource string `json:"dataSource"` + NodeID string `json:"nodeId"` + AvailableDataSources []string `json:"availableDataSources"` + WebsocketToken string `json:"websocketToken"` +} + +func (t *tunnelSession) Close() error { + _ = t.ws.Close(websocket.StatusNormalClosure, "") + + req, err := http.NewRequest("DELETE", fmt.Sprintf("https://%v/api/tokens/%v", t.bastion.dnsName, t.session.AuthToken), nil) + if err != nil { + return err + } + + req.Header.Add("X-Node-Id", t.session.NodeID) + + resp, err := t.bastion.httpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode == 404 { + return nil + } + + if resp.StatusCode != 204 { + return fmt.Errorf("unexpected status code: %v", resp.StatusCode) + } + + if t.bastion.httpTransport != nil { + t.bastion.httpTransport.CloseIdleConnections() + } + + return nil +} + +func (b *Bastion) newSessionToken(targetHost string, port uint16) (*sessionToken, error) { + + token, err := b.credential.GetToken(context.Background(), policy.TokenRequestOptions{ + Scopes: []string{fmt.Sprintf("%s/.default", cloud.AzurePublic.Services[cloud.ResourceManager].Endpoint)}, + }) + + if err != nil { + return nil, err + } + + apiUrl := fmt.Sprintf("https://%v/api/tokens", b.dnsName) + + // target_resource_id = f"/subscriptions/{get_subscription_id(cmd.cli_ctx)}/resourceGroups/{resource_group_name}/providers/Microsoft.Network/bh-hostConnect/{target_ip_address}" + data := url.Values{} + data.Set("resourceId", fmt.Sprintf("/subscriptions/%v/resourceGroups/%v/providers/Microsoft.Network/bh-hostConnect/%v", b.subscriptionID, b.resourceGroupName, targetHost)) + data.Set("protocol", "tcptunnel") + data.Set("workloadHostPort", fmt.Sprintf("%v", port)) + data.Set("aztoken", token.Token) + data.Set("hostname", targetHost) + + req, err := http.NewRequest("POST", apiUrl, strings.NewReader(data.Encode())) + if err != nil { + return nil, err + } + + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + resp, err := b.httpClient.Do(req) // TODO client settings + if err != nil { + return nil, err + } + + defer resp.Body.Close() + + if resp.StatusCode != 200 { + return nil, fmt.Errorf("error creating tunnel: %v", resp.Status) + } + + var response sessionToken + + if err := json.NewDecoder(resp.Body).Decode(&response); err != nil { + return nil, err + } + + return &response, nil +} + +func (t *tunnelSession) Pipe(conn net.Conn) error { + + defer t.Close() + defer conn.Close() + + done := make(chan error, 2) + + go func() { + for { + _, data, err := t.ws.Read(context.Background()) + if err != nil { + done <- err + return + } + + if _, err := io.Copy(conn, bytes.NewReader(data)); err != nil { + done <- err + return + } + } + }() + + go func() { + buf := make([]byte, 4096) // 4096 is copy from az cli bastion code + + for { + n, err := conn.Read(buf) + if err != nil { + done <- err + return + } + + if err := t.ws.Write(context.Background(), websocket.MessageBinary, buf[:n]); err != nil { + done <- err + return + } + } + }() + + return <-done +} + +func sshClientConfig(user string, privateKey []byte) (*ssh.ClientConfig, error) { + signer, err := ssh.ParsePrivateKey(privateKey) + if err != nil { + return nil, err + } + + return &ssh.ClientConfig{ + User: user, + Auth: []ssh.AuthMethod{ + ssh.PublicKeys(signer), + }, + HostKeyAlgorithms: AllowedSSHPrefixes, + HostKeyCallback: func(hostname string, remote net.Addr, key ssh.PublicKey) error { + if !slices.Contains(AllowedSSHPrefixes, key.Type()) { + return fmt.Errorf("unexpected host key type: %s", key.Type()) + } + return nil + }, + Timeout: 5 * time.Second, + }, nil +} + +func DialSSHOverBastion( + ctx context.Context, + bastion *Bastion, + vmPrivateIP string, + sshPrivateKey []byte, +) (*ssh.Client, error) { + + // Create Bastion tunnel session (SSH = port 22) + tunnel, err := bastion.NewTunnelSession( + vmPrivateIP, + 22, + ) + if err != nil { + return nil, err + } + + // Create in-memory connection pair + sshSide, tunnelSide := net.Pipe() + + // Start Bastion tunnel piping + go func() { + _ = tunnel.Pipe(tunnelSide) + fmt.Printf("Closed tunnel for VM IP %s\n", vmPrivateIP) + }() + + // SSH client configuration + sshConfig, err := sshClientConfig("azureuser", sshPrivateKey) + if err != nil { + return nil, err + } + + // Establish SSH over the Bastion tunnel + sshConn, chans, reqs, err := ssh.NewClientConn( + sshSide, + vmPrivateIP, + sshConfig, + ) + if err != nil { + sshSide.Close() + return nil, err + } + + return ssh.NewClient(sshConn, chans, reqs), nil +} diff --git a/e2e/cache.go b/e2e/cache.go index 26eedca5e4f..0f417fc4b7c 100644 --- a/e2e/cache.go +++ b/e2e/cache.go @@ -9,7 +9,7 @@ import ( "github.com/Azure/agentbaker/e2e/config" "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v7" ) // cachedFunc creates a thread-safe memoized version of a function. @@ -157,49 +157,49 @@ var ClusterKubenet = cachedFunc(clusterKubenet) // clusterKubenet creates a basic cluster using kubenet networking func clusterKubenet(ctx context.Context, request ClusterRequest) (*Cluster, error) { - return prepareCluster(ctx, getKubenetClusterModel("abe2e-kubenet-v3", request.Location, request.K8sSystemPoolSKU), false, false) + return prepareCluster(ctx, getKubenetClusterModel("abe2e-kubenet-v4", request.Location, request.K8sSystemPoolSKU), false, false) } var ClusterKubenetAirgap = cachedFunc(clusterKubenetAirgap) // clusterKubenetAirgap creates an airgapped kubenet cluster (no internet access) func clusterKubenetAirgap(ctx context.Context, request ClusterRequest) (*Cluster, error) { - return prepareCluster(ctx, getKubenetClusterModel("abe2e-kubenet-airgap-v2", request.Location, request.K8sSystemPoolSKU), true, false) + return prepareCluster(ctx, getKubenetClusterModel("abe2e-kubenet-airgap-v3", request.Location, request.K8sSystemPoolSKU), true, false) } var ClusterKubenetAirgapNonAnon = cachedFunc(clusterKubenetAirgapNonAnon) // clusterKubenetAirgapNonAnon creates an airgapped kubenet cluster with non-anonymous image pulls func clusterKubenetAirgapNonAnon(ctx context.Context, request ClusterRequest) (*Cluster, error) { - return prepareCluster(ctx, getKubenetClusterModel("abe2e-kubenet-nonanonpull-airgap-v2", request.Location, request.K8sSystemPoolSKU), true, true) + return prepareCluster(ctx, getKubenetClusterModel("abe2e-kubenet-nonanonpull-airgap-v3", request.Location, request.K8sSystemPoolSKU), true, true) } var ClusterAzureNetwork = cachedFunc(clusterAzureNetwork) // clusterAzureNetwork creates a cluster with Azure CNI networking func clusterAzureNetwork(ctx context.Context, request ClusterRequest) (*Cluster, error) { - return prepareCluster(ctx, getAzureNetworkClusterModel("abe2e-azure-network-v2", request.Location, request.K8sSystemPoolSKU), false, false) + return prepareCluster(ctx, getAzureNetworkClusterModel("abe2e-azure-network-v3", request.Location, request.K8sSystemPoolSKU), false, false) } var ClusterAzureOverlayNetwork = cachedFunc(clusterAzureOverlayNetwork) // clusterAzureOverlayNetwork creates a cluster with Azure CNI Overlay networking func clusterAzureOverlayNetwork(ctx context.Context, request ClusterRequest) (*Cluster, error) { - return prepareCluster(ctx, getAzureOverlayNetworkClusterModel("abe2e-azure-overlay-network-v2", request.Location, request.K8sSystemPoolSKU), false, false) + return prepareCluster(ctx, getAzureOverlayNetworkClusterModel("abe2e-azure-overlay-network-v3", request.Location, request.K8sSystemPoolSKU), false, false) } var ClusterAzureOverlayNetworkDualStack = cachedFunc(clusterAzureOverlayNetworkDualStack) // clusterAzureOverlayNetworkDualStack creates a dual-stack (IPv4+IPv6) Azure CNI Overlay cluster func clusterAzureOverlayNetworkDualStack(ctx context.Context, request ClusterRequest) (*Cluster, error) { - return prepareCluster(ctx, getAzureOverlayNetworkDualStackClusterModel("abe2e-azure-overlay-dualstack-v2", request.Location, request.K8sSystemPoolSKU), false, false) + return prepareCluster(ctx, getAzureOverlayNetworkDualStackClusterModel("abe2e-azure-overlay-dualstack-v3", request.Location, request.K8sSystemPoolSKU), false, false) } var ClusterCiliumNetwork = cachedFunc(clusterCiliumNetwork) // clusterCiliumNetwork creates a cluster with Cilium CNI networking func clusterCiliumNetwork(ctx context.Context, request ClusterRequest) (*Cluster, error) { - return prepareCluster(ctx, getCiliumNetworkClusterModel("abe2e-cilium-network-v2", request.Location, request.K8sSystemPoolSKU), false, false) + return prepareCluster(ctx, getCiliumNetworkClusterModel("abe2e-cilium-network-v3", request.Location, request.K8sSystemPoolSKU), false, false) } // isNotFoundErr checks if an error represents a "not found" response from Azure API diff --git a/e2e/cluster.go b/e2e/cluster.go index 802f05ae0d0..19dae5e2586 100644 --- a/e2e/cluster.go +++ b/e2e/cluster.go @@ -8,16 +8,18 @@ import ( "errors" "fmt" "net/http" + "net/netip" "strings" "time" "github.com/Azure/agentbaker/e2e/config" "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v6" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v3" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v7" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v8" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v7" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources/v3" "github.com/google/uuid" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -37,8 +39,7 @@ type Cluster struct { KubeletIdentity *armcontainerservice.UserAssignedIdentity SubnetID string ClusterParams *ClusterParams - Maintenance *armcontainerservice.MaintenanceConfiguration - DebugPod *corev1.Pod + Bastion *Bastion } // Returns true if the cluster is configured with Azure CNI @@ -66,7 +67,12 @@ func prepareCluster(ctx context.Context, cluster *armcontainerservice.ManagedClu return nil, fmt.Errorf("get or create cluster: %w", err) } - maintenance, err := getOrCreateMaintenanceConfiguration(ctx, cluster) + bastion, err := getOrCreateBastion(ctx, cluster) + if err != nil { + return nil, fmt.Errorf("get or create bastion: %w", err) + } + + _, err = getOrCreateMaintenanceConfiguration(ctx, cluster) if err != nil { return nil, fmt.Errorf("get or create maintenance configuration: %w", err) } @@ -130,19 +136,13 @@ func prepareCluster(ctx context.Context, cluster *armcontainerservice.ManagedClu return nil, fmt.Errorf("extracting cluster parameters: %w", err) } - hostPod, err := kube.GetHostNetworkDebugPod(ctx) - if err != nil { - return nil, fmt.Errorf("get host network debug pod: %w", err) - } - return &Cluster{ Model: cluster, Kube: kube, KubeletIdentity: kubeletIdentity, SubnetID: subnetID, - Maintenance: maintenance, ClusterParams: clusterParams, - DebugPod: hostPod, + Bastion: bastion, }, nil } @@ -455,7 +455,7 @@ func createNewMaintenanceConfiguration(ctx context.Context, cluster *armcontaine Schedule: &armcontainerservice.Schedule{ Weekly: &armcontainerservice.WeeklySchedule{ DayOfWeek: to.Ptr(armcontainerservice.WeekDayMonday), - IntervalWeeks: to.Ptr[int32](4), + IntervalWeeks: to.Ptr[int32](1), }, }, }, @@ -470,6 +470,211 @@ func createNewMaintenanceConfiguration(ctx context.Context, cluster *armcontaine return &maintenance, nil } +func getOrCreateBastion(ctx context.Context, cluster *armcontainerservice.ManagedCluster) (*Bastion, error) { + nodeRG := *cluster.Properties.NodeResourceGroup + bastionName := fmt.Sprintf("%s-bastion", *cluster.Name) + + existing, err := config.Azure.BastionHosts.Get(ctx, nodeRG, bastionName, nil) + var azErr *azcore.ResponseError + if errors.As(err, &azErr) && azErr.StatusCode == http.StatusNotFound { + return createNewBastion(ctx, cluster) + } + if err != nil { + return nil, fmt.Errorf("failed to get bastion %q in rg %q: %w", bastionName, nodeRG, err) + } + + return NewBastion(config.Azure.Credential, config.Config.SubscriptionID, nodeRG, *existing.BastionHost.Properties.DNSName), nil +} + +func createNewBastion(ctx context.Context, cluster *armcontainerservice.ManagedCluster) (*Bastion, error) { + nodeRG := *cluster.Properties.NodeResourceGroup + location := *cluster.Location + bastionName := fmt.Sprintf("%s-bastion", *cluster.Name) + publicIPName := fmt.Sprintf("%s-bastion-pip", *cluster.Name) + publicIPName = sanitizeAzureResourceName(publicIPName) + + vnet, err := getClusterVNet(ctx, nodeRG) + if err != nil { + return nil, fmt.Errorf("get cluster vnet in rg %q: %w", nodeRG, err) + } + + // Azure Bastion requires a dedicated subnet named AzureBastionSubnet. Standard SKU (required for + // native client support/tunneling) requires at least a /26. + bastionSubnetName := "AzureBastionSubnet" + bastionSubnetPrefix := "10.226.0.0/26" + if _, err := netip.ParsePrefix(bastionSubnetPrefix); err != nil { + return nil, fmt.Errorf("invalid bastion subnet prefix %q: %w", bastionSubnetPrefix, err) + } + + var bastionSubnetID string + bastionSubnet, subnetGetErr := config.Azure.Subnet.Get(ctx, nodeRG, vnet.name, bastionSubnetName, nil) + if subnetGetErr != nil { + var subnetAzErr *azcore.ResponseError + if !errors.As(subnetGetErr, &subnetAzErr) || subnetAzErr.StatusCode != http.StatusNotFound { + return nil, fmt.Errorf("get subnet %q in vnet %q rg %q: %w", bastionSubnetName, vnet.name, nodeRG, subnetGetErr) + } + + logf(ctx, "creating subnet %s in VNet %s (rg %s)", bastionSubnetName, vnet.name, nodeRG) + subnetParams := armnetwork.Subnet{ + Properties: &armnetwork.SubnetPropertiesFormat{ + AddressPrefix: to.Ptr(bastionSubnetPrefix), + }, + } + subnetPoller, err := config.Azure.Subnet.BeginCreateOrUpdate(ctx, nodeRG, vnet.name, bastionSubnetName, subnetParams, nil) + if err != nil { + return nil, fmt.Errorf("failed to start creating bastion subnet: %w", err) + } + bastionSubnet, err := subnetPoller.PollUntilDone(ctx, config.DefaultPollUntilDoneOptions) + if err != nil { + return nil, fmt.Errorf("failed to create bastion subnet: %w", err) + } + bastionSubnetID = *bastionSubnet.ID + } else { + bastionSubnetID = *bastionSubnet.ID + } + + // Public IP for Bastion + pipParams := armnetwork.PublicIPAddress{ + Location: to.Ptr(location), + SKU: &armnetwork.PublicIPAddressSKU{ + Name: to.Ptr(armnetwork.PublicIPAddressSKUNameStandard), + }, + Properties: &armnetwork.PublicIPAddressPropertiesFormat{ + PublicIPAllocationMethod: to.Ptr(armnetwork.IPAllocationMethodStatic), + }, + } + + logf(ctx, "creating bastion public IP %s (rg %s)", publicIPName, nodeRG) + pipPoller, err := config.Azure.PublicIPAddresses.BeginCreateOrUpdate(ctx, nodeRG, publicIPName, pipParams, nil) + if err != nil { + return nil, fmt.Errorf("failed to start creating bastion public IP: %w", err) + } + pipResp, err := pipPoller.PollUntilDone(ctx, config.DefaultPollUntilDoneOptions) + if err != nil { + return nil, fmt.Errorf("failed to create bastion public IP: %w", err) + } + if pipResp.ID == nil { + return nil, fmt.Errorf("bastion public IP response missing ID") + } + + bastionHost := armnetwork.BastionHost{ + Location: to.Ptr(location), + SKU: &armnetwork.SKU{ + Name: to.Ptr(armnetwork.BastionHostSKUNameStandard), + }, + Properties: &armnetwork.BastionHostPropertiesFormat{ + // Native client support is enabled via tunneling. + EnableTunneling: to.Ptr(true), + IPConfigurations: []*armnetwork.BastionHostIPConfiguration{ + { + Name: to.Ptr("bastion-ipcfg"), + Properties: &armnetwork.BastionHostIPConfigurationPropertiesFormat{ + Subnet: &armnetwork.SubResource{ + ID: to.Ptr(bastionSubnetID), + }, + PublicIPAddress: &armnetwork.SubResource{ + ID: pipResp.ID, + }, + }, + }, + }, + }, + } + + logf(ctx, "creating bastion %s (native client/tunneling enabled) in rg %s", bastionName, nodeRG) + bastionPoller, err := config.Azure.BastionHosts.BeginCreateOrUpdate(ctx, nodeRG, bastionName, bastionHost, nil) + if err != nil { + return nil, fmt.Errorf("failed to start creating bastion: %w", err) + } + resp, err := bastionPoller.PollUntilDone(ctx, config.DefaultPollUntilDoneOptions) + if err != nil { + return nil, fmt.Errorf("failed to create bastion: %w", err) + } + + bastion := NewBastion(config.Azure.Credential, config.Config.SubscriptionID, nodeRG, *resp.BastionHost.Properties.DNSName) + + if err := verifyBastion(ctx, cluster, bastion); err != nil { + return nil, fmt.Errorf("failed to verify bastion: %w", err) + } + return bastion, nil +} + +func verifyBastion(ctx context.Context, cluster *armcontainerservice.ManagedCluster, bastion *Bastion) error { + nodeRG := *cluster.Properties.NodeResourceGroup + vmssName, err := getSystemPoolVMSSName(ctx, cluster) + if err != nil { + return err + } + + var vmssVM *armcompute.VirtualMachineScaleSetVM + pager := config.Azure.VMSSVM.NewListPager(nodeRG, vmssName, nil) + if pager.More() { + page, err := pager.NextPage(ctx) + if err != nil { + return fmt.Errorf("list vmss vms for %q in rg %q: %w", vmssName, nodeRG, err) + } + if len(page.Value) > 0 { + vmssVM = page.Value[0] + } + } + + vmPrivateIP, err := getPrivateIPFromVMSSVM(ctx, nodeRG, vmssName, *vmssVM.InstanceID) + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + sshClient, err := DialSSHOverBastion(ctx, bastion, vmPrivateIP, config.SysSSHPrivateKey) + if err != nil { + return err + } + + defer sshClient.Close() + + result, err := runSSHCommandWithPrivateKeyFile(ctx, sshClient, "uname -a", false) + if err != nil { + return err + } + if strings.Contains(result.stdout, vmssName) { + return nil + } + return fmt.Errorf("Executed ssh on wrong VM, Expected %s: %s", vmssName, result.stdout) +} + +func getSystemPoolVMSSName(ctx context.Context, cluster *armcontainerservice.ManagedCluster) (string, error) { + nodeRG := *cluster.Properties.NodeResourceGroup + var systemPoolName string + for _, pool := range cluster.Properties.AgentPoolProfiles { + if strings.EqualFold(string(*pool.Mode), "System") { + systemPoolName = *pool.Name + } + } + pager := config.Azure.VMSS.NewListPager(nodeRG, nil) + if pager.More() { + page, err := pager.NextPage(ctx) + if err != nil { + return "", fmt.Errorf("list vmss in rg %q: %w", nodeRG, err) + } + for _, vmss := range page.Value { + if strings.Contains(strings.ToLower(*vmss.Name), strings.ToLower(systemPoolName)) { + return *vmss.Name, nil + } + } + } + return "", fmt.Errorf("no matching VMSS found for system pool %q in rg %q", systemPoolName, nodeRG) +} + +func sanitizeAzureResourceName(name string) string { + // Azure resource name restrictions vary by type. For our usage here (Public IP name) we just + // keep it simple and strip problematic characters. + replacer := strings.NewReplacer("/", "-", "\\", "-", ":", "-", "_", "-", " ", "-") + name = replacer.Replace(name) + name = strings.Trim(name, "-") + if len(name) > 80 { + name = name[:80] + } + return name +} + type VNet struct { name string subnetId string diff --git a/e2e/config/azure.go b/e2e/config/azure.go index ac8bdd0c01d..e21664f6e32 100644 --- a/e2e/config/azure.go +++ b/e2e/config/azure.go @@ -21,15 +21,15 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v6" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v3" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v7" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry/v2" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v8" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v7" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources/v3" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage/v3" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service" @@ -40,6 +40,7 @@ import ( type AzureClient struct { AKS *armcontainerservice.ManagedClustersClient AzureFirewall *armnetwork.AzureFirewallsClient + BastionHosts *armnetwork.BastionHostsClient Blob *azblob.Client StorageContainers *armstorage.BlobContainersClient CacheRulesClient *armcontainerregistry.CacheRulesClient @@ -156,6 +157,16 @@ func NewAzureClient() (*AzureClient, error) { return nil, fmt.Errorf("create public ip addresses client: %w", err) } + cloud.BastionHosts, err = armnetwork.NewBastionHostsClient(Config.SubscriptionID, credential, opts) + if err != nil { + return nil, fmt.Errorf("create bastion hosts client: %w", err) + } + + cloud.BastionHosts, err = armnetwork.NewBastionHostsClient(Config.SubscriptionID, credential, opts) + if err != nil { + return nil, fmt.Errorf("create bastion hosts client: %w", err) + } + cloud.RegistriesClient, err = armcontainerregistry.NewRegistriesClient(Config.SubscriptionID, credential, opts) if err != nil { return nil, fmt.Errorf("failed to create registry client: %w", err) diff --git a/e2e/config/config.go b/e2e/config/config.go index 6fdf540e7a4..c93d120f073 100644 --- a/e2e/config/config.go +++ b/e2e/config/config.go @@ -1,7 +1,12 @@ package config import ( + "crypto/ed25519" + "crypto/rand" + "encoding/base64" + "encoding/pem" "fmt" + "os" "reflect" "sort" "strings" @@ -10,6 +15,7 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" "github.com/caarlos0/env/v11" "github.com/joho/godotenv" + "golang.org/x/crypto/ssh" ) var ( @@ -20,6 +26,8 @@ var ( DefaultPollUntilDoneOptions = &runtime.PollUntilDoneOptions{ Frequency: time.Second, } + VMSSHPublicKey, VMSSHPrivateKey, SysSSHPublicKey, SysSSHPrivateKey []byte + VMSSHPrivateKeyFileName, SysSSHPrivateKeyFileName string ) func ResourceGroupName(location string) string { @@ -27,11 +35,11 @@ func ResourceGroupName(location string) string { } func PrivateACRNameNotAnon(location string) string { - return "privateace2enonanonpull" + location // will have anonymous pull enabled + return "e2eprivateacrnonanon" + location // will have anonymous pull enabled } func PrivateACRName(location string) string { - return "privateacre2e" + location // will not have anonymous pull enabled + return "e2eprivateacr" + location // will not have anonymous pull enabled } type Configuration struct { @@ -70,6 +78,8 @@ type Configuration struct { TestTimeoutCluster time.Duration `env:"TEST_TIMEOUT_CLUSTER" envDefault:"20m"` TestTimeoutVMSS time.Duration `env:"TEST_TIMEOUT_VMSS" envDefault:"17m"` WindowsAdminPassword string `env:"WINDOWS_ADMIN_PASSWORD"` + SysSSHPublicKey string `env:"SYS_SSH_PUBLIC_KEY"` + SysSSHPrivateKeyB64 string `env:"SYS_SSH_PRIVATE_KEY_B64"` } func (c *Configuration) BlobStorageAccount() string { @@ -117,6 +127,7 @@ func (c *Configuration) VMIdentityResourceID(location string) string { } func mustLoadConfig() *Configuration { + VMSSHPublicKey, VMSSHPrivateKeyFileName = mustGetNewED25519KeyPair() err := godotenv.Load(".env") if err != nil { fmt.Printf("Error loading .env file: %s\n", err) @@ -125,9 +136,97 @@ func mustLoadConfig() *Configuration { if err := env.Parse(cfg); err != nil { panic(err) } + if cfg.SysSSHPublicKey == "" { + SysSSHPublicKey = VMSSHPublicKey + } else { + SysSSHPublicKey = []byte(cfg.SysSSHPublicKey) + } + if cfg.SysSSHPrivateKeyB64 == "" { + SysSSHPrivateKeyFileName = VMSSHPrivateKeyFileName + } else { + SysSSHPrivateKey, err = base64.StdEncoding.DecodeString(cfg.SysSSHPrivateKeyB64) + if err != nil { + panic(err) + } + + SysSSHPrivateKeyFileName, err = writePrivateKeyToTempFile(SysSSHPrivateKey) + if err != nil { + panic(err) + } + } + return cfg } +func mustGetNewED25519KeyPair() ([]byte, string) { + public, privateKeyFileName, err := getNewED25519KeyPair() + if err != nil { + panic(fmt.Sprintf("failed to generate RSA key pair: %v", err)) + } + + return public, privateKeyFileName +} + +// Returns a newly generated RSA public/private key pair with the private key in PEM format. +func getNewED25519KeyPair() (publicKeyBytes []byte, privateKeyFileName string, e error) { + publicKey, privateKey, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + return nil, "", fmt.Errorf("failed to create rsa private key: %w", err) + } + + sshPubKey, err := ssh.NewPublicKey(publicKey) + if err != nil { + return nil, "", fmt.Errorf("failed to create ssh public key: %w", err) + } + + publicKeyBytes = ssh.MarshalAuthorizedKey(sshPubKey) + + // ----- PRIVATE KEY (OpenSSH format) ----- + pemBlock, err := ssh.MarshalPrivateKey(privateKey, "azureuser") + if err != nil { + return nil, "", err + } + + VMSSHPrivateKey = pem.EncodeToMemory(pemBlock) + + privateKeyFileName, err = writePrivateKeyToTempFile(VMSSHPrivateKey) + if err != nil { + return nil, "", fmt.Errorf("failed to write private key to temp file: %w", err) + } + + return +} + +func writePrivateKeyToTempFile(key []byte) (string, error) { + // Create temp file with secure permissions + tmpFile, err := os.CreateTemp("", "private-key-*") + if err != nil { + return "", err + } + + // Ensure file permissions are restricted (owner read/write only) + if err := tmpFile.Chmod(0600); err != nil { + tmpFile.Close() + os.Remove(tmpFile.Name()) + return "", err + } + + // Write key + if _, err := tmpFile.Write(key); err != nil { + tmpFile.Close() + os.Remove(tmpFile.Name()) + return "", err + } + + // Close file (important!) + if err := tmpFile.Close(); err != nil { + os.Remove(tmpFile.Name()) + return "", err + } + + return tmpFile.Name(), nil +} + func GetPrivateACRName(isNonAnonymousPull bool, location string) string { privateACRName := PrivateACRName(location) if isNonAnonymousPull { diff --git a/e2e/config/vhd.go b/e2e/config/vhd.go index e5fa6178a42..a2d4796cb6f 100644 --- a/e2e/config/vhd.go +++ b/e2e/config/vhd.go @@ -323,9 +323,7 @@ func GetRandomLinuxAMD64VHD() *Image { vhds := []*Image{ VHDUbuntu2404Gen2Containerd, VHDUbuntu2204Gen2Containerd, - VHDAzureLinuxV2Gen2, VHDAzureLinuxV3Gen2, - VHDCBLMarinerV2Gen2, } // Return a random VHD from the list diff --git a/e2e/exec.go b/e2e/exec.go index 5a37a031182..28c62bdf4c1 100644 --- a/e2e/exec.go +++ b/e2e/exec.go @@ -3,20 +3,30 @@ package e2e import ( "bytes" "context" + "crypto/rand" "fmt" + "path/filepath" + "strconv" "strings" + "sync" "time" - "github.com/Azure/agentbaker/e2e/config" - "github.com/google/uuid" + scp "github.com/bramvdbogaerde/go-scp" + "golang.org/x/crypto/ssh" corev1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/remotecommand" ) +var bufferPool = sync.Pool{ + New: func() any { + return new(bytes.Buffer) + }, +} + type podExecResult struct { exitCode string - stderr, stdout *bytes.Buffer + stderr, stdout string } func (r podExecResult) String() string { @@ -27,67 +37,114 @@ func (r podExecResult) String() string { ----------------------------------- begin stdout -----------------------------------, %s ----------------------------------- end stdout ------------------------------------ -`, r.exitCode, r.stderr.String(), r.stdout.String()) +`, r.exitCode, r.stderr, r.stdout) } -func sshKeyName(vmPrivateIP string) string { - return fmt.Sprintf("sshkey%s", strings.ReplaceAll(vmPrivateIP, ".", "")) - +func cleanupBastionTunnel(sshClient *ssh.Client) { + // We have to do this because az network tunnel creates a new detached process for tunnel + if sshClient != nil { + _ = sshClient.Close() + } } -func sshString(vmPrivateIP string) string { - return fmt.Sprintf(`ssh -i %[1]s -o PasswordAuthentication=no -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ConnectTimeout=5 azureuser@%[2]s`, sshKeyName(vmPrivateIP), vmPrivateIP) +func runSSHCommand( + ctx context.Context, + client *ssh.Client, + command string, + isWindows bool, +) (*podExecResult, error) { + return runSSHCommandWithPrivateKeyFile(ctx, client, command, isWindows) } -func quoteForBash(command string) string { - return fmt.Sprintf("'%s'", strings.ReplaceAll(command, "'", "'\"'\"'")) -} +func copyScriptToRemoteIfRequired(ctx context.Context, client *ssh.Client, command string, isWindows bool) (string, error) { + if !strings.Contains(command, "\n") && !isWindows { + return command, nil + } -func execScriptOnVm(ctx context.Context, s *Scenario, vmPrivateIP, jumpboxPodName string, script string) (*podExecResult, error) { - // Assuming uploadSSHKey has been called before this function - s.T.Helper() - /* - This works in a way that doesn't rely on the node having joined the cluster: - * We create a linux pod on a different node. - * on that pod, we create a script file containing the script passed into this method. - * Then we scp the script to the node under test. - * Then we execute the script using an interpreter (powershell or bash) based on the OS of the node. - */ - identifier := uuid.New().String() - var scriptFileName, remoteScriptFileName, interpreter string - - if s.IsWindows() { - interpreter = "powershell" - scriptFileName = fmt.Sprintf("script_file_%s.ps1", identifier) - remoteScriptFileName = fmt.Sprintf("c:/%s", scriptFileName) + randBytes := make([]byte, 16) + rand.Read(randBytes) + + var remotePath, remoteCommand string + if isWindows { + remotePath = fmt.Sprintf("c:/script_file_%x.ps1", randBytes) + remoteCommand = fmt.Sprintf("powershell %s", remotePath) } else { - interpreter = "bash" - scriptFileName = fmt.Sprintf("script_file_%s.sh", identifier) - remoteScriptFileName = scriptFileName + remotePath = filepath.Join("/home/azureuser", fmt.Sprintf("remote_script_%x.sh", randBytes)) + remoteCommand = remotePath + } + + scpClient, err := scp.NewClientBySSH(client) + if err != nil { + return "", err } + defer scpClient.Close() - steps := []string{ - "set -x", - fmt.Sprintf("echo %[1]s > %[2]s", quoteForBash(script), scriptFileName), - fmt.Sprintf("chmod 0755 %s", scriptFileName), - fmt.Sprintf(`scp -i %[1]s -o PasswordAuthentication=no -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ConnectTimeout=5 %[3]s azureuser@%[2]s:%[4]s`, sshKeyName(vmPrivateIP), vmPrivateIP, scriptFileName, remoteScriptFileName), - fmt.Sprintf("%s %s %s", sshString(vmPrivateIP), interpreter, remoteScriptFileName), + copyCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + return remoteCommand, scpClient.Copy(copyCtx, + strings.NewReader(command), + remotePath, + "0755", + int64(len(command))) +} + +func runSSHCommandWithPrivateKeyFile( + ctx context.Context, + client *ssh.Client, + command string, + isWindows bool, +) (*podExecResult, error) { + if client == nil { + return nil, fmt.Errorf("Permission denied: ssh client is nil") + } + var err error + command, err = copyScriptToRemoteIfRequired(ctx, client, command, isWindows) + if err != nil { + return nil, err } - joinedSteps := strings.Join(steps, " && ") + session, err := client.NewSession() + if err != nil { + return nil, err + } + defer session.Close() + + stdout := bufferPool.Get().(*bytes.Buffer) + stderr := bufferPool.Get().(*bytes.Buffer) + stdout.Reset() + stderr.Reset() + + defer bufferPool.Put(stdout) + defer bufferPool.Put(stderr) + session.Stdout = stdout + session.Stderr = stderr - kube := s.Runtime.Cluster.Kube - execResult, err := execOnPrivilegedPod(ctx, kube, defaultNamespace, jumpboxPodName, joinedSteps) + err = session.Run(command) + + exitCode := 0 if err != nil { - return nil, fmt.Errorf("error executing command on pod: %w", err) + if exitErr, ok := err.(*ssh.ExitError); ok { + exitCode = exitErr.ExitStatus() + } else if _, ok := err.(*ssh.ExitMissingError); ok { + // Bastion closed channel early – ignore + err = nil + } else { + return nil, err // real SSH failure + } } - return execResult, nil + return &podExecResult{ + exitCode: strconv.Itoa(exitCode), + stdout: stdout.String(), + stderr: stderr.String(), + }, nil } -func execOnPrivilegedPod(ctx context.Context, kube *Kubeclient, namespace string, podName string, bashCommand string) (*podExecResult, error) { - privilegedCommand := append(privilegedCommandArray(), bashCommand) - return execOnPod(ctx, kube, namespace, podName, privilegedCommand) +func execScriptOnVm(ctx context.Context, s *Scenario, vm *ScenarioVM, script string) (*podExecResult, error) { + s.T.Helper() + + return runSSHCommand(ctx, vm.SSHClient, script, s.IsWindows()) } func execOnUnprivilegedPod(ctx context.Context, kube *Kubeclient, namespace string, podName string, bashCommand string) (*podExecResult, error) { @@ -98,7 +155,7 @@ func execOnUnprivilegedPod(ctx context.Context, kube *Kubeclient, namespace stri // isRetryableConnectionError checks if the error is a transient connection issue that should be retried func isRetryableConnectionError(err error) bool { errorMsg := err.Error() - return strings.Contains(errorMsg, "error dialing backend: EOF") || + return strings.Contains(errorMsg, "error dialing backend") || strings.Contains(errorMsg, "connection refused") || strings.Contains(errorMsg, "dial tcp") || strings.Contains(errorMsg, "i/o timeout") || @@ -175,48 +232,14 @@ func attemptExecOnPod(ctx context.Context, kube *Kubeclient, namespace, podName return &podExecResult{ exitCode: exitCode, - stdout: &stdout, - stderr: &stderr, + stdout: stdout.String(), + stderr: stderr.String(), }, nil } -func privilegedCommandArray() []string { - return []string{ - "chroot", - "/proc/1/root", - "bash", - "-c", - } -} - func unprivilegedCommandArray() []string { return []string{ "bash", "-c", } } - -func uploadSSHKey(ctx context.Context, s *Scenario, vmPrivateIP string) error { - s.T.Helper() - steps := []string{ - fmt.Sprintf("echo '%[1]s' > %[2]s", string(SSHKeyPrivate), sshKeyName(vmPrivateIP)), - fmt.Sprintf("chmod 0600 %s", sshKeyName(vmPrivateIP)), - } - joinedSteps := strings.Join(steps, " && ") - kube := s.Runtime.Cluster.Kube - _, err := execOnPrivilegedPod(ctx, kube, defaultNamespace, s.Runtime.Cluster.DebugPod.Name, joinedSteps) - if err != nil { - return fmt.Errorf("error executing command on pod: %w", err) - } - if config.Config.KeepVMSS { - s.T.Logf("VM will be preserved after the test finishes, PLEASE MANUALLY DELETE THE VMSS. Set KEEP_VMSS=false to delete it automatically after the test finishes") - } else { - s.T.Logf("VM will be automatically deleted after the test finishes, to preserve it for debugging purposes set KEEP_VMSS=true or pause the test with a breakpoint before the test finishes or failed") - } - result := "SSH Instructions: (may take a few minutes for the VM to be ready for SSH)\n========================\n" - // We combine the az aks get credentials in the same line so we don't overwrite the user's kubeconfig. - result += fmt.Sprintf(`kubectl --kubeconfig <(az aks get-credentials --subscription "%s" --resource-group "%s" --name "%s" -f -) exec -it %s -- bash -c "chroot /proc/1/root /bin/bash -c '%s'"`, config.Config.SubscriptionID, config.ResourceGroupName(s.Location), *s.Runtime.Cluster.Model.Name, s.Runtime.Cluster.DebugPod.Name, sshString(vmPrivateIP)) - s.T.Log(result) - - return nil -} diff --git a/e2e/go.mod b/e2e/go.mod index 5398beb59d9..89ca96c9d0a 100644 --- a/e2e/go.mod +++ b/e2e/go.mod @@ -5,25 +5,27 @@ go 1.24.0 require ( github.com/Azure/agentbaker v0.20240503.0 github.com/Azure/agentbaker/aks-node-controller v0.0.0-20241215075802-f13a779d5362 - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1 - github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1 - github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2 v2.2.0 - github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6 v6.3.0 - github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry v1.3.0-beta.2 - github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v6 v6.4.0 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v3 v3.0.0-beta.2 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v7 v7.2.0 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry/v2 v2.0.0 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v8 v8.2.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.2.0 - github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6 v6.2.0 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v7 v7.2.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns v1.3.0 - github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 - github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources/v3 v3.0.1 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage/v3 v3.0.0 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2 github.com/blang/semver v3.5.1+incompatible + github.com/bramvdbogaerde/go-scp v1.6.0 github.com/caarlos0/env/v11 v11.3.1 + github.com/coder/websocket v1.8.14 github.com/joho/godotenv v1.5.1 github.com/sanity-io/litter v1.5.5 github.com/stretchr/testify v1.11.1 github.com/tidwall/gjson v1.18.0 - golang.org/x/crypto v0.45.0 + golang.org/x/crypto v0.46.0 k8s.io/api v0.34.2 k8s.io/apimachinery v0.34.2 k8s.io/client-go v0.34.2 @@ -55,8 +57,8 @@ require ( ) require ( - github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect - github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect github.com/Masterminds/semver v1.5.0 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.12.2 // indirect @@ -67,7 +69,7 @@ require ( github.com/go-openapi/jsonreference v0.21.0 // indirect github.com/go-openapi/swag v0.23.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang-jwt/jwt/v5 v5.2.3 // indirect + github.com/golang-jwt/jwt/v5 v5.3.0 // indirect github.com/google/gnostic-models v0.7.0 // indirect github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect @@ -87,11 +89,11 @@ require ( github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect github.com/x448/float16 v0.8.4 // indirect - golang.org/x/net v0.47.0 // indirect + golang.org/x/net v0.48.0 // indirect golang.org/x/oauth2 v0.30.0 // indirect - golang.org/x/sys v0.38.0 // indirect - golang.org/x/term v0.37.0 // indirect - golang.org/x/text v0.31.0 // indirect + golang.org/x/sys v0.39.0 // indirect + golang.org/x/term v0.38.0 // indirect + golang.org/x/text v0.32.0 // indirect golang.org/x/time v0.12.0 // indirect google.golang.org/protobuf v1.36.6 // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/e2e/go.sum b/e2e/go.sum index 04a8cbb73c7..4cb8f289179 100644 --- a/e2e/go.sum +++ b/e2e/go.sum @@ -1,37 +1,37 @@ -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1 h1:Wc1ml6QlJs2BHQ/9Bqu1jiyggbsSjramq2oUmp5WeIo= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1/go.mod h1:Ot/6aikWnKWi4l9QB7qVSwa8iMphQNqkWALMoNT3rzM= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1 h1:B+blDbyVIG3WaikNxPnhPiJ1MThR03b3vKGtER95TP4= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1/go.mod h1:JdM5psgjfBf5fo2uWOZhflPWyDBZ/O/CNAH9CtsuZE4= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 h1:JXg2dwJUmPB9JmtVmdEB16APJ7jurfbY5jnfXpJoRMc= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0/go.mod h1:YD5h/ldMsG0XiIw7PdyNhLxaM317eFh5yNLccNfGdyw= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1/go.mod h1:IYus9qsFobWIc2YVwe/WPjcnyCkPKtnHAqUYeebc8z0= github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY= github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 h1:FPKJS1T+clwv+OLGt13a8UjqeRuh0O4SJ3lUriThc+4= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1/go.mod h1:j2chePtV91HrC22tGoRX3sGY42uF13WzmmV80/OdVAA= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2 v2.2.0 h1:Hp+EScFOu9HeCbeW8WU2yQPJd4gGwhMgKxWe+G6jNzw= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2 v2.2.0/go.mod h1:/pz8dyNQe+Ey3yBp/XuYz7oqX8YDNWVpPB0hH3XWfbc= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6 v6.3.0 h1:Dc9miZr1Mhaqbb3cmJCRokkG16uk8JKkqOADf084zy4= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6 v6.3.0/go.mod h1:CHo9QYhWEvrKVeXsEMJSl2bpmYYNu6aG12JsSaFBXlY= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry v1.3.0-beta.2 h1:aPGtEjUanJvEPn+TBf+DXtbPKqTq0neulbRal05C6mM= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry v1.3.0-beta.2/go.mod h1:ceW/VisQm9LRUTNhEav2Nfvcx3rUexASkoF1L+hsQA8= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v5 v5.0.0 h1:5n7dPVqsWfVKw+ZiEKSd3Kzu7gwBkbEBkeXb8rgaE9Q= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v5 v5.0.0/go.mod h1:HcZY0PHPo/7d75p99lB6lK0qYOP4vLRJUBpiehYXtLQ= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v6 v6.4.0 h1:Fq2CrvgmaYGTAL4LdKF/rmGCMXb2n/61LwMVOlHj5Dc= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v6 v6.4.0/go.mod h1:jEpP2jjzNDVWS0Aay8nyoyVIK/MQBSX2NQv6r9FcVMk= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0 h1:PTFGRSlMKCQelWwxUyYVEUqseBJVemLyqWJjvMyt0do= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0/go.mod h1:LRr2FzBTQlONPPa5HREE5+RjSCTXl7BwOvYOaWTqCaI= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.1.0 h1:2qsIIvxVT+uE6yrNldntJKlLRgxGbZ85kgtz5SNBhMw= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.1.0/go.mod h1:AW8VEadnhw9xox+VaVd9sP7NjzOAnaZBLRH6Tq3cJ38= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/managementgroups/armmanagementgroups v1.0.0 h1:pPvTJ1dY0sA35JOeFq6TsY2xj6Z85Yo23Pj4wCCvu4o= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/managementgroups/armmanagementgroups v1.0.0/go.mod h1:mLfWfj8v3jfWKsL9G4eoBoXVcsqcIUTapmdKy7uGOp0= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 h1:9iefClla7iYpfYWdzPCRDozdmndjTm8DXdpCzPajMgA= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2/go.mod h1:XtLgD3ZD34DAaVIIAyG3objl5DynM3CQ/vMcbBNJZGI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v3 v3.0.0-beta.2 h1:qiir/pptnHqp6hV8QwV+IExYIf6cPsXBfUDUXQ27t2Y= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v3 v3.0.0-beta.2/go.mod h1:jVRrRDLCOuif95HDYC23ADTMlvahB7tMdl519m9Iyjc= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v7 v7.2.0 h1:XQ+/r6WORQ1Gmz0z0XTJixAbuOxSQvPpNlcPgziXPis= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v7 v7.2.0/go.mod h1:3WoHXiNq+/VSiljks+B3s0y3qwxyASJpSozY0zlDmgA= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry/v2 v2.0.0 h1:1a20YdnQEjzrrKfAXXMY8pKFvVkIDQqHGryKqC0dnuk= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry/v2 v2.0.0/go.mod h1:BVagqxlJtc2zcpd4VenwKpC/ADV23xH34AxBfChVXcc= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v8 v8.2.0 h1:aXzpyYcHexm3eSlvy6g7r3cshXtGcEg6VJpOdrN0Us0= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v8 v8.2.0/go.mod h1:vs/o7so4c3csg/CM0LDrqxSKDxcKgeYbgI3zaL6vu7U= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.1.1 h1:1kpY4qe+BGAH2ykv4baVSqyx+AY5VjXeJ15SldlU6hs= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.1.1/go.mod h1:nT6cWpWdUt+g81yuKmjeYPUtI73Ak3yQIT4PVVsCEEQ= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.2.0 h1:z4YeiSXxnUI+PqB46Yj6MZA3nwb1CcJIkEMDrzUd8Cs= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.2.0/go.mod h1:rko9SzMxcMk0NJsNAxALEGaTYyy79bNRwxgJfrH0Spw= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6 v6.2.0 h1:HYGD75g0bQ3VO/Omedm54v4LrD3B1cGImuRF3AJ5wLo= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6 v6.2.0/go.mod h1:ulHyBFJOI0ONiRL4vcJTmS7rx18jQQlEPmAgo80cRdM= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v7 v7.2.0 h1:DgqO2jYgDEqmN8W5sPP+ZU7Tfxyn+i9RqXtNsX6Enb8= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v7 v7.2.0/go.mod h1:FBChJszHNRdH5AYJ+Y/NgWilJihKa5WcSlFrNnj2eY0= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns v1.3.0 h1:yzrctSl9GMIQ5lHu7jc8olOsGjWDCsBpJhWqfGa/YIM= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns v1.3.0/go.mod h1:GE4m0rnnfwLGX0Y9A9A25Zx5N/90jneT5ABevqzhuFQ= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armdeployments v0.2.0 h1:bYq3jfB2x36hslKMHyge3+esWzROtJNk/4dCjsKlrl4= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armdeployments v0.2.0/go.mod h1:fewgRjNVE84QVVh798sIMFb7gPXPp7NmnekGnboSnXk= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 h1:Dd+RhdJn0OTtVGaeDLZpcumkIVCtA/3/Fo42+eoYvVM= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0/go.mod h1:5kakwfW5CjC9KK+Q4wjXAg+ShuIm2mBMua0ZFj2C8PE= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources/v3 v3.0.1 h1:guyQA4b8XB2sbJZXzUnOF9mn0WDBv/ZT7me9wTipKtE= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources/v3 v3.0.1/go.mod h1:8h8yhzh9o+0HeSIhUxYny+rEQajScrfIpNktvgYG3Q8= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage/v3 v3.0.0 h1:tqGq5xt/rNU57Eb52rf6bvrNWoKPSwLDVUQrJnF4C5U= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage/v3 v3.0.0/go.mod h1:HfDdtu9K0iFBSMMxFsHJPkAAxFWd2IUOW8HU8kEdF3Y= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2 h1:FwladfywkNirM+FZYLBR2kBz5C8Tg0fw5w5Y7meRXWI= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2/go.mod h1:vv5Ad0RrIoT1lJFdWBZwt4mB1+j+V8DUroixmKDTCdk= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= @@ -40,8 +40,8 @@ github.com/Azure/go-autorest/autorest/to v0.4.1 h1:CxNHBqdzTr7rLtdrtb5CMjJcDut+W github.com/Azure/go-autorest/autorest/to v0.4.1/go.mod h1:EtaofgU4zmtvn1zT2ARsjRFdq9vXx0YWtmElwL+GZ9M= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= -github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs= -github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= +github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs= +github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= @@ -54,12 +54,14 @@ github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df h1:GSoSVRLo github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df/go.mod h1:hiVxq5OP2bUGBRNS3Z/bt/reCLFNbdcST6gISi1fiOM= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/bramvdbogaerde/go-scp v1.6.0 h1:lDh0lUuz1dbIhJqlKLwWT7tzIRONCp1Mtx3pgQVaLQo= +github.com/bramvdbogaerde/go-scp v1.6.0/go.mod h1:on2aH5AxaFb2G0N5Vsdy6B0Ml7k9HuHSwfo1y0QzAbQ= github.com/caarlos0/env/v11 v11.3.1 h1:cArPWC15hWmEt+gWk7YBi7lEXTXCvpaSdCiZE2X5mCA= github.com/caarlos0/env/v11 v11.3.1/go.mod h1:qupehSf/Y0TUTsxKywqRt/vJjN5nz6vauiYEUUr8P4U= -github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= -github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/clarketm/json v1.17.1 h1:U1IxjqJkJ7bRK4L6dyphmoO840P6bdhPdbbLySourqI= github.com/clarketm/json v1.17.1/go.mod h1:ynr2LRfb0fQU34l07csRNBTcivjySLLiY1YzQqKVfdo= +github.com/coder/websocket v1.8.14 h1:9L0p0iKiNOibykf283eHkKUHHrpG7f65OE3BhhO7v9g= +github.com/coder/websocket v1.8.14/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg= github.com/coreos/butane v0.25.1 h1:Nm2WDRD7h3f6GUpazGlge1o417Z+eIC9bQlkpgVdNms= github.com/coreos/butane v0.25.1/go.mod h1:N5JMWID5tmPsfsp3SR9w9xQk32rru8RDHSTerQiq8vI= github.com/coreos/go-json v0.0.0-20230131223807-18775e0fb4fb h1:rmqyI19j3Z/74bIRhuC59RB442rXUazKNueVpfJPxg4= @@ -75,8 +77,6 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= -github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU= github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= @@ -102,8 +102,8 @@ github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZ github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt/v5 v5.2.3 h1:kkGXqQOBSDDWRhWNXTFpqGSCMyh/PLnqUvMGJPDJDs0= -github.com/golang-jwt/jwt/v5 v5.2.3/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= +github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= @@ -161,8 +161,6 @@ github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/redis/go-redis/v9 v9.8.0 h1:q3nRvjrlge/6UD7eTu/DSg2uYiU2mCL0G/uzBWqhicI= -github.com/redis/go-redis/v9 v9.8.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/sanity-io/litter v1.5.5 h1:iE+sBxPBzoK6uaEP5Lt3fHNgpKcHXc/A2HGETy0uJQo= @@ -202,16 +200,16 @@ go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= -golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= +golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= +golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= -golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= +golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -221,22 +219,22 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= -golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= +golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= -golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= -golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= +golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= +golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/e2e/kube.go b/e2e/kube.go index 034ddd468d4..f8efd38c28e 100644 --- a/e2e/kube.go +++ b/e2e/kube.go @@ -72,10 +72,10 @@ func getClusterKubeClient(ctx context.Context, resourceGroupName, clusterName st }, nil } -func (k *Kubeclient) WaitUntilPodRunning(ctx context.Context, namespace string, labelSelector string, fieldSelector string) (*corev1.Pod, error) { +func (k *Kubeclient) WaitUntilPodRunningWithRetry(ctx context.Context, namespace string, labelSelector string, fieldSelector string, maxRetries int) (*corev1.Pod, error) { var pod *corev1.Pod - err := wait.PollUntilContextTimeout(ctx, time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) { + err := wait.PollUntilContextTimeout(ctx, 3*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) { pods, err := k.Typed.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ FieldSelector: fieldSelector, LabelSelector: labelSelector, @@ -103,7 +103,13 @@ func (k *Kubeclient) WaitUntilPodRunning(ctx context.Context, namespace string, if err == nil { for _, event := range events.Items { if event.Reason == "FailedCreatePodSandBox" { - return false, fmt.Errorf("pod %s has FailedCreatePodSandBox event: %s", pod.Name, event.Message) + maxRetries-- + sandboxErr := fmt.Errorf("pod %s has FailedCreatePodSandBox event: %s", pod.Name, event.Message) + if maxRetries <= 0 { + return false, sandboxErr + } + k.Typed.CoreV1().Pods(pod.Namespace).Delete(ctx, pod.Name, metav1.DeleteOptions{GracePeriodSeconds: to.Ptr(int64(0))}) + return false, nil // Keep polling } } } @@ -133,6 +139,10 @@ func (k *Kubeclient) WaitUntilPodRunning(ctx context.Context, namespace string, return pod, err } +func (k *Kubeclient) WaitUntilPodRunning(ctx context.Context, namespace string, labelSelector string, fieldSelector string) (*corev1.Pod, error) { + return k.WaitUntilPodRunningWithRetry(ctx, namespace, labelSelector, fieldSelector, 0) +} + func (k *Kubeclient) WaitUntilNodeReady(ctx context.Context, t testing.TB, vmssName string) string { startTime := time.Now() t.Logf("waiting for node %s to be ready in k8s API", vmssName) @@ -189,18 +199,13 @@ func (k *Kubeclient) WaitUntilNodeReady(ctx context.Context, t testing.TB, vmssN return node.Name } -// GetHostNetworkDebugPod returns a pod that's a member of the 'debug' daemonset, running on an aks-nodepool node. -func (k *Kubeclient) GetHostNetworkDebugPod(ctx context.Context) (*corev1.Pod, error) { - return k.WaitUntilPodRunning(ctx, defaultNamespace, fmt.Sprintf("app=%s", hostNetworkDebugAppLabel), "") -} - // GetPodNetworkDebugPodForNode returns a pod that's a member of the 'debugnonhost' daemonset running in the cluster - this will return // the name of the pod that is running on the node created for specifically for the test case which is running validation checks. func (k *Kubeclient) GetPodNetworkDebugPodForNode(ctx context.Context, kubeNodeName string) (*corev1.Pod, error) { if kubeNodeName == "" { return nil, fmt.Errorf("kubeNodeName must not be empty") } - return k.WaitUntilPodRunning(ctx, defaultNamespace, fmt.Sprintf("app=%s", podNetworkDebugAppLabel), "spec.nodeName="+kubeNodeName) + return k.WaitUntilPodRunningWithRetry(ctx, defaultNamespace, fmt.Sprintf("app=%s", podNetworkDebugAppLabel), "spec.nodeName="+kubeNodeName, 3) } func logPodDebugInfo(ctx context.Context, kube *Kubeclient, pod *corev1.Pod) { diff --git a/e2e/node_config.go b/e2e/node_config.go index 10d288cb1d4..0d02eb79f73 100644 --- a/e2e/node_config.go +++ b/e2e/node_config.go @@ -355,7 +355,7 @@ func baseTemplateLinux(t testing.TB, location string, k8sVersion string, arch st NodeStatusUpdateFrequency: "", LoadBalancerSku: "Standard", ExcludeMasterFromStandardLB: nil, - AzureCNIURLLinux: "https://packages.aks.azure.com/azure-cni/v1.1.8/binaries/azure-vnet-cni-linux-amd64-v1.1.8.tgz", + AzureCNIURLLinux: "https://packages.aks.azure.com/azure-cni/v1.6.21/binaries/azure-vnet-cni-linux-amd64-v1.6.21.tgz", AzureCNIURLARM64Linux: "", AzureCNIURLWindows: "", MaximumLoadBalancerRuleCount: 250, diff --git a/e2e/scenario_gpu_managed_experience_test.go b/e2e/scenario_gpu_managed_experience_test.go index be544a5d960..a926eabe77c 100644 --- a/e2e/scenario_gpu_managed_experience_test.go +++ b/e2e/scenario_gpu_managed_experience_test.go @@ -10,7 +10,7 @@ import ( "github.com/Azure/agentbaker/e2e/config" "github.com/Azure/agentbaker/pkg/agent/datamodel" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v7" "github.com/stretchr/testify/require" ) diff --git a/e2e/scenario_test.go b/e2e/scenario_test.go index 6e281f2a4ee..50a21dc9772 100644 --- a/e2e/scenario_test.go +++ b/e2e/scenario_test.go @@ -12,8 +12,8 @@ import ( "github.com/Azure/agentbaker/e2e/toolkit" "github.com/Azure/agentbaker/pkg/agent/datamodel" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v6" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v7" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v8" "github.com/stretchr/testify/require" ) @@ -180,15 +180,15 @@ func Test_Flatcar_SecureTLSBootstrapping_BootstrapToken_Fallback(t *testing.T) { }) } -func Test_AzureLinuxV2_AirGap(t *testing.T) { +func Test_AzureLinuxV3_AirGap(t *testing.T) { RunScenario(t, &Scenario{ - Description: "Tests that a node using a AzureLinuxV2 (CgroupV2) VHD can be properly bootstrapped", + Description: "Tests that a node using a AzureLinuxV3 (CgroupV2) VHD can be properly bootstrapped", Tags: Tags{ Airgap: true, }, Config: Config{ Cluster: ClusterKubenetAirgap, - VHD: config.VHDAzureLinuxV2Gen2, + VHD: config.VHDAzureLinuxV3Gen2, BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) { nbc.OutboundType = datamodel.OutboundTypeBlock nbc.ContainerService.Properties.SecurityProfile = &datamodel.SecurityProfile{ @@ -226,70 +226,6 @@ func Test_AzureLinuxV3_SecureTLSBootstrapping_BootstrapToken_Fallback(t *testing }) } -func Test_AzureLinuxV2_ARM64(t *testing.T) { - RunScenario(t, &Scenario{ - Description: "Tests that a node using a AzureLinuxV2 (CgroupV2) VHD on ARM64 architecture can be properly bootstrapped", - Config: Config{ - Cluster: ClusterKubenet, - VHD: config.VHDAzureLinuxV2Gen2Arm64, - BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) { - nbc.AgentPoolProfile.VMSize = "Standard_D2pds_V5" - nbc.IsARM64 = true - }, - VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { - vmss.SKU.Name = to.Ptr("Standard_D2pds_V5") - }, - }, - }) -} - -func Test_AzureLinuxV2_ARM64_Scriptless(t *testing.T) { - RunScenario(t, &Scenario{ - Description: "Tests that a node using a AzureLinuxV2 (CgroupV2) VHD on ARM64 architecture can be properly bootstrapped", - Config: Config{ - Cluster: ClusterKubenet, - VHD: config.VHDAzureLinuxV2Gen2Arm64, - AKSNodeConfigMutator: func(config *aksnodeconfigv1.Configuration) { - config.VmSize = "Standard_D2pds_V5" - }, - VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { - vmss.SKU.Name = to.Ptr("Standard_D2pds_V5") - }, - }, - }) -} - -func Test_AzureLinuxV2_ARM64AirGap(t *testing.T) { - RunScenario(t, &Scenario{ - Description: "Tests that a node using a AzureLinuxV2 (CgroupV2) VHD on ARM64 architecture can be properly bootstrapped", - Tags: Tags{ - Airgap: true, - }, - Config: Config{ - Cluster: ClusterKubenetAirgap, - VHD: config.VHDAzureLinuxV2Gen2Arm64, - BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) { - nbc.AgentPoolProfile.VMSize = "Standard_D2pds_V5" - nbc.IsARM64 = true - - nbc.OutboundType = datamodel.OutboundTypeBlock - nbc.ContainerService.Properties.SecurityProfile = &datamodel.SecurityProfile{ - PrivateEgress: &datamodel.PrivateEgress{ - Enabled: true, - ContainerRegistryServer: fmt.Sprintf("%s.azurecr.io", config.PrivateACRName(config.Config.DefaultLocation)), - }, - } - }, - VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { - vmss.SKU.Name = to.Ptr("Standard_D2pds_V5") - }, - Validator: func(ctx context.Context, s *Scenario) { - ValidateDirectoryContent(ctx, s, "/opt/azure", []string{"outbound-check-skipped"}) - }, - }, - }) -} - func Test_AzureLinuxV3_AirGap_Package_Install(t *testing.T) { RunScenario(t, &Scenario{ Description: "Tests that a node using a AzureLinuxV3 VHD on ARM64 architecture can be properly bootstrapped", @@ -321,42 +257,12 @@ func Test_AzureLinuxV3_AirGap_Package_Install(t *testing.T) { }) } -func Test_AzureLinuxV2_ARM64_ArtifactSourceCache(t *testing.T) { - RunScenario(t, &Scenario{ - Description: "Tests that a node using a AzureLinuxV2 (CgroupV2) VHD on ARM64 architecture can be properly bootstrapped", - Tags: Tags{ - Airgap: false, - }, - Config: Config{ - Cluster: ClusterKubenet, - VHD: config.VHDAzureLinuxV2Gen2Arm64, - BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) { - nbc.AgentPoolProfile.VMSize = "Standard_D2pds_V5" - nbc.IsARM64 = true - - nbc.ContainerService.Properties.SecurityProfile = &datamodel.SecurityProfile{ - PrivateEgress: &datamodel.PrivateEgress{ - Enabled: true, - ContainerRegistryServer: "mcr.microsoft.com", - }, - } - }, - VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { - vmss.SKU.Name = to.Ptr("Standard_D2pds_V5") - }, - Validator: func(ctx context.Context, s *Scenario) { - ValidateDirectoryContent(ctx, s, "/opt/azure", []string{"outbound-check-skipped"}) - }, - }, - }) -} - -func Test_AzureLinuxV2_AzureCNI(t *testing.T) { +func Test_AzureLinuxV3_AzureCNI(t *testing.T) { RunScenario(t, &Scenario{ - Description: "azurelinuxv2 scenario on a cluster configured with Azure CNI", + Description: "azurelinuxv3 scenario on a cluster configured with Azure CNI", Config: Config{ Cluster: ClusterAzureNetwork, - VHD: config.VHDAzureLinuxV2Gen2, + VHD: config.VHDAzureLinuxV3Gen2, BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) { nbc.ContainerService.Properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin = string(armcontainerservice.NetworkPluginAzure) nbc.AgentPoolProfile.KubernetesConfig.NetworkPlugin = string(armcontainerservice.NetworkPluginAzure) @@ -365,351 +271,18 @@ func Test_AzureLinuxV2_AzureCNI(t *testing.T) { }) } -func Test_AzureLinuxV2_ChronyRestarts(t *testing.T) { +func Test_AzureLinuxV3_ChronyRestarts(t *testing.T) { RunScenario(t, &Scenario{ Description: "Tests that the chrony service restarts if it is killed", Config: Config{ Cluster: ClusterKubenet, - VHD: config.VHDAzureLinuxV2Gen2, - BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) { - }, - Validator: func(ctx context.Context, s *Scenario) { - ValidateFileHasContent(ctx, s, "/etc/systemd/system/chronyd.service.d/10-chrony-restarts.conf", "Restart=always") - ValidateFileHasContent(ctx, s, "/etc/systemd/system/chronyd.service.d/10-chrony-restarts.conf", "RestartSec=5") - ServiceCanRestartValidator(ctx, s, "chronyd", 10) - }, - }, - }) -} - -func Test_AzureLinuxV2_CustomSysctls(t *testing.T) { - customSysctls := map[string]string{ - "net.ipv4.ip_local_port_range": "32768 62535", - "net.netfilter.nf_conntrack_max": "2097152", - "net.netfilter.nf_conntrack_buckets": "524288", - "net.ipv4.tcp_keepalive_intvl": "90", - "net.ipv4.ip_local_reserved_ports": "", - } - customContainerdUlimits := map[string]string{ - "LimitMEMLOCK": "75000", - "LimitNOFILE": "1048", - } - RunScenario(t, &Scenario{ - Description: "tests that a AzureLinuxV2 (CgroupV2) VHD can be properly bootstrapped when supplied custom node config that contains custom sysctl settings", - Config: Config{ - Cluster: ClusterKubenet, - VHD: config.VHDAzureLinuxV2Gen2, - BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) { - customLinuxConfig := &datamodel.CustomLinuxOSConfig{ - Sysctls: &datamodel.SysctlConfig{ - NetNetfilterNfConntrackMax: to.Ptr(toolkit.StrToInt32(customSysctls["net.netfilter.nf_conntrack_max"])), - NetNetfilterNfConntrackBuckets: to.Ptr(toolkit.StrToInt32(customSysctls["net.netfilter.nf_conntrack_buckets"])), - NetIpv4IpLocalPortRange: customSysctls["net.ipv4.ip_local_port_range"], - NetIpv4TcpkeepaliveIntvl: to.Ptr(toolkit.StrToInt32(customSysctls["net.ipv4.tcp_keepalive_intvl"])), - }, - UlimitConfig: &datamodel.UlimitConfig{ - MaxLockedMemory: customContainerdUlimits["LimitMEMLOCK"], - NoFile: customContainerdUlimits["LimitNOFILE"], - }, - } - nbc.AgentPoolProfile.CustomLinuxOSConfig = customLinuxConfig - }, - Validator: func(ctx context.Context, s *Scenario) { - ValidateUlimitSettings(ctx, s, customContainerdUlimits) - ValidateSysctlConfig(ctx, s, customSysctls) - }, - }, - }) -} - -// Returns config for the 'gpu' E2E scenario -func Test_AzureLinuxV2_GPU(t *testing.T) { - RunScenario(t, &Scenario{ - Description: "Tests that a GPU-enabled node using a AzureLinuxV2 (CgroupV2) VHD can be properly bootstrapped", - Tags: Tags{ - GPU: true, - }, - Config: Config{ - Cluster: ClusterKubenet, - VHD: config.VHDAzureLinuxV2Gen2, - BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) { - nbc.AgentPoolProfile.VMSize = "Standard_NC6s_v3" - nbc.ConfigGPUDriverIfNeeded = true - nbc.EnableGPUDevicePluginIfNeeded = false - nbc.EnableNvidia = true - }, - VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { - vmss.SKU.Name = to.Ptr("Standard_NC6s_v3") - }, - Validator: func(ctx context.Context, s *Scenario) { - }, - }, - }) -} - -func Test_AzureLinuxV2_GPUAzureCNI(t *testing.T) { - RunScenario(t, &Scenario{ - Description: "AzureLinux V2 (CgroupV2) gpu scenario on cluster configured with Azure CNI", - Tags: Tags{ - GPU: true, - }, - Config: Config{ - Cluster: ClusterAzureNetwork, - VHD: config.VHDAzureLinuxV2Gen2, - BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) { - nbc.ContainerService.Properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin = string(armcontainerservice.NetworkPluginAzure) - nbc.AgentPoolProfile.KubernetesConfig.NetworkPlugin = string(armcontainerservice.NetworkPluginAzure) - nbc.AgentPoolProfile.VMSize = "Standard_NC6s_v3" - nbc.ConfigGPUDriverIfNeeded = true - nbc.EnableGPUDevicePluginIfNeeded = false - nbc.EnableNvidia = true - }, - VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { - vmss.SKU.Name = to.Ptr("Standard_NC6s_v3") - }, - Validator: func(ctx context.Context, s *Scenario) { - }, - }, - }) -} - -func Test_AzureLinuxV2_GPUAzureCNI_Scriptless(t *testing.T) { - RunScenario(t, &Scenario{ - Description: "AzureLinux V2 (CgroupV2) gpu scenario on cluster configured with Azure CNI", - Tags: Tags{ - GPU: true, - }, - Config: Config{ - Cluster: ClusterAzureNetwork, - VHD: config.VHDAzureLinuxV2Gen2, - AKSNodeConfigMutator: func(config *aksnodeconfigv1.Configuration) { - config.NetworkConfig.NetworkPlugin = aksnodeconfigv1.NetworkPlugin_NETWORK_PLUGIN_AZURE - config.VmSize = "Standard_NC6s_v3" - config.GpuConfig.ConfigGpuDriver = true - config.GpuConfig.GpuDevicePlugin = false - config.GpuConfig.EnableNvidia = to.Ptr(true) - }, - VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { - vmss.SKU.Name = to.Ptr("Standard_NC6s_v3") - }, - Validator: func(ctx context.Context, s *Scenario) { - }, - }, - }) -} - -func Test_MarinerV2(t *testing.T) { - RunScenario(t, &Scenario{ - Description: "Tests that a node using a MarinerV2 VHD can be properly bootstrapped", - Config: Config{ - Cluster: ClusterKubenet, - VHD: config.VHDCBLMarinerV2Gen2, - BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) { - }, - Validator: func(ctx context.Context, s *Scenario) { - ValidateInstalledPackageVersion(ctx, s, "moby-containerd", components.GetExpectedPackageVersions("containerd", "mariner", "current")[0]) - }, - }, - }) -} - -func Test_MarinerV2_AirGap(t *testing.T) { - RunScenario(t, &Scenario{ - Description: "Tests that a node using a MarinerV2 VHD can be properly bootstrapped", - Tags: Tags{ - Airgap: true, - }, - Config: Config{ - Cluster: ClusterKubenetAirgap, - VHD: config.VHDCBLMarinerV2Gen2, - BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) { - nbc.OutboundType = datamodel.OutboundTypeBlock - nbc.ContainerService.Properties.SecurityProfile = &datamodel.SecurityProfile{ - PrivateEgress: &datamodel.PrivateEgress{ - Enabled: true, - ContainerRegistryServer: fmt.Sprintf("%s.azurecr.io", config.PrivateACRName(config.Config.DefaultLocation)), - }, - } - }, - Validator: func(ctx context.Context, s *Scenario) { - ValidateDirectoryContent(ctx, s, "/opt/azure", []string{"outbound-check-skipped"}) - }, - }, - }) -} - -func Test_MarinerV2_ARM64(t *testing.T) { - RunScenario(t, &Scenario{ - Description: "Tests that a node using a MarinerV2 VHD on ARM64 architecture can be properly bootstrapped", - Config: Config{ - Cluster: ClusterKubenet, - VHD: config.VHDCBLMarinerV2Gen2Arm64, - BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) { - nbc.AgentPoolProfile.VMSize = "Standard_D2pds_V5" - nbc.IsARM64 = true - }, - VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { - vmss.SKU.Name = to.Ptr("Standard_D2pds_V5") - }, - }, - }) -} - -func Test_MarinerV2_ARM64AirGap(t *testing.T) { - RunScenario(t, &Scenario{ - Description: "Tests that a node using a MarinerV2 VHD on ARM64 architecture can be properly bootstrapped", - Tags: Tags{ - Airgap: true, - }, - Config: Config{ - Cluster: ClusterKubenetAirgap, - VHD: config.VHDCBLMarinerV2Gen2Arm64, - BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) { - nbc.AgentPoolProfile.VMSize = "Standard_D2pds_V5" - nbc.IsARM64 = true - - nbc.OutboundType = datamodel.OutboundTypeBlock - nbc.ContainerService.Properties.SecurityProfile = &datamodel.SecurityProfile{ - PrivateEgress: &datamodel.PrivateEgress{ - Enabled: true, - ContainerRegistryServer: fmt.Sprintf("%s.azurecr.io", config.PrivateACRName(config.Config.DefaultLocation)), - }, - } - }, - VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { - vmss.SKU.Name = to.Ptr("Standard_D2pds_V5") - }, - Validator: func(ctx context.Context, s *Scenario) { - ValidateDirectoryContent(ctx, s, "/opt/azure", []string{"outbound-check-skipped"}) - }, - }, - }) -} - -// Merge test case MarinerV2 AzureCNI with MarinerV2 ChronyRestarts -func Test_MarinerV2_AzureCNI_ChronyRestarts(t *testing.T) { - RunScenario(t, &Scenario{ - Description: "Test marinerv2 scenario on a cluster configured with Azure CNI and the chrony service restarts if it is killed", - Config: Config{ - Cluster: ClusterAzureNetwork, - VHD: config.VHDCBLMarinerV2Gen2, + VHD: config.VHDAzureLinuxV3Gen2, BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) { - nbc.ContainerService.Properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin = string(armcontainerservice.NetworkPluginAzure) - nbc.AgentPoolProfile.KubernetesConfig.NetworkPlugin = string(armcontainerservice.NetworkPluginAzure) }, Validator: func(ctx context.Context, s *Scenario) { - ServiceCanRestartValidator(ctx, s, "chronyd", 10) ValidateFileHasContent(ctx, s, "/etc/systemd/system/chronyd.service.d/10-chrony-restarts.conf", "Restart=always") ValidateFileHasContent(ctx, s, "/etc/systemd/system/chronyd.service.d/10-chrony-restarts.conf", "RestartSec=5") - }, - }, - }) -} - -// Merge scriptless test case MarinerV2 AzureCNI with MarinerV2 ChronyRestarts -func Test_MarinerV2_AzureCNI_ChronyRestarts_Scriptless(t *testing.T) { - RunScenario(t, &Scenario{ - Description: "Test marinerv2 scenario on a cluster configured with Azure CNI and the chrony service restarts if it is killed", - Config: Config{ - Cluster: ClusterAzureNetwork, - VHD: config.VHDCBLMarinerV2Gen2, - AKSNodeConfigMutator: func(config *aksnodeconfigv1.Configuration) { - config.NetworkConfig.NetworkPlugin = aksnodeconfigv1.NetworkPlugin_NETWORK_PLUGIN_AZURE - }, - Validator: func(ctx context.Context, s *Scenario) { ServiceCanRestartValidator(ctx, s, "chronyd", 10) - ValidateFileHasContent(ctx, s, "/etc/systemd/system/chronyd.service.d/10-chrony-restarts.conf", "Restart=always") - ValidateFileHasContent(ctx, s, "/etc/systemd/system/chronyd.service.d/10-chrony-restarts.conf", "RestartSec=5") - }, - }, - }) -} - -func Test_MarinerV2_CustomSysctls(t *testing.T) { - customSysctls := map[string]string{ - "net.ipv4.ip_local_port_range": "32768 62535", - "net.netfilter.nf_conntrack_max": "2097152", - "net.netfilter.nf_conntrack_buckets": "524288", - "net.ipv4.tcp_keepalive_intvl": "90", - "net.ipv4.ip_local_reserved_ports": "", - } - customContainerdUlimits := map[string]string{ - "LimitMEMLOCK": "75000", - "LimitNOFILE": "1048", - } - RunScenario(t, &Scenario{ - Description: "tests that a MarinerV2 VHD can be properly bootstrapped when supplied custom node config that contains custom sysctl settings", - Config: Config{ - Cluster: ClusterKubenet, - VHD: config.VHDCBLMarinerV2Gen2, - BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) { - customLinuxConfig := &datamodel.CustomLinuxOSConfig{ - Sysctls: &datamodel.SysctlConfig{ - NetNetfilterNfConntrackMax: to.Ptr(toolkit.StrToInt32(customSysctls["net.netfilter.nf_conntrack_max"])), - NetNetfilterNfConntrackBuckets: to.Ptr(toolkit.StrToInt32(customSysctls["net.netfilter.nf_conntrack_buckets"])), - NetIpv4IpLocalPortRange: customSysctls["net.ipv4.ip_local_port_range"], - NetIpv4TcpkeepaliveIntvl: to.Ptr(toolkit.StrToInt32(customSysctls["net.ipv4.tcp_keepalive_intvl"])), - }, - UlimitConfig: &datamodel.UlimitConfig{ - MaxLockedMemory: customContainerdUlimits["LimitMEMLOCK"], - NoFile: customContainerdUlimits["LimitNOFILE"], - }, - } - nbc.AgentPoolProfile.CustomLinuxOSConfig = customLinuxConfig - }, - Validator: func(ctx context.Context, s *Scenario) { - ValidateUlimitSettings(ctx, s, customContainerdUlimits) - ValidateSysctlConfig(ctx, s, customSysctls) - }, - }, - }) -} - -func Test_MarinerV2_GPU(t *testing.T) { - RunScenario(t, &Scenario{ - Description: "Tests that a GPU-enabled node using a MarinerV2 VHD can be properly bootstrapped", - Tags: Tags{ - GPU: true, - }, - Config: Config{ - Cluster: ClusterKubenet, - VHD: config.VHDCBLMarinerV2Gen2, - BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) { - nbc.AgentPoolProfile.VMSize = "Standard_NC6s_v3" - nbc.ConfigGPUDriverIfNeeded = true - nbc.EnableGPUDevicePluginIfNeeded = false - nbc.EnableNvidia = true - }, - VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { - vmss.SKU.Name = to.Ptr("Standard_NC6s_v3") - }, - Validator: func(ctx context.Context, s *Scenario) { - }, - }, - }) -} - -func Test_MarinerV2_GPUAzureCNI(t *testing.T) { - RunScenario(t, &Scenario{ - Description: "MarinerV2 gpu scenario on cluster configured with Azure CNI", - Tags: Tags{ - GPU: true, - }, - Config: Config{ - Cluster: ClusterAzureNetwork, - VHD: config.VHDCBLMarinerV2Gen2, - BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) { - nbc.ContainerService.Properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin = string(armcontainerservice.NetworkPluginAzure) - nbc.AgentPoolProfile.KubernetesConfig.NetworkPlugin = string(armcontainerservice.NetworkPluginAzure) - nbc.AgentPoolProfile.VMSize = "Standard_NC6s_v3" - nbc.ConfigGPUDriverIfNeeded = true - nbc.EnableGPUDevicePluginIfNeeded = false - nbc.EnableNvidia = true - }, - VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { - vmss.SKU.Name = to.Ptr("Standard_NC6s_v3") - }, - Validator: func(ctx context.Context, s *Scenario) { }, }, }) @@ -818,31 +391,12 @@ func Test_Ubuntu2204_EntraIDSSH_Scriptless(t *testing.T) { }) } -func Test_Ubuntu2204_DisableSSH(t *testing.T) { +func Test_AzureLinuxV3_DisableSSH(t *testing.T) { RunScenario(t, &Scenario{ - Description: "Tests that a node using Ubuntu 2204 VHD with SSH disabled can be properly bootstrapped and SSH daemon is disabled", + Description: "Tests that a node using AzureLinuxV3 VHD with SSH disabled can be properly bootstrapped and SSH daemon is disabled", Config: Config{ Cluster: ClusterKubenet, - VHD: config.VHDUbuntu2204Gen2Containerd, - BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) { - nbc.SSHStatus = datamodel.SSHOff - }, - SkipSSHConnectivityValidation: true, // Skip SSH connectivity validation since SSH is down - SkipDefaultValidation: true, // Skip default validation since it requires SSH connectivity - Validator: func(ctx context.Context, s *Scenario) { - // Validate SSH daemon is disabled via RunCommand - ValidateSSHServiceDisabled(ctx, s) - }, - }, - }) -} - -func Test_AzureLinuxV2_DisableSSH(t *testing.T) { - RunScenario(t, &Scenario{ - Description: "Tests that a node using AzureLinuxV2 VHD with SSH disabled can be properly bootstrapped and SSH daemon is disabled", - Config: Config{ - Cluster: ClusterKubenet, - VHD: config.VHDAzureLinuxV2Gen2, + VHD: config.VHDAzureLinuxV3Gen2, BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) { nbc.SSHStatus = datamodel.SSHOff }, @@ -856,12 +410,12 @@ func Test_AzureLinuxV2_DisableSSH(t *testing.T) { }) } -func Test_MarinerV2_DisableSSH(t *testing.T) { +func Test_Ubuntu2204_DisableSSH(t *testing.T) { RunScenario(t, &Scenario{ - Description: "Tests that a node using MarinerV2 VHD with SSH disabled can be properly bootstrapped and SSH daemon is disabled", + Description: "Tests that a node using Ubuntu 2204 VHD with SSH disabled can be properly bootstrapped and SSH daemon is disabled", Config: Config{ Cluster: ClusterKubenet, - VHD: config.VHDCBLMarinerV2Gen2, + VHD: config.VHDUbuntu2204Gen2Containerd, BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) { nbc.SSHStatus = datamodel.SSHOff }, @@ -1120,12 +674,12 @@ func Test_Ubuntu2204_ChronyRestarts_Taints_And_Tolerations_Scriptless(t *testing }) } -func Test_AzureLinuxV2_CustomCATrust(t *testing.T) { +func Test_AzureLinuxV3_CustomCATrust(t *testing.T) { RunScenario(t, &Scenario{ - Description: "Tests that a node using the Azure Linux 2204 VHD can be properly bootstrapped and custom CA was correctly added", + Description: "Tests that a node using the Azure Linux V3 VHD can be properly bootstrapped and custom CA was correctly added", Config: Config{ Cluster: ClusterKubenet, - VHD: config.VHDAzureLinuxV2Gen2, + VHD: config.VHDAzureLinuxV3Gen2, BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) { nbc.CustomCATrustConfig = &datamodel.CustomCATrustConfig{ CustomCATrustCerts: []string{ @@ -1522,7 +1076,7 @@ func Test_AzureLinux_Skip_Binary_Cleanup(t *testing.T) { Description: "tests that an AzureLinux node will skip binary cleanup and can be properly bootstrapped", Config: Config{ Cluster: ClusterKubenet, - VHD: config.VHDAzureLinuxV2Gen2, + VHD: config.VHDAzureLinuxV3Gen2, BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) {}, VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { if vmss.Tags == nil { @@ -1661,12 +1215,12 @@ func Test_Ubuntu2204_MessageOfTheDay(t *testing.T) { }) } -func Test_AzureLinuxV2_MessageOfTheDay(t *testing.T) { +func Test_AzureLinuxV3_MessageOfTheDay(t *testing.T) { RunScenario(t, &Scenario{ - Description: "Tests that a node using a AzureLinuxV2 can be bootstrapped and message of the day is added to the node", + Description: "Tests that a node using a AzureLinuxV3 can be bootstrapped and message of the day is added to the node", Config: Config{ Cluster: ClusterKubenet, - VHD: config.VHDAzureLinuxV2Gen2, + VHD: config.VHDAzureLinuxV3Gen2, BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) { nbc.AgentPoolProfile.MessageOfTheDay = "Zm9vYmFyDQo=" // base64 for foobar }, @@ -1678,12 +1232,12 @@ func Test_AzureLinuxV2_MessageOfTheDay(t *testing.T) { }) } -func Test_AzureLinuxV2_MessageOfTheDay_Scriptless(t *testing.T) { +func Test_AzureLinuxV3_MessageOfTheDay_Scriptless(t *testing.T) { RunScenario(t, &Scenario{ - Description: "Tests that a node using a AzureLinuxV2 can be bootstrapped and message of the day is added to the node", + Description: "Tests that a node using a AzureLinuxV3 can be bootstrapped and message of the day is added to the node", Config: Config{ Cluster: ClusterKubenet, - VHD: config.VHDAzureLinuxV2Gen2, + VHD: config.VHDAzureLinuxV3Gen2, AKSNodeConfigMutator: func(config *aksnodeconfigv1.Configuration) { config.MessageOfTheDay = "Zm9vYmFyDQo=" // base64 for foobar }, @@ -1695,12 +1249,12 @@ func Test_AzureLinuxV2_MessageOfTheDay_Scriptless(t *testing.T) { }) } -func Test_AzureLinuxV2_LocalDns_Disabled_Scriptless(t *testing.T) { +func Test_AzureLinuxV3LocalDns_Disabled_Scriptless(t *testing.T) { RunScenario(t, &Scenario{ - Description: "Tests that a node using a AzureLinuxV2 can be bootstrapped with localdns disabled", + Description: "Tests that a node using a AzureLinuxV3 can be bootstrapped with localdns disabled", Config: Config{ Cluster: ClusterAzureNetwork, - VHD: config.VHDAzureLinuxV2Gen2, + VHD: config.VHDAzureLinuxV3Gen2, AKSNodeConfigMutator: func(config *aksnodeconfigv1.Configuration) { config.LocalDnsProfile = &aksnodeconfigv1.LocalDnsProfile{ EnableLocalDns: false, @@ -1715,6 +1269,46 @@ func Test_AzureLinuxV2_LocalDns_Disabled_Scriptless(t *testing.T) { }) } +func Test_AzureLinuxV3_CustomSysctls(t *testing.T) { + customSysctls := map[string]string{ + "net.ipv4.ip_local_port_range": "32768 62535", + "net.netfilter.nf_conntrack_max": "2097152", + "net.netfilter.nf_conntrack_buckets": "524288", + "net.ipv4.tcp_keepalive_intvl": "90", + "net.ipv4.ip_local_reserved_ports": "", + } + customContainerdUlimits := map[string]string{ + "LimitMEMLOCK": "75000", + "LimitNOFILE": "1048", + } + RunScenario(t, &Scenario{ + Description: "tests that a AzureLinuxV3 (CgroupV2) VHD can be properly bootstrapped when supplied custom node config that contains custom sysctl settings", + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDAzureLinuxV3Gen2, + BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) { + customLinuxConfig := &datamodel.CustomLinuxOSConfig{ + Sysctls: &datamodel.SysctlConfig{ + NetNetfilterNfConntrackMax: to.Ptr(toolkit.StrToInt32(customSysctls["net.netfilter.nf_conntrack_max"])), + NetNetfilterNfConntrackBuckets: to.Ptr(toolkit.StrToInt32(customSysctls["net.netfilter.nf_conntrack_buckets"])), + NetIpv4IpLocalPortRange: customSysctls["net.ipv4.ip_local_port_range"], + NetIpv4TcpkeepaliveIntvl: to.Ptr(toolkit.StrToInt32(customSysctls["net.ipv4.tcp_keepalive_intvl"])), + }, + UlimitConfig: &datamodel.UlimitConfig{ + MaxLockedMemory: customContainerdUlimits["LimitMEMLOCK"], + NoFile: customContainerdUlimits["LimitNOFILE"], + }, + } + nbc.AgentPoolProfile.CustomLinuxOSConfig = customLinuxConfig + }, + Validator: func(ctx context.Context, s *Scenario) { + ValidateUlimitSettings(ctx, s, customContainerdUlimits) + ValidateSysctlConfig(ctx, s, customSysctls) + }, + }, + }) +} + func Test_Ubuntu2204_KubeletCustomConfig(t *testing.T) { RunScenario(t, &Scenario{ Tags: Tags{ @@ -1742,18 +1336,18 @@ func Test_Ubuntu2204_KubeletCustomConfig(t *testing.T) { }) } -func Test_AzureLinuxV2_KubeletCustomConfig(t *testing.T) { +func Test_AzureLinuxV3_KubeletCustomConfig(t *testing.T) { RunScenario(t, &Scenario{ Tags: Tags{ KubeletCustomConfig: true, }, - Description: "tests that a node on azure linux v2 bootstrapped with kubelet custom config for seccomp set to non default values", + Description: "tests that a node on azure linux v3 bootstrapped with kubelet custom config for seccomp set to non default values", Config: Config{ Cluster: ClusterKubenet, - VHD: config.VHDAzureLinuxV2Gen2, + VHD: config.VHDAzureLinuxV3Gen2, BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) { - nbc.ContainerService.Properties.AgentPoolProfiles[0].Distro = "aks-azurelinux-v2-gen2" - nbc.AgentPoolProfile.Distro = "aks-azurelinux-v2-gen2" + nbc.ContainerService.Properties.AgentPoolProfiles[0].Distro = "aks-azurelinux-v3-gen2" + nbc.AgentPoolProfile.Distro = "aks-azurelinux-v3-gen2" customKubeletConfig := &datamodel.CustomKubeletConfig{ SeccompDefault: to.Ptr(true), } @@ -1764,21 +1358,21 @@ func Test_AzureLinuxV2_KubeletCustomConfig(t *testing.T) { kubeletConfigFilePath := "/etc/default/kubeletconfig.json" ValidateFileHasContent(ctx, s, kubeletConfigFilePath, `"seccompDefault": true`) ValidateKubeletHasFlags(ctx, s, kubeletConfigFilePath) - ValidateInstalledPackageVersion(ctx, s, "moby-containerd", components.GetExpectedPackageVersions("containerd", "mariner", "current")[0]) + ValidateInstalledPackageVersion(ctx, s, "containerd2", components.GetExpectedPackageVersions("containerd", "azurelinux", "v3.0")[0]) }, }, }) } -func Test_AzureLinuxV2_KubeletCustomConfig_Scriptless(t *testing.T) { +func Test_AzureLinuxV3_KubeletCustomConfig_Scriptless(t *testing.T) { RunScenario(t, &Scenario{ Tags: Tags{ KubeletCustomConfig: true, }, - Description: "tests that a node on azure linux v2 bootstrapped with kubelet custom config for seccomp set to non default values", + Description: "tests that a node on azure linux v3 bootstrapped with kubelet custom config for seccomp set to non default values", Config: Config{ Cluster: ClusterKubenet, - VHD: config.VHDAzureLinuxV2Gen2, + VHD: config.VHDAzureLinuxV3Gen2, AKSNodeConfigMutator: func(config *aksnodeconfigv1.Configuration) { config.KubeletConfig.KubeletConfigFileConfig.SeccompDefault = true }, @@ -1786,7 +1380,82 @@ func Test_AzureLinuxV2_KubeletCustomConfig_Scriptless(t *testing.T) { kubeletConfigFilePath := "/etc/default/kubeletconfig.json" ValidateFileHasContent(ctx, s, kubeletConfigFilePath, `"seccompDefault": true`) ValidateKubeletHasFlags(ctx, s, kubeletConfigFilePath) - ValidateInstalledPackageVersion(ctx, s, "moby-containerd", components.GetExpectedPackageVersions("containerd", "mariner", "current")[0]) + ValidateInstalledPackageVersion(ctx, s, "containerd2", components.GetExpectedPackageVersions("containerd", "azurelinux", "v3.0")[0]) + }, + }, + }) +} + +func Test_AzureLinuxV3_GPU(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests that a GPU-enabled node using a AzureLinuxV3 (CgroupV2) VHD can be properly bootstrapped", + Tags: Tags{ + GPU: true, + }, + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDAzureLinuxV3Gen2, + BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) { + nbc.AgentPoolProfile.VMSize = "Standard_NC6s_v3" + nbc.ConfigGPUDriverIfNeeded = true + nbc.EnableGPUDevicePluginIfNeeded = false + nbc.EnableNvidia = true + }, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + vmss.SKU.Name = to.Ptr("Standard_NC6s_v3") + }, + Validator: func(ctx context.Context, s *Scenario) { + }, + }, + }) +} + +func Test_AzureLinuxV3_GPUAzureCNI(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "AzureLinux V3 (CgroupV2) gpu scenario on cluster configured with Azure CNI", + Tags: Tags{ + GPU: true, + }, + Config: Config{ + Cluster: ClusterAzureNetwork, + VHD: config.VHDAzureLinuxV3Gen2, + BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) { + nbc.ContainerService.Properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin = string(armcontainerservice.NetworkPluginAzure) + nbc.AgentPoolProfile.KubernetesConfig.NetworkPlugin = string(armcontainerservice.NetworkPluginAzure) + nbc.AgentPoolProfile.VMSize = "Standard_NC6s_v3" + nbc.ConfigGPUDriverIfNeeded = true + nbc.EnableGPUDevicePluginIfNeeded = false + nbc.EnableNvidia = true + }, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + vmss.SKU.Name = to.Ptr("Standard_NC6s_v3") + }, + Validator: func(ctx context.Context, s *Scenario) { + }, + }, + }) +} + +func Test_AzureLinuxV3_GPUAzureCNI_Scriptless(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "AzureLinux V3 (CgroupV2) gpu scenario on cluster configured with Azure CNI", + Tags: Tags{ + GPU: true, + }, + Config: Config{ + Cluster: ClusterAzureNetwork, + VHD: config.VHDAzureLinuxV3Gen2, + AKSNodeConfigMutator: func(config *aksnodeconfigv1.Configuration) { + config.NetworkConfig.NetworkPlugin = aksnodeconfigv1.NetworkPlugin_NETWORK_PLUGIN_AZURE + config.VmSize = "Standard_NC6s_v3" + config.GpuConfig.ConfigGpuDriver = true + config.GpuConfig.GpuDevicePlugin = false + config.GpuConfig.EnableNvidia = to.Ptr(true) + }, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + vmss.SKU.Name = to.Ptr("Standard_NC6s_v3") + }, + Validator: func(ctx context.Context, s *Scenario) { }, }, }) diff --git a/e2e/scenario_win_test.go b/e2e/scenario_win_test.go index 89bcf583602..d57046ac339 100644 --- a/e2e/scenario_win_test.go +++ b/e2e/scenario_win_test.go @@ -12,7 +12,7 @@ import ( "github.com/Azure/agentbaker/e2e/config" "github.com/Azure/agentbaker/pkg/agent/datamodel" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v7" ) func EmptyBootstrapConfigMutator(configuration *datamodel.NodeBootstrappingConfiguration) {} diff --git a/e2e/test_helpers.go b/e2e/test_helpers.go index 79393b05e67..bcecfe9efd5 100644 --- a/e2e/test_helpers.go +++ b/e2e/test_helpers.go @@ -20,7 +20,7 @@ import ( "github.com/Azure/agentbaker/e2e/toolkit" "github.com/Azure/agentbaker/pkg/agent/datamodel" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v7" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/util/wait" ctrruntimelog "sigs.k8s.io/controller-runtime/pkg/log" @@ -28,9 +28,8 @@ import ( ) var ( - logf = toolkit.Logf - log = toolkit.Log - SSHKeyPrivate, SSHKeyPublic = mustGetNewRSAKeyPair() + logf = toolkit.Logf + log = toolkit.Log ) // it's important to share context between tests to allow graceful shutdown @@ -121,6 +120,8 @@ func runScenarioWithPreProvision(t *testing.T, original *Scenario) { customVHD = CreateImage(ctx, stage1) customVHDJSON, _ := json.MarshalIndent(customVHD, "", " ") t.Logf("Created custom VHD image: %s", string(customVHDJSON)) + cleanupBastionTunnel(firstStage.Runtime.VM.SSHClient) + firstStage.Runtime.VM.SSHClient = nil } firstStage.Config.VMConfigMutator = func(vmss *armcompute.VirtualMachineScaleSet) { if original.VMConfigMutator != nil { @@ -205,23 +206,24 @@ func runScenario(t testing.TB, s *Scenario) { Location: s.Location, K8sSystemPoolSKU: s.K8sSystemPoolSKU, }) - require.NoError(s.T, err, "failed to get cluster") + // in some edge cases cluster cache is broken and nil cluster is returned // need to find the root cause and fix it, this should help to catch such cases require.NotNil(t, cluster) + s.Runtime = &ScenarioRuntime{ Cluster: cluster, VMSSName: generateVMSSName(s), } // use shorter timeout for faster feedback on test failures - ctx, cancel := context.WithTimeout(ctx, config.Config.TestTimeoutVMSS) + vmssCtx, cancel := context.WithTimeout(ctx, config.Config.TestTimeoutVMSS) defer cancel() - s.Runtime.VM = prepareAKSNode(ctx, s) + s.Runtime.VM = prepareAKSNode(vmssCtx, s) t.Logf("Choosing the private ACR %q for the vm validation", config.GetPrivateACRName(s.Tags.NonAnonymousACR, s.Location)) - validateVM(ctx, s) + validateVM(vmssCtx, s) } func prepareAKSNode(ctx context.Context, s *Scenario) *ScenarioVM { @@ -246,7 +248,7 @@ func prepareAKSNode(ctx context.Context, s *Scenario) *ScenarioVM { s.AKSNodeConfigMutator(nodeconfig) s.Runtime.AKSNodeConfig = nodeconfig } - publicKeyData := datamodel.PublicKey{KeyData: string(SSHKeyPublic)} + publicKeyData := datamodel.PublicKey{KeyData: string(config.VMSSHPublicKey)} // check it all. if s.Runtime.NBC != nil && s.Runtime.NBC.ContainerService != nil && s.Runtime.NBC.ContainerService.Properties != nil && s.Runtime.NBC.ContainerService.Properties.LinuxProfile != nil { @@ -334,7 +336,7 @@ func ValidateNodeCanRunAPod(ctx context.Context, s *Scenario) { ValidatePodRunning(ctx, s, podWindows(s, fmt.Sprintf("nanoserver%d", i), pod)) } } else { - ValidatePodRunning(ctx, s, podHTTPServerLinux(s)) + ValidatePodRunningWithRetry(ctx, s, podHTTPServerLinux(s), 3) } } @@ -483,21 +485,21 @@ func addTrustedLaunchToVMSS(properties *armcompute.VirtualMachineScaleSetPropert return properties } -func createVMExtensionLinuxAKSNode(location *string) (*armcompute.VirtualMachineScaleSetExtension, error) { +func createVMExtensionLinuxAKSNode(_ *string) (*armcompute.VirtualMachineScaleSetExtension, error) { // Default to "westus" if location is nil. - region := "westus" - if location != nil { - region = *location - } + // region := "westus" + // if location != nil { + // region = *location + // } extensionName := "Compute.AKS.Linux.AKSNode" publisher := "Microsoft.AKS" - + extensionVersion := "1.374" // NOTE (@surajssd): If this is gonna be called multiple times, then find a way to cache the latest version. - extensionVersion, err := config.Azure.GetLatestVMExtensionImageVersion(context.TODO(), region, extensionName, publisher) - if err != nil { - return nil, fmt.Errorf("getting latest VM extension image version: %v", err) - } + // extensionVersion, err := config.Azure.GetLatestVMExtensionImageVersion(context.TODO(), region, extensionName, publisher) + // if err != nil { + // return nil, fmt.Errorf("getting latest VM extension image version: %v", err) + // } return &armcompute.VirtualMachineScaleSetExtension{ Name: to.Ptr(extensionName), @@ -720,16 +722,16 @@ func validateSSHConnectivity(ctx context.Context, s *Scenario) error { // attemptSSHConnection performs a single SSH connectivity check func attemptSSHConnection(ctx context.Context, s *Scenario) error { - connectionTest := fmt.Sprintf("%s echo 'SSH_CONNECTION_OK'", sshString(s.Runtime.VM.PrivateIP)) - connectionResult, err := execOnPrivilegedPod(ctx, s.Runtime.Cluster.Kube, defaultNamespace, s.Runtime.Cluster.DebugPod.Name, connectionTest) + var connectionResult *podExecResult + var err error + connectionResult, err = runSSHCommand(ctx, s.Runtime.VM.SSHClient, "echo 'SSH_CONNECTION_OK'", s.IsWindows()) - if err != nil || !strings.Contains(connectionResult.stdout.String(), "SSH_CONNECTION_OK") { - output := "" - if connectionResult != nil { - output = connectionResult.String() - } + if err != nil { + return fmt.Errorf("SSH connection to %s failed: %s", s.Runtime.VM.PrivateIP, err) + } - return fmt.Errorf("SSH connection to %s failed: %s: %s", s.Runtime.VM.PrivateIP, err, output) + if !strings.Contains(connectionResult.stdout, "SSH_CONNECTION_OK") { + return fmt.Errorf("SSH_CONNECTION_OK not found on %s: %s", s.Runtime.VM.PrivateIP, connectionResult.String()) } s.T.Logf("SSH connectivity to %s verified successfully", s.Runtime.VM.PrivateIP) diff --git a/e2e/types.go b/e2e/types.go index 405cc313a20..18608e833af 100644 --- a/e2e/types.go +++ b/e2e/types.go @@ -16,8 +16,9 @@ import ( "github.com/Azure/agentbaker/e2e/config" "github.com/Azure/agentbaker/pkg/agent/datamodel" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v7" "github.com/stretchr/testify/require" + "golang.org/x/crypto/ssh" ) type Tags struct { @@ -144,6 +145,7 @@ type ScenarioVM struct { VMSS *armcompute.VirtualMachineScaleSet VM *armcompute.VirtualMachineScaleSetVM PrivateIP string + SSHClient *ssh.Client } // Config represents the configuration of an AgentBaker E2E scenario. diff --git a/e2e/validation.go b/e2e/validation.go index 95e4855bf52..aa98bf4e40d 100644 --- a/e2e/validation.go +++ b/e2e/validation.go @@ -15,22 +15,24 @@ import ( v1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func ValidatePodRunning(ctx context.Context, s *Scenario, pod *corev1.Pod) { +func validatePodRunning(ctx context.Context, s *Scenario, pod *corev1.Pod) error { kube := s.Runtime.Cluster.Kube truncatePodName(s.T, pod) start := time.Now() s.T.Logf("creating pod %q", pod.Name) _, err := kube.Typed.CoreV1().Pods(pod.Namespace).Create(ctx, pod, v1.CreateOptions{}) - require.NoErrorf(s.T, err, "failed to create pod %q", pod.Name) - s.T.Cleanup(func() { + if err != nil { + return fmt.Errorf("failed to create pod %q: %v", pod.Name, err) + } + defer func() { ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) defer cancel() err := kube.Typed.CoreV1().Pods(pod.Namespace).Delete(ctx, pod.Name, v1.DeleteOptions{GracePeriodSeconds: to.Ptr(int64(0))}) if err != nil { s.T.Logf("couldn't not delete pod %s: %v", pod.Name, err) } - }) + }() _, err = kube.WaitUntilPodRunning(ctx, pod.Namespace, "", "metadata.name="+pod.Name) if err != nil { @@ -38,12 +40,29 @@ func ValidatePodRunning(ctx context.Context, s *Scenario, pod *corev1.Pod) { if jsonError != nil { jsonString = []byte(jsonError.Error()) } - require.NoErrorf(s.T, err, "failed to wait for pod %q to be in running state. Pod data: %s", pod.Name, jsonString) + return fmt.Errorf("failed to wait for pod %q to be in running state. Pod data: %s, Error: %v", pod.Name, jsonString, err) } timeForReady := time.Since(start) toolkit.LogDuration(ctx, timeForReady, time.Minute, fmt.Sprintf("Time for pod %q to get ready was %s", pod.Name, timeForReady)) s.T.Logf("node health validation: test pod %q is running on node %q", pod.Name, s.Runtime.VM.KubeName) + return nil +} + +func ValidatePodRunningWithRetry(ctx context.Context, s *Scenario, pod *corev1.Pod, maxRetries int) { + var err error + for i := 0; i < maxRetries && err != nil; i++ { + err = validatePodRunning(ctx, s, pod) + if err != nil { + time.Sleep(1 * time.Second) + s.T.Logf("retrying pod %q validation (%d/%d)", pod.Name, i+1, maxRetries) + } + } + require.NoErrorf(s.T, err, "failed to validate pod running %q", pod.Name) +} + +func ValidatePodRunning(ctx context.Context, s *Scenario, pod *corev1.Pod) { + require.NoErrorf(s.T, validatePodRunning(ctx, s, pod), "failed to validate pod running %q", pod.Name) } func ValidateCommonLinux(ctx context.Context, s *Scenario) { @@ -84,7 +103,7 @@ func ValidateCommonLinux(ctx context.Context, s *Scenario) { } execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, "sudo cat /etc/default/kubelet", 0, "could not read kubelet config") - require.NotContains(s.T, execResult.stdout.String(), "--dynamic-config-dir", "kubelet flag '--dynamic-config-dir' should not be present in /etc/default/kubelet\nContents:\n%s") + require.NotContains(s.T, execResult.stdout, "--dynamic-config-dir", "kubelet flag '--dynamic-config-dir' should not be present in /etc/default/kubelet\nContents:\n%s") _ = execScriptOnVMForScenarioValidateExitCode(ctx, s, "sudo curl http://168.63.129.16:32526/vmSettings", 0, "curl to wireserver failed") diff --git a/e2e/validators.go b/e2e/validators.go index 3842a8a7162..48c0620eedd 100644 --- a/e2e/validators.go +++ b/e2e/validators.go @@ -15,7 +15,7 @@ import ( "time" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v7" "github.com/blang/semver" "github.com/tidwall/gjson" @@ -42,7 +42,7 @@ func ValidateTLSBootstrapping(ctx context.Context, s *Scenario) { func validateTLSBootstrappingLinux(ctx context.Context, s *Scenario) { ValidateDirectoryContent(ctx, s, "/var/lib/kubelet", []string{"kubeconfig"}) ValidateDirectoryContent(ctx, s, "/var/lib/kubelet/pki", []string{"kubelet-client-current.pem"}) - kubeletLogs := execScriptOnVMForScenarioValidateExitCode(ctx, s, "sudo journalctl -u kubelet", 0, "could not retrieve kubelet logs with journalctl").stdout.String() + kubeletLogs := execScriptOnVMForScenarioValidateExitCode(ctx, s, "sudo journalctl -u kubelet", 0, "could not retrieve kubelet logs with journalctl").stdout switch { case s.SecureTLSBootstrappingEnabled() && s.Tags.BootstrapTokenFallback: s.T.Logf("will validate bootstrapping mode: secure TLS bootstrapping failure with bootstrap token fallback") @@ -209,13 +209,11 @@ func ValidateSSHServiceEnabled(ctx context.Context, s *Scenario) { // Verify socket-based activation is disabled execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, "systemctl is-active ssh.socket", 3, "could not check ssh.socket status") - stdout := execResult.stdout.String() - require.Contains(s.T, stdout, "inactive", "ssh.socket should be inactive") + require.Contains(s.T, execResult.stdout, "inactive", "ssh.socket should be inactive") // Check that systemd recognizes SSH service should be active at boot execResult = execScriptOnVMForScenarioValidateExitCode(ctx, s, "systemctl is-enabled ssh.service", 0, "could not check ssh.service status") - stdout = execResult.stdout.String() - require.Contains(s.T, stdout, "enabled", "ssh.service should be enabled at boot") + require.Contains(s.T, execResult.stdout, "enabled", "ssh.service should be enabled at boot") } func ValidateDirectoryContent(ctx context.Context, s *Scenario, path string, files []string) { @@ -233,9 +231,8 @@ func ValidateDirectoryContent(ctx context.Context, s *Scenario, path string, fil } } execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(steps, "\n"), 0, "could not get directory contents") - stdout := execResult.stdout.String() for _, file := range files { - require.Contains(s.T, stdout, file, "expected to find file %s within directory %s, but did not.\nDirectory contents:\n%s", file, path, stdout) + require.Contains(s.T, execResult.stdout, file, "expected to find file %s within directory %s, but did not.\nDirectory contents:\n%s", file, path, execResult.stdout) } } @@ -250,9 +247,8 @@ func ValidateSysctlConfig(ctx context.Context, s *Scenario, customSysctls map[st fmt.Sprintf("sudo sysctl %s | sed -E 's/([0-9])\\s+([0-9])/\\1 \\2/g'", strings.Join(keysToCheck, " ")), } execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "systmctl command failed") - stdout := execResult.stdout.String() for name, value := range customSysctls { - require.Contains(s.T, stdout, fmt.Sprintf("%s = %v", name, value), "expected to find %s set to %v, but was not.\nStdout:\n%s", name, value, stdout) + require.Contains(s.T, execResult.stdout, fmt.Sprintf("%s = %v", name, value), "expected to find %s set to %v, but was not.\nStdout:\n%s", name, value, execResult.stdout) } } @@ -263,8 +259,7 @@ func ValidateNvidiaSMINotInstalled(ctx context.Context, s *Scenario) { "sudo nvidia-smi", } execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 1, "") - stderr := execResult.stderr.String() - require.Contains(s.T, stderr, "nvidia-smi: command not found", "expected stderr to contain 'nvidia-smi: command not found', but got %q", stderr) + require.Contains(s.T, execResult.stderr, "nvidia-smi: command not found", "expected stderr to contain 'nvidia-smi: command not found', but got %q", execResult.stderr) } func ValidateNvidiaSMIInstalled(ctx context.Context, s *Scenario) { @@ -352,7 +347,7 @@ func fileExist(ctx context.Context, s *Scenario, fileName string) bool { fmt.Sprintf("if (Test-Path -Path '%s') { exit 0 } else { exit 1 }", fileName), } execResult := execScriptOnVMForScenario(ctx, s, strings.Join(steps, "\n")) - s.T.Logf("stdout: %s\nstderr: %s", execResult.stdout.String(), execResult.stderr.String()) + s.T.Logf("stdout: %s\nstderr: %s", execResult.stdout, execResult.stderr) return execResult.exitCode == "0" } else { steps := []string{ @@ -517,7 +512,7 @@ func ValidateUlimitSettings(ctx context.Context, s *Scenario, ulimits map[string execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, command, 0, "could not read containerd.service file") for name, value := range ulimits { - require.Contains(s.T, execResult.stdout.String(), fmt.Sprintf("%s=%v", name, value), "expected to find %s set to %v, but was not", name, value) + require.Contains(s.T, execResult.stdout, fmt.Sprintf("%s=%v", name, value), "expected to find %s set to %v, but was not", name, value) } } @@ -532,7 +527,7 @@ func execOnVMForScenarioOnUnprivilegedPod(ctx context.Context, s *Scenario, cmd func execScriptOnVMForScenario(ctx context.Context, s *Scenario, cmd string) *podExecResult { s.T.Helper() - result, err := execScriptOnVm(ctx, s, s.Runtime.VM.PrivateIP, s.Runtime.Cluster.DebugPod.Name, cmd) + result, err := execScriptOnVm(ctx, s, s.Runtime.VM, cmd) require.NoError(s.T, err, "failed to execute command on VM") return result } @@ -543,7 +538,7 @@ func execScriptOnVMForScenarioValidateExitCode(ctx context.Context, s *Scenario, expectedExitCodeStr := fmt.Sprint(expectedExitCode) if expectedExitCodeStr != execResult.exitCode { - s.T.Logf("Command: %s\nStdout: %s\nStderr: %s", cmd, execResult.stdout.String(), execResult.stderr.String()) + s.T.Logf("Command: %s\nStdout: %s\nStderr: %s", cmd, execResult.stdout, execResult.stderr) s.T.Fatalf("expected exit code %s, but got %s\nCommand: %s\n%s", expectedExitCodeStr, execResult.exitCode, cmd, additionalErrorMessage) } return execResult @@ -563,7 +558,7 @@ func ValidateInstalledPackageVersion(ctx context.Context, s *Scenario, component } }() execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, installedCommand, 0, "could not get package list") - for _, line := range strings.Split(execResult.stdout.String(), "\n") { + for _, line := range strings.Split(execResult.stdout, "\n") { if strings.Contains(line, component) && strings.Contains(line, version) { s.T.Logf("found %s %s in the installed packages", component, version) return @@ -575,7 +570,7 @@ func ValidateInstalledPackageVersion(ctx context.Context, s *Scenario, component func ValidateKubeletNodeIP(ctx context.Context, s *Scenario) { s.T.Helper() execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, "sudo cat /etc/default/kubelet", 0, "could not read kubelet config") - stdout := execResult.stdout.String() + stdout := execResult.stdout // Search for "--node-ip" flag and its value. matches := regexp.MustCompile(`--node-ip=([a-zA-Z0-9.,]*)`).FindStringSubmatch(stdout) @@ -606,7 +601,7 @@ func ValidateMultipleKubeProxyVersionsExist(ctx context.Context, s *Scenario) { return } - versions := bytes.NewBufferString(strings.TrimSpace(execResult.stdout.String())) + versions := bytes.NewBufferString(strings.TrimSpace(execResult.stdout)) versionMap := make(map[string]struct{}) for _, version := range strings.Split(versions.String(), "\n") { if version != "" { @@ -628,7 +623,7 @@ func ValidateKubeletHasNotStopped(ctx context.Context, s *Scenario) { s.T.Helper() command := "sudo journalctl -u kubelet" execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, command, 0, "could not retrieve kubelet logs with journalctl") - stdout := strings.ToLower(execResult.stdout.String()) + stdout := strings.ToLower(execResult.stdout) assert.NotContains(s.T, stdout, "stopped kubelet") assert.Contains(s.T, stdout, "started kubelet") } @@ -645,7 +640,7 @@ func ValidateKubeletHasFlags(ctx context.Context, s *Scenario, filePath string) s.T.Helper() execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, "sudo journalctl -u kubelet", 0, "could not retrieve kubelet logs with journalctl") configFileFlags := fmt.Sprintf("FLAG: --config=\"%s\"", filePath) - require.Containsf(s.T, execResult.stdout.String(), configFileFlags, "expected to find flag %s, but not found", "config") + require.Containsf(s.T, execResult.stdout, configFileFlags, "expected to find flag %s, but not found", "config") } // Waits until the specified resource is available on the given node. @@ -692,7 +687,7 @@ func ValidateContainerd2Properties(ctx context.Context, s *Scenario, versions [] execResult := execOnVMForScenarioOnUnprivilegedPod(ctx, s, "containerd config dump ") // validate containerd config dump has no warnings - require.NotContains(s.T, execResult.stdout.String(), "level=warning", "do not expect warning message when converting config file %", execResult.stdout.String()) + require.NotContains(s.T, execResult.stdout, "level=warning", "do not expect warning message when converting config file %", execResult.stdout) } func ValidateContainerRuntimePlugins(ctx context.Context, s *Scenario) { @@ -950,7 +945,7 @@ func ValidateWindowsProcessHasCliArguments(ctx context.Context, s *Scenario, pro podExecResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(steps, "\n"), 0, "could not validate command has parameters - might mean file does not have params, might mean something went wrong") - actualArgs := strings.Split(podExecResult.stdout.String(), " ") + actualArgs := strings.Split(podExecResult.stdout, " ") for i := range arguments { expectedArgument := arguments[i] @@ -971,7 +966,7 @@ func validateWindowsProccessArgumentString(ctx context.Context, s *Scenario, pro fmt.Sprintf("(Get-CimInstance Win32_Process -Filter \"name='%[1]s'\")[0].CommandLine", processName), } podExecResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(steps, "\n"), 0, "could not validate command argument string - might mean file does not have params, might mean something went wrong") - argString := podExecResult.stdout.String() + argString := podExecResult.stdout for _, str := range substrings { assert(s.T, argString, str) } @@ -989,7 +984,7 @@ func ValidateWindowsVersionFromWindowsSettings(ctx context.Context, s *Scenario, osMajorVersion := versionSliced[0] podExecResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(steps, "\n"), 0, "could not validate command has parameters - might mean file does not have params, might mean something went wrong") - podExecResultStdout := strings.TrimSpace(podExecResult.stdout.String()) + podExecResultStdout := strings.TrimSpace(podExecResult.stdout) s.T.Logf("Found windows version in windows_settings: \"%s\": \"%s\" (\"%s\")", windowsVersion, osMajorVersion, osVersion) s.T.Logf("Windows version returned from VM \"%s\"", podExecResultStdout) @@ -1004,7 +999,7 @@ func ValidateWindowsProductName(ctx context.Context, s *Scenario, productName st } podExecResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(steps, "\n"), 0, "could not validate command has parameters - might mean file does not have params, might mean something went wrong") - podExecResultStdout := strings.TrimSpace(podExecResult.stdout.String()) + podExecResultStdout := strings.TrimSpace(podExecResult.stdout) require.Contains(s.T, podExecResultStdout, productName) } @@ -1016,7 +1011,7 @@ func ValidateWindowsDisplayVersion(ctx context.Context, s *Scenario, displayVers } podExecResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(steps, "\n"), 0, "could not validate command has parameters - might mean file does not have params, might mean something went wrong") - podExecResultStdout := strings.TrimSpace(podExecResult.stdout.String()) + podExecResultStdout := strings.TrimSpace(podExecResult.stdout) s.T.Logf("Windows display version returned from VM \"%s\". Expected display version \"%s\"", podExecResultStdout, displayVersion) @@ -1086,9 +1081,9 @@ func dllLoadedWindows(ctx context.Context, s *Scenario, dllName string) bool { fmt.Sprintf("tasklist /m %s", dllName), } execResult := execScriptOnVMForScenario(ctx, s, strings.Join(steps, "\n")) - dllLoaded := strings.Contains(execResult.stdout.String(), dllName) + dllLoaded := strings.Contains(execResult.stdout, dllName) - s.T.Logf("stdout: %s\nstderr: %s", execResult.stdout.String(), execResult.stderr.String()) + s.T.Logf("stdout: %s\nstderr: %s", execResult.stdout, execResult.stderr) return dllLoaded } @@ -1110,7 +1105,7 @@ func GetFieldFromJsonObjectOnNode(ctx context.Context, s *Scenario, fileName str podExecResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(steps, "\n"), 0, "could not validate command has parameters - might mean file does not have params, might mean something went wrong") - return podExecResult.stdout.String() + return podExecResult.stdout } // ValidateTaints checks if the node has the expected taints that are set in the kubelet config with --register-with-taints flag @@ -1118,14 +1113,14 @@ func ValidateTaints(ctx context.Context, s *Scenario, expectedTaints string) { s.T.Helper() node, err := s.Runtime.Cluster.Kube.Typed.CoreV1().Nodes().Get(ctx, s.Runtime.VM.KubeName, metav1.GetOptions{}) require.NoError(s.T, err, "failed to get node %q", s.Runtime.VM.KubeName) - actualTaints := "" - for i, taint := range node.Spec.Taints { - actualTaints += fmt.Sprintf("%s=%s:%s", taint.Key, taint.Value, taint.Effect) - // add a comma if it's not the last element - if i < len(node.Spec.Taints)-1 { - actualTaints += "," + var taints []string + for _, taint := range node.Spec.Taints { + if strings.Contains(taint.Key, "node.kubernetes.io") { + continue } + taints = append(taints, fmt.Sprintf("%s=%s:%s", taint.Key, taint.Value, taint.Effect)) } + actualTaints := strings.Join(taints, ",") require.Equal(s.T, expectedTaints, actualTaints, "expected node %q to have taint %q, but got %q", s.Runtime.VM.KubeName, expectedTaints, actualTaints) } @@ -1174,8 +1169,8 @@ func ValidateLocalDNSResolution(ctx context.Context, s *Scenario, server string) testdomain := "bing.com" command := fmt.Sprintf("dig %s +timeout=1 +tries=1", testdomain) execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, command, 0, "dns resolution failed") - assert.Contains(s.T, execResult.stdout.String(), "status: NOERROR") - assert.Contains(s.T, execResult.stdout.String(), fmt.Sprintf("SERVER: %s", server)) + assert.Contains(s.T, execResult.stdout, "status: NOERROR") + assert.Contains(s.T, execResult.stdout, fmt.Sprintf("SERVER: %s", server)) } // ValidateJournalctlOutput checks if specific content exists in the systemd service logs @@ -1464,7 +1459,7 @@ func ValidateMIGModeEnabled(ctx context.Context, s *Scenario) { } execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "MIG mode is not enabled") - stdout := strings.TrimSpace(execResult.stdout.String()) + stdout := strings.TrimSpace(execResult.stdout) s.T.Logf("MIG mode status: %s", stdout) require.Contains(s.T, stdout, "Enabled", "expected MIG mode to be enabled, but got: %s", stdout) s.T.Logf("MIG mode is enabled") @@ -1483,7 +1478,7 @@ func ValidateMIGInstancesCreated(ctx context.Context, s *Scenario, migProfile st } execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "MIG instances with profile "+migProfile+" were not found") - stdout := execResult.stdout.String() + stdout := execResult.stdout require.Contains(s.T, stdout, migProfile, "expected to find MIG profile %s in output, but did not.\nOutput:\n%s", migProfile, stdout) require.NotContains(s.T, stdout, "No MIG-enabled devices found", "no MIG devices were created.\nOutput:\n%s", stdout) s.T.Logf("MIG instances with profile %s are created", migProfile) @@ -1502,7 +1497,7 @@ func ValidateIPTablesCompatibleWithCiliumEBPF(ctx context.Context, s *Scenario) command := fmt.Sprintf("sudo iptables -t %s -S", table) execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, command, 0, fmt.Sprintf("failed to get iptables rules for table %s", table)) - stdout := execResult.stdout.String() + stdout := execResult.stdout rules := strings.Split(strings.TrimSpace(stdout), "\n") // Get patterns for this table @@ -1616,7 +1611,7 @@ func ValidateAppArmorBasic(ctx context.Context, s *Scenario) { "cat /sys/module/apparmor/parameters/enabled", } execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "failed to check AppArmor kernel parameter") - stdout := strings.TrimSpace(execResult.stdout.String()) + stdout := strings.TrimSpace(execResult.stdout) require.Equal(s.T, "Y", stdout, "expected AppArmor to be enabled in kernel") // Check if apparmor.service is active @@ -1625,7 +1620,7 @@ func ValidateAppArmorBasic(ctx context.Context, s *Scenario) { "systemctl is-active apparmor.service", } execResult = execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "apparmor.service is not active") - stdout = strings.TrimSpace(execResult.stdout.String()) + stdout = strings.TrimSpace(execResult.stdout) require.Equal(s.T, "active", stdout, "expected apparmor.service to be active") // Check if AppArmor is enforcing by checking current process profile diff --git a/e2e/vmss.go b/e2e/vmss.go index 557e53c623f..967dc4efeeb 100644 --- a/e2e/vmss.go +++ b/e2e/vmss.go @@ -2,12 +2,12 @@ package e2e import ( "context" + crand "crypto/rand" - "crypto/rsa" - "crypto/x509" + "encoding/base64" "encoding/json" - "encoding/pem" + "errors" "fmt" "io" @@ -25,9 +25,8 @@ import ( "github.com/Azure/agentbaker/pkg/agent/datamodel" "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v7" "github.com/stretchr/testify/require" - "golang.org/x/crypto/ssh" ) const ( @@ -279,14 +278,10 @@ func CreateVMSS(ctx context.Context, s *Scenario, resourceGroupName string) (*Sc } s.T.Cleanup(func() { - cleanupVMSS(ctx, s, vm.PrivateIP) + defer cleanupBastionTunnel(vm.SSHClient) + cleanupVMSS(ctx, s, vm) }) - err = uploadSSHKey(ctx, s, vm.PrivateIP) - if err != nil { - return vm, fmt.Errorf("failed to upload ssh key: %w", err) - } - vmssResp, err := operation.PollUntilDone(ctx, config.DefaultPollUntilDoneOptions) if err != nil { return vm, err @@ -298,10 +293,18 @@ func CreateVMSS(ctx context.Context, s *Scenario, resourceGroupName string) (*Sc return vm, fmt.Errorf("failed to wait for VM to reach running state: %w", err) } + if !s.Config.SkipSSHConnectivityValidation { + vm.SSHClient, err = DialSSHOverBastion(ctx, s.Runtime.Cluster.Bastion, vm.PrivateIP, config.VMSSHPrivateKey) + if err != nil { + return vm, fmt.Errorf("failed to start bastion tunnel: %w", err) + } + } + return &ScenarioVM{ VMSS: &vmssResp.VirtualMachineScaleSet, PrivateIP: vm.PrivateIP, VM: vm.VM, + SSHClient: vm.SSHClient, }, nil } @@ -438,19 +441,19 @@ func skipTestIfSKUNotAvailableErr(t testing.TB, err error) { } } -func cleanupVMSS(ctx context.Context, s *Scenario, privateIP string) { +func cleanupVMSS(ctx context.Context, s *Scenario, vm *ScenarioVM) { // original context can be cancelled, but we still want to collect the logs ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Minute) defer cancel() defer deleteVMSS(ctx, s) - extractLogsFromVM(ctx, s, privateIP) + extractLogsFromVM(ctx, s, vm) } -func extractLogsFromVM(ctx context.Context, s *Scenario, privateIP string) { +func extractLogsFromVM(ctx context.Context, s *Scenario, vm *ScenarioVM) { if s.IsWindows() { extractLogsFromVMWindows(ctx, s) } else { - err := extractLogsFromVMLinux(ctx, s, privateIP) + err := extractLogsFromVMLinux(ctx, s, vm) if err != nil { s.T.Logf("failed to extract logs from VM: %s", err) } else { @@ -524,7 +527,7 @@ func extractBootDiagnostics(ctx context.Context, s *Scenario) error { return nil } -func extractLogsFromVMLinux(ctx context.Context, s *Scenario, privateIP string) error { +func extractLogsFromVMLinux(ctx context.Context, s *Scenario, vm *ScenarioVM) error { syslogHandle := "syslog" if s.VHD.OS == config.OSMariner || s.VHD.OS == config.OSAzureLinux { syslogHandle = "messages" @@ -544,14 +547,15 @@ func extractLogsFromVMLinux(ctx context.Context, s *Scenario, privateIP string) commandList["secure-tls-bootstrap.log"] = "sudo cat /var/log/azure/aks/secure-tls-bootstrap.log" } - pod, err := s.Runtime.Cluster.Kube.GetHostNetworkDebugPod(ctx) - if err != nil { - return fmt.Errorf("failed to get host network debug pod: %w", err) + isAzureCNI, err := s.Runtime.Cluster.IsAzureCNI() + if err == nil && isAzureCNI { + commandList["azure-vnet.log"] = "sudo cat /var/log/azure-vnet.log" + commandList["azure-vnet-ipam.log"] = "sudo cat /var/log/azure-vnet-ipam.log" } var logFiles = map[string]string{} for file, sourceCmd := range commandList { - execResult, err := execScriptOnVm(ctx, s, privateIP, pod.Name, sourceCmd) + execResult, err := execScriptOnVm(ctx, s, vm, sourceCmd) if err != nil { s.T.Logf("error executing %s: %s", sourceCmd, err) continue @@ -718,7 +722,7 @@ func deleteVMSS(ctx context.Context, s *Scenario) { defer cancel() if config.Config.KeepVMSS { s.T.Logf("vmss %q will be retained for debugging purposes, please make sure to manually delete it later", s.Runtime.VMSSName) - if err := writeToFile(s.T, "sshkey", string(SSHKeyPrivate)); err != nil { + if err := writeToFile(s.T, "sshkey", string(config.VMSSHPrivateKey)); err != nil { s.T.Logf("failed to write retained vmss %s private ssh key to disk: %s", s.Runtime.VMSSName, err) } return @@ -763,49 +767,6 @@ func addPodIPConfigsForAzureCNI(vmss *armcompute.VirtualMachineScaleSet, vmssNam return nil } -func mustGetNewRSAKeyPair() ([]byte, []byte) { - private, public, err := getNewRSAKeyPair() - if err != nil { - panic(fmt.Sprintf("failed to generate RSA key pair: %v", err)) - } - return private, public -} - -// Returns a newly generated RSA public/private key pair with the private key in PEM format. -func getNewRSAKeyPair() (privatePEMBytes []byte, publicKeyBytes []byte, e error) { - privateKey, err := rsa.GenerateKey(crand.Reader, 4096) - if err != nil { - return nil, nil, fmt.Errorf("failed to create rsa private key: %w", err) - } - - err = privateKey.Validate() - if err != nil { - return nil, nil, fmt.Errorf("failed to validate rsa private key: %w", err) - } - - publicRsaKey, err := ssh.NewPublicKey(&privateKey.PublicKey) - if err != nil { - return nil, nil, fmt.Errorf("failed to convert private to public key: %w", err) - } - - publicKeyBytes = ssh.MarshalAuthorizedKey(publicRsaKey) - - // Get ASN.1 DER format - privDER := x509.MarshalPKCS1PrivateKey(privateKey) - - // pem.Block - privBlock := pem.Block{ - Type: "RSA PRIVATE KEY", - Headers: nil, - Bytes: privDER, - } - - // Private key in PEM format - privatePEMBytes = pem.EncodeToMemory(&privBlock) - - return -} - func generateVMSSNameLinux(t testing.TB) string { name := fmt.Sprintf("%s-%s-%s", randomLowercaseString(4), time.Now().Format(time.DateOnly), t.Name()) name = strings.ReplaceAll(name, "_", "") @@ -857,7 +818,7 @@ func getBaseVMSSModel(s *Scenario, customData, cseCmd string) armcompute.Virtual SSH: &armcompute.SSHConfiguration{ PublicKeys: []*armcompute.SSHPublicKey{ { - KeyData: to.Ptr(string(SSHKeyPublic)), + KeyData: to.Ptr(string(config.VMSSHPublicKey)), Path: to.Ptr("/home/azureuser/.ssh/authorized_keys"), }, }, diff --git a/go.mod b/go.mod index 9192df45d61..a49113ab9de 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/Azure/agentbaker -go 1.23.0 +go 1.24.0 require ( github.com/Azure/go-autorest/autorest/to v0.4.1 @@ -39,9 +39,9 @@ require ( github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/spf13/pflag v1.0.9 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/net v0.43.0 // indirect - golang.org/x/sys v0.35.0 // indirect - golang.org/x/text v0.28.0 // indirect + golang.org/x/net v0.48.0 // indirect + golang.org/x/sys v0.39.0 // indirect + golang.org/x/text v0.32.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect ) diff --git a/go.sum b/go.sum index 0a3530f18a2..e16b78e7235 100644 --- a/go.sum +++ b/go.sum @@ -116,8 +116,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= -golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= +golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= +golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -131,17 +131,17 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= -golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= -golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= -golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= +golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= +golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/parts/common/components.json b/parts/common/components.json index 3fc1f2ff566..46b76d8957f 100644 --- a/parts/common/components.json +++ b/parts/common/components.json @@ -449,7 +449,7 @@ "renovateTag": "registry=https://mcr.microsoft.com, name=oss/kubernetes/azure-cloud-node-manager", "latestVersion": "v1.33.6-windows-hpc", "previousLatestVersion": "v1.33.3-windows-hpc" - } + } ] }, { @@ -458,7 +458,29 @@ "multiArchVersionsV2": [ { "renovateTag": "registry=https://mcr.microsoft.com, name=oss/v2/kubernetes/azure-cloud-node-manager", - "latestVersion": "v1.34.3" + "latestVersion": "v1.32.11-1" + }, + { + "renovateTag": "registry=https://mcr.microsoft.com, name=oss/v2/kubernetes/azure-cloud-node-manager", + "latestVersion": "v1.33.6-1" + }, + { + "renovateTag": "registry=https://mcr.microsoft.com, name=oss/v2/kubernetes/azure-cloud-node-manager", + "latestVersion": "v1.34.3-1" + } + ], + "windowsVersions": [ + { + "renovateTag": "registry=https://mcr.microsoft.com, name=oss/v2/kubernetes/azure-cloud-node-manager", + "latestVersion": "v1.32.11-windows-hpc-1" + }, + { + "renovateTag": "registry=https://mcr.microsoft.com, name=oss/v2/kubernetes/azure-cloud-node-manager", + "latestVersion": "v1.33.6-windows-hpc-1" + }, + { + "renovateTag": "registry=https://mcr.microsoft.com, name=oss/v2/kubernetes/azure-cloud-node-manager", + "latestVersion": "v1.34.3-windows-hpc-1" } ] }, @@ -859,6 +881,31 @@ } } }, + { + "name": "windows credential provider dalec", + "windowsDownloadLocation": "c:\\akse-cache\\azure-acr-credential-provider\\", + "downloadURIs": { + "windows": { + "default": { + "versionsV2": [ + { + "renovateTag": "", + "latestVersion": "1.32.11-1" + }, + { + "renovateTag": "", + "latestVersion": "1.33.6-1" + }, + { + "renovateTag": "", + "latestVersion": "1.34.3-1" + } + ], + "downloadURL": "https://packages.aks.azure.com/dalec-packages/azure-acr-credential-provider/$($version.Split('-')[0])/windows/amd64/azure-acr-credential-provider_${version}_amd64.zip" + } + } + } + }, { "name": "windows calico", "windowsDownloadLocation": "c:\\akse-cache\\calico\\", @@ -1490,31 +1537,61 @@ "ubuntu": { "r2404": { "versionsV2": [ + { + "k8sVersion": "1.32", + "renovateTag": "name=azure-acr-credential-provider, repository=production, os=ubuntu, release=24.04", + "latestVersion": "1.32.11-ubuntu24.04u1" + }, + { + "k8sVersion": "1.33", + "renovateTag": "name=azure-acr-credential-provider, repository=production, os=ubuntu, release=24.04", + "latestVersion": "1.33.6-ubuntu24.04u1" + }, { "k8sVersion": "1.34", "renovateTag": "name=azure-acr-credential-provider, repository=production, os=ubuntu, release=24.04", - "latestVersion": "1.34.1-ubuntu24.04u3", - "previousLatestVersion": "1.34.0-ubuntu24.04u2" + "latestVersion": "1.34.3-ubuntu24.04u1", + "previousLatestVersion": "1.34.1-ubuntu24.04u3" } ] }, "r2204": { "versionsV2": [ + { + "k8sVersion": "1.32", + "renovateTag": "name=azure-acr-credential-provider, repository=production, os=ubuntu, release=22.04", + "latestVersion": "1.32.11-ubuntu22.04u1" + }, + { + "k8sVersion": "1.33", + "renovateTag": "name=azure-acr-credential-provider, repository=production, os=ubuntu, release=22.04", + "latestVersion": "1.33.6-ubuntu22.04u1" + }, { "k8sVersion": "1.34", "renovateTag": "name=azure-acr-credential-provider, repository=production, os=ubuntu, release=22.04", - "latestVersion": "1.34.1-ubuntu22.04u3", - "previousLatestVersion": "1.34.0-ubuntu22.04u2" + "latestVersion": "1.34.3-ubuntu22.04u1", + "previousLatestVersion": "1.34.1-ubuntu22.04u3" } ] }, "r2004": { "versionsV2": [ + { + "k8sVersion": "1.32", + "renovateTag": "name=azure-acr-credential-provider, repository=production, os=ubuntu, release=20.04", + "latestVersion": "1.32.11-ubuntu20.04u1" + }, + { + "k8sVersion": "1.33", + "renovateTag": "name=azure-acr-credential-provider, repository=production, os=ubuntu, release=20.04", + "latestVersion": "1.33.6-ubuntu20.04u1" + }, { "k8sVersion": "1.34", "renovateTag": "name=azure-acr-credential-provider, repository=production, os=ubuntu, release=20.04", - "latestVersion": "1.34.1-ubuntu20.04u3", - "previousLatestVersion": "1.34.0-ubuntu20.04u2" + "latestVersion": "1.34.3-ubuntu20.04u1", + "previousLatestVersion": "1.34.1-ubuntu20.04u3" } ] } @@ -1522,11 +1599,21 @@ "azurelinux": { "v3.0": { "versionsV2": [ + { + "k8sVersion": "1.32", + "renovateTag": "name=azure-acr-credential-provider, repository=production, os=azurelinux, release=3.0", + "latestVersion": "1.32.11-1.azl3" + }, + { + "k8sVersion": "1.33", + "renovateTag": "name=azure-acr-credential-provider, repository=production, os=azurelinux, release=3.0", + "latestVersion": "1.33.6-1.azl3" + }, { "k8sVersion": "1.34", "renovateTag": "name=azure-acr-credential-provider, repository=production, os=azurelinux, release=3.0", - "latestVersion": "1.34.1-1.azl3", - "previousLatestVersion": "1.34.0-2.azl3" + "latestVersion": "1.34.3-1.azl3", + "previousLatestVersion": "1.34.1-1.azl3" } ] } diff --git a/parts/linux/cloud-init/artifacts/cse_helpers.sh b/parts/linux/cloud-init/artifacts/cse_helpers.sh index 6822d0f7093..0615e8aa809 100755 --- a/parts/linux/cloud-init/artifacts/cse_helpers.sh +++ b/parts/linux/cloud-init/artifacts/cse_helpers.sh @@ -688,6 +688,17 @@ should_enforce_kube_pmc_install() { echo "${should_enforce,,}" } +update_kubelet_eviction_flags() { + set -x + body=$(curl -fsSL -H "Metadata: true" --noproxy "*" "http://169.254.169.254/metadata/instance?api-version=2021-02-01") + ret=$? + if [ "$ret" -ne 0 ]; then + return $ret + fi + eviction_flags=$(echo "$body" | jq -r '.compute.tagsList[] | select(.name == "UpdateKubeletEvictionFlags") | .value') + echo "${eviction_flags,,}" +} + e2e_mock_azure_china_cloud() { set -x body=$(curl -fsSL -H "Metadata: true" --noproxy "*" "http://169.254.169.254/metadata/instance?api-version=2021-02-01") @@ -1001,6 +1012,33 @@ updateKubeBinaryRegistryURL() { fi } +extractKubeletEvictionFlags() { + local eviction_flags_string=$1 + local eviction_flags="" + + if grep -e "|" <<< "$eviction_flags_string" > /dev/null 2>&1; then + while grep -e "=" <<< "$eviction_flags_string" > /dev/null 2>&1; do + local flag_key_value="${eviction_flags_string%%|*}" + if [ -n "$eviction_flags" ]; then + eviction_flags="${eviction_flags} ${flag_key_value}" + else + eviction_flags="${flag_key_value}" + fi + eviction_flags_string="${eviction_flags_string/${flag_key_value}/}" + done + else + eviction_flags="${eviction_flags_string}" + fi + + # Trim leading spaces (if any) and echo result for the caller to capture + if [ -n "$eviction_flags" ]; then + local trimmed="${eviction_flags#${eviction_flags%%[! ]*}}" + echo "$trimmed" + else + echo "" + fi +} + # removes the specified FLAG_STRING (which should be in the form of 'key=value') from KUBELET_FLAGS removeKubeletFlag() { local FLAG_STRING=$1 diff --git a/parts/linux/cloud-init/artifacts/cse_install.sh b/parts/linux/cloud-init/artifacts/cse_install.sh index 99d6edcb679..67776cb3ca3 100755 --- a/parts/linux/cloud-init/artifacts/cse_install.sh +++ b/parts/linux/cloud-init/artifacts/cse_install.sh @@ -535,7 +535,12 @@ installToolFromBootstrapProfileRegistry() { # Try to pull distro-specific packages (e.g., .deb for Ubuntu) from registry local download_root="/tmp/kubernetes/downloads" # /opt folder will return permission error - tool_package_url="${registry_server}/aks/packages/kubernetes/${tool_name}:v${version}" + version_tag="${version}" + if [ "${version}" != "v*" ]; then + version_tag="v${version_tag}" + fi + version_tag="${version_tag/\~/-}" + tool_package_url="${registry_server}/aks/packages/kubernetes/${tool_name}:${version_tag}" tool_download_dir="${download_root}/${tool_name}" mkdir -p "${tool_download_dir}" diff --git a/parts/linux/cloud-init/artifacts/cse_main.sh b/parts/linux/cloud-init/artifacts/cse_main.sh index f4e171cf52a..b75fca274eb 100755 --- a/parts/linux/cloud-init/artifacts/cse_main.sh +++ b/parts/linux/cloud-init/artifacts/cse_main.sh @@ -164,6 +164,17 @@ function basePrep { # TODO: Remove tag and usages once 1.34.0 is GA. export -f should_enforce_kube_pmc_install SHOULD_ENFORCE_KUBE_PMC_INSTALL=$(retrycmd_silent 10 1 10 bash -cx should_enforce_kube_pmc_install) + + # UpdateKubeletEvictionFlags is a nodepool or cluster tag we curl from IMDS. + export -f update_kubelet_eviction_flags + export -f extractKubeletEvictionFlags + RAW_EVICTION_FLAGS=$(retrycmd_silent 10 1 10 bash -cx update_kubelet_eviction_flags) + if [ -n "$RAW_EVICTION_FLAGS" ]; then + UPDATED_KUBELET_EVICTION_FLAGS=$(extractKubeletEvictionFlags "$RAW_EVICTION_FLAGS") + else + UPDATED_KUBELET_EVICTION_FLAGS="" + fi + logs_to_events "AKS.CSE.configureKubeletAndKubectl" configureKubeletAndKubectl createKubeManifestDir @@ -254,6 +265,13 @@ Environment="KUBELET_CONTAINER_RUNTIME_FLAG=--container-runtime=remote" EOF fi + if [ -n "${UPDATED_KUBELET_EVICTION_FLAGS}" ]; then + tee "/etc/systemd/system/kubelet.service.d/10-kubelet-eviction-flags.conf" > /dev/null < /dev/null <