Skip to content

Commit 14bad23

Browse files
awesomenixCopilot
awesomenix
and
Copilot
committed
chore: update ubuntu scriptless e2e flow
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
1 parent ed3d302 commit 14bad23

File tree

6 files changed

+71
-19
lines changed

6 files changed

+71
-19
lines changed

‎e2e/scenario_test.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -409,8 +409,9 @@ func Test_Ubuntu2204_Scriptless(t *testing.T) {
409409
RunScenario(t, &Scenario{
410410
Description: "tests that a new ubuntu 2204 node using self contained installer can be properly bootstrapped",
411411
Config: Config{
412-
Cluster: ClusterKubenet,
413-
VHD: config.VHDUbuntu2204Gen2Containerd,
412+
Cluster: ClusterKubenet,
413+
VHD: config.VHDUbuntu2204Gen2Containerd,
414+
UseCustomDataOnlyProvisioning: true,
414415
Validator: func(ctx context.Context, s *Scenario) {
415416
ValidateFileHasContent(ctx, s, "/var/log/azure/aks-node-controller.log", "aks-node-controller finished successfully")
416417
},

‎e2e/test_helpers.go‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -301,8 +301,10 @@ func prepareAKSNode(ctx context.Context, s *Scenario) (*ScenarioVM, error) {
301301
require.NoError(s.T, err, "create vmss %q, check %s for vm logs", s.Runtime.VMSSName, testDir(s.T))
302302
}
303303

304-
err = getCustomScriptExtensionStatus(s, scenarioVM.VM)
305-
require.NoError(s.T, err)
304+
if !s.Config.UseCustomDataOnlyProvisioning {
305+
err = getCustomScriptExtensionStatus(s, scenarioVM.VM)
306+
require.NoError(s.T, err)
307+
}
306308

307309
if !s.Config.SkipDefaultValidation {
308310
vmssCreatedAt := time.Now() // Record the start time

‎e2e/types.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -163,6 +163,11 @@ type Config struct {
163163
// AKSNodeConfigMutator if defined then aks-node-controller will be used to provision nodes
164164
AKSNodeConfigMutator func(*aksnodeconfigv1.Configuration)
165165

166+
// UseCustomDataOnlyProvisioning switches an AKSNodeConfig scenario to a CustomData-only E2E flow.
167+
// It omits the VMSS Custom Script Extension and uses CustomData to run aks-node-controller provision
168+
// directly during cloud-init instead.
169+
UseCustomDataOnlyProvisioning bool
170+
166171
// VMConfigMutator is a function which mutates the base VMSS model according to the scenario's requirements
167172
VMConfigMutator func(*armcompute.VirtualMachineScaleSet)
168173

‎e2e/vmss.go‎

Lines changed: 42 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
"testing"
1919
"time"
2020

21+
aksnodeconfigv1 "github.com/Azure/agentbaker/aks-node-controller/pkg/gen/aksnodeconfig/v1"
2122
"github.com/Azure/agentbaker/aks-node-controller/pkg/nodeconfigutils"
2223
"github.com/Azure/agentbaker/e2e/config"
2324
"github.com/Azure/agentbaker/e2e/toolkit"
@@ -144,26 +145,55 @@ coreos:
144145
return base64.StdEncoding.EncodeToString([]byte(customDataYAML)), nil
145146
}
146147

148+
// CustomDataWithDirectProvision writes the aks-node-controller config to a test-only path and
149+
// runs provisioning directly from cloud-init runcmd, avoiding the VMSS Custom Script Extension.
150+
func CustomDataWithDirectProvision(cfg *aksnodeconfigv1.Configuration) (string, error) {
151+
cloudConfigTemplate := `#cloud-config
152+
write_files:
153+
- path: /opt/azure/containers/aks-node-controller-config-runcmd.json
154+
permissions: "0755"
155+
owner: root
156+
content: !!binary |
157+
%s
158+
runcmd:
159+
- /opt/azure/containers/aks-node-controller provision --provision-config=/opt/azure/containers/aks-node-controller-config-runcmd.json
160+
`
161+
162+
aksNodeConfigJSON, err := nodeconfigutils.MarshalConfigurationV1(cfg)
163+
if err != nil {
164+
return "", fmt.Errorf("failed to marshal nbc, error: %w", err)
165+
}
166+
encodedAksNodeConfigJSON := base64.StdEncoding.EncodeToString(aksNodeConfigJSON)
167+
customDataYAML := fmt.Sprintf(cloudConfigTemplate, encodedAksNodeConfigJSON)
168+
return base64.StdEncoding.EncodeToString([]byte(customDataYAML)), nil
169+
}
170+
147171
func createVMSSModel(ctx context.Context, s *Scenario) armcompute.VirtualMachineScaleSet {
148172
cluster := s.Runtime.Cluster
149173
var nodeBootstrapping *datamodel.NodeBootstrapping
150174
ab, err := agent.NewAgentBaker()
151175
require.NoError(s.T, err)
152176
var cse, customData string
153177
if s.Runtime.AKSNodeConfig != nil {
154-
cse = nodeconfigutils.CSE
155-
customData = func() string {
156-
if config.Config.DisableScriptLessCompilation {
157-
data, err := nodeconfigutils.CustomData(s.Runtime.AKSNodeConfig)
158-
require.NoError(s.T, err, "failed to generate custom data from AKSNodeConfig")
178+
if s.Config.UseCustomDataOnlyProvisioning {
179+
data, err := CustomDataWithDirectProvision(s.Runtime.AKSNodeConfig)
180+
require.NoError(s.T, err, "failed to generate custom data from AKSNodeConfig with direct provision")
181+
customData = data
182+
} else {
183+
cse = nodeconfigutils.CSE
184+
customData = func() string {
185+
if config.Config.DisableScriptLessCompilation {
186+
data, err := nodeconfigutils.CustomData(s.Runtime.AKSNodeConfig)
187+
require.NoError(s.T, err, "failed to generate custom data from AKSNodeConfig")
188+
return data
189+
}
190+
binaryURL, err := CachedCompileAndUploadAKSNodeController(ctx, s.VHD.Arch)
191+
require.NoError(s.T, err, "failed to compile and upload aks-node-controller binary")
192+
data, err := CustomDataWithHack(s, binaryURL)
193+
require.NoError(s.T, err, "failed to generate custom data from AKSNodeConfig with hack")
159194
return data
160-
}
161-
binaryURL, err := CachedCompileAndUploadAKSNodeController(ctx, s.VHD.Arch)
162-
require.NoError(s.T, err, "failed to compile and upload aks-node-controller binary")
163-
data, err := CustomDataWithHack(s, binaryURL)
164-
require.NoError(s.T, err, "failed to generate custom data from AKSNodeConfig with hack")
165-
return data
166-
}()
195+
}()
196+
}
167197

168198
} else {
169199
nodeBootstrapping, err = ab.GetNodeBootstrapping(ctx, s.Runtime.NBC)

‎vhdbuilder/packer/install-dependencies.sh‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ echo "Components downloaded in this VHD build (some of the below components migh
4747
capture_benchmark "${SCRIPT_NAME}_source_packer_files_and_declare_variables"
4848

4949
installLocalCloudInitPackages() {
50-
local package_dir="/home/packer/cloud-init-packages"
50+
local package_dir="/opt/azure/cloud-init-packages"
5151
local cloud_init_base_pkg
5252
local cloud_init_azure_pkg
5353
local cloud_init_pkg
@@ -66,8 +66,10 @@ installLocalCloudInitPackages() {
6666
fi
6767

6868
echo "Installing cloud-init packages from $package_dir"
69-
wait_for_apt_locks
70-
dpkg -i "$cloud_init_base_pkg" "$cloud_init_azure_pkg" "$cloud_init_pkg"
69+
apt_get_update || exit $ERR_APT_UPDATE_TIMEOUT
70+
installDebPackageFromFile "$cloud_init_base_pkg" || exit $ERR_APT_INSTALL_TIMEOUT
71+
installDebPackageFromFile "$cloud_init_azure_pkg" || exit $ERR_APT_INSTALL_TIMEOUT
72+
installDebPackageFromFile "$cloud_init_pkg" || exit $ERR_APT_INSTALL_TIMEOUT
7173
}
7274

7375
echo "Logging the kernel after purge and reinstall + reboot: $(uname -r)"

‎vhdbuilder/packer/packer_source.sh‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -290,6 +290,18 @@ copyPackerFiles() {
290290
CLOUD_INIT_STATUS_CHECK_DEST=/opt/azure/containers/cloud-init-status-check.sh
291291
cpAndMode $CLOUD_INIT_STATUS_CHECK_SRC $CLOUD_INIT_STATUS_CHECK_DEST 0744
292292

293+
CLOUD_INIT_BASE_PKG_SRC=/home/packer/cloud-init-packages/cloud-init-base_all.deb
294+
CLOUD_INIT_BASE_PKG_DEST=/opt/azure/cloud-init-packages/cloud-init-base_all.deb
295+
cpAndMode $CLOUD_INIT_BASE_PKG_SRC $CLOUD_INIT_BASE_PKG_DEST 0644
296+
297+
CLOUD_INIT_AZURE_PKG_SRC=/home/packer/cloud-init-packages/cloud-init-azure_all.deb
298+
CLOUD_INIT_AZURE_PKG_DEST=/opt/azure/cloud-init-packages/cloud-init-azure_all.deb
299+
cpAndMode $CLOUD_INIT_AZURE_PKG_SRC $CLOUD_INIT_AZURE_PKG_DEST 0644
300+
301+
CLOUD_INIT_PKG_SRC=/home/packer/cloud-init-packages/cloud-init_all.deb
302+
CLOUD_INIT_PKG_DEST=/opt/azure/cloud-init-packages/cloud-init_all.deb
303+
cpAndMode $CLOUD_INIT_PKG_SRC $CLOUD_INIT_PKG_DEST 0644
304+
293305
NOTICE_SRC=/home/packer/NOTICE.txt
294306
NOTICE_DEST=/NOTICE.txt
295307

0 commit comments

Comments
 (0)