Skip to content

[Do not merge] CI Experiments #10633

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 5 commits into
base: main
Choose a base branch
from
+43 −14
Draft

[Do not merge] CI Experiments #10633

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
15b5aa6
pass -test.short flag in integration tests for PRs
pchila Oct 17, 2025
c387886
Skip Test*Flavor* when test.short is set
pchila Oct 17, 2025
1c05101
Reduce installuninstall iteration to 10 when test.short is specified
pchila Oct 17, 2025
1094773
eagerly start integration tests for PRs
pchila Oct 17, 2025
2da30de
switch to n2-highcpu-8 for unit tests
pchila Oct 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions .buildkite/integration.pipeline.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ steps:
limit: 1
agents:
provider: "aws"
instanceType: "c6g.4xlarge"
instanceType: "c6g.2xlarge"
image: "${IMAGE_UBUNTU_2204_ARM_64}"

- label: "Packaging: Ubuntu arm64 FIPS"
Expand All @@ -87,7 +87,7 @@ steps:
limit: 1
agents:
provider: "aws"
instanceType: "c6g.4xlarge"
instanceType: "c6g.2xlarge"
image: "${IMAGE_UBUNTU_2204_ARM_64}"

- label: "Packaging: windows/amd64 zip"
Expand Down Expand Up @@ -119,7 +119,7 @@ steps:
limit: 1
agents:
provider: "aws"
instanceType: "c6g.4xlarge"
instanceType: "c6g.2xlarge"
image: "${IMAGE_UBUNTU_2204_ARM_64}"

- label: "Packaging: Containers linux/amd64"
Expand Down Expand Up @@ -186,16 +186,16 @@ steps:
- build/distributions/**
agents:
provider: "aws"
instanceType: "c6g.4xlarge"
instanceType: "c6g.2xlarge"
diskSizeGb: 200
image: "${IMAGE_UBUNTU_2204_ARM_64}"

- label: "Triggering Integration tests"
depends_on:
- int-packaging
# depends_on:
# - int-packaging
command: "buildkite-agent pipeline upload .buildkite/bk.integration.pipeline.yml"

- label: "Triggering custom FIPS integration tests"
depends_on:
- int-packaging
# depends_on:
# - int-packaging
command: "buildkite-agent pipeline upload .buildkite/bk.integration-fips.pipeline.yml"
11 changes: 7 additions & 4 deletions .buildkite/pipeline.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ steps:
agents:
provider: "gcp"
image: "${IMAGE_UBUNTU_2204_X86_64}"
machineType: "n2-highcpu-8"
retry:
automatic:
limit: 1
Expand All @@ -56,6 +57,7 @@ steps:
agents:
provider: "gcp"
image: "${IMAGE_UBUNTU_2204_X86_64}"
machineType: "n2-highcpu-8"
retry:
automatic:
limit: 1
Expand All @@ -76,6 +78,7 @@ steps:
agents:
provider: "gcp"
image: "${IMAGE_UBUNTU_2204_X86_64}"
machineType: "n2-highcpu-8"
retry:
automatic:
limit: 1
Expand Down Expand Up @@ -112,7 +115,7 @@ steps:
agents:
provider: "gcp"
image: "${IMAGE_WIN_2022}"
machine_type: "n2-standard-8"
machineType: "n2-standard-8"
disk_size: 200
disk_type: "pd-ssd"
retry:
Expand All @@ -132,7 +135,7 @@ steps:
agents:
provider: "gcp"
image: "${IMAGE_WIN_2016}"
machine_type: "n2-standard-8"
machineType: "n2-standard-8"
disk_size: 200
disk_type: "pd-ssd"
retry:
Expand Down Expand Up @@ -192,7 +195,7 @@ steps:
agents:
provider: "gcp"
image: "${IMAGE_WIN_10}"
machine_type: "n2-standard-8"
machineType: "n2-standard-8"
disk_type: "pd-ssd"
retry:
automatic:
Expand All @@ -211,7 +214,7 @@ steps:
agents:
provider: "gcp"
image: "${IMAGE_WIN_11}"
machine_type: "n2-standard-8"
machineType: "n2-standard-8"
disk_type: "pd-ssd"
retry:
automatic:
Expand Down
7 changes: 6 additions & 1 deletion .buildkite/scripts/buildkite-integration-tests.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,12 @@ try
# This 2-hour timeout provides enough room for future, potentially longer tests,
# while still enforcing a reasonable upper limit on total execution time.
# See: https://pkg.go.dev/cmd/go#hdr-Testing_flags
$gotestArgs = @("-tags=integration", "-test.shuffle=on", "-test.timeout=2h0m0s", "$env:TEST_PACKAGE", "-v", "-args", "-integration.groups=$GROUP_NAME", "-integration.sudo=$TEST_SUDO")
$gotestFlags = @("-test.shuffle=on", "-test.timeout=2h0m0s")
if(-not ([string]::IsNullOrEmpty($env:BUILDKITE_PULL_REQUEST)))
{
$gotestFlags += "-test.short"
}
$gotestArgs = @("-tags=integration", ${gotestFlags}, "$env:TEST_PACKAGE", "-v", "-args", "-integration.groups=$GROUP_NAME", "-integration.sudo=$TEST_SUDO")
& gotestsum --no-color -f standard-quiet --junitfile-hide-skipped-tests --junitfile "${outputXML}" --jsonfile "${outputJSON}" -- @gotestArgs
$TestsExitCode = $LASTEXITCODE

Expand Down
7 changes: 6 additions & 1 deletion .buildkite/scripts/buildkite-integration-tests.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,12 @@ echo "~~~ Integration tests: ${GROUP_NAME}"
# This 2-hour timeout provides enough room for future, potentially longer tests,
# while still enforcing a reasonable upper limit on total execution time.
# See: https://pkg.go.dev/cmd/go#hdr-Testing_flags
GOTEST_ARGS=(-tags integration -test.shuffle on -test.timeout 2h0m0s "${TEST_PACKAGE}" -v -args "-integration.groups=${GROUP_NAME}" "-integration.sudo=${TEST_SUDO}" "-integration.fips=${FIPS:-false}")

GOTEST_OPTS="-test.shuffle on -test.timeout 2h0m0s"
if [[ "${BUILDKITE_PULL_REQUEST:="false"}" != "false" ]]; then
GOTEST_OPTS="${GOTEST_OPTS} -test.short"
fi
GOTEST_ARGS=(-tags integration ${GOTEST_OPTS} "${TEST_PACKAGE}" -v -args "-integration.groups=${GROUP_NAME}" "-integration.sudo=${TEST_SUDO}" "-integration.fips=${FIPS:-false}")
set +e
TEST_BINARY_NAME="elastic-agent" AGENT_VERSION="${AGENT_VERSION}" SNAPSHOT=true \
gotestsum --no-color -f standard-quiet --junitfile-hide-skipped-tests --junitfile "${outputXML}" --jsonfile "${outputJSON}" -- "${GOTEST_ARGS[@]}"
Expand Down
4 changes: 4 additions & 0 deletions testing/integration/ess/install_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -719,6 +719,10 @@ func TestRepeatedInstallUninstallFleet(t *testing.T) {
}

func iterations() int {
if testing.Short() {
return 10
}

// If running in CI, reduce the number of iterations to speed up the test.
if os.Getenv("BUILDKITE_PULL_REQUEST") != "" {
return 50
Expand Down
12 changes: 12 additions & 0 deletions testing/integration/ess/upgrade_standalone_flavors_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,10 @@ func TestStandaloneUpgrade_Flavor_Basic(t *testing.T) {
Sudo: true, // requires Agent installation
})

if testing.Short() {
t.Skip("skipping test in short mode")
}

minVersion := upgradetest.Version_9_0_0_SNAPSHOT
currentVersion, err := version.ParseVersion(define.Version())
require.NoError(t, err)
Expand Down Expand Up @@ -88,6 +92,10 @@ func TestStandaloneUpgrade_Flavor_Servers(t *testing.T) {
Sudo: true, // requires Agent installation
})

if testing.Short() {
t.Skip("skipping test in short mode")
}

minVersion := upgradetest.Version_9_0_0_SNAPSHOT
currentVersion, err := version.ParseVersion(define.Version())
require.NoError(t, err)
Expand Down Expand Up @@ -145,6 +153,10 @@ func TestStandaloneUpgrade_Flavor_UpgradeFromUnflavored(t *testing.T) {
Sudo: true, // requires Agent installation
})

if testing.Short() {
t.Skip("skipping test in short mode")
}

minVersion := upgradetest.Version_9_0_0_SNAPSHOT
currentVersion, err := version.ParseVersion(define.Version())
require.NoError(t, err)
Expand Down