Skip to content

Commit 0e0fa81

Browse files
theyoprsttheyoprst
committed
SCHED-863: Rewrite E2E from test framework to CLI binary
Replace go test -based e2e with a standalone CLI at cmd/e2e. This removes the terratest dependency and the artificial test harness that had no assertions or test cases.
1 parent d07f06e commit 0e0fa81

File tree

12 files changed

+527
-462
lines changed

12 files changed

+527
-462
lines changed

‎.github/workflows/e2e_test.yml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -173,6 +173,7 @@ jobs:
173173
fetch-depth: 0
174174

175175
- name: Terraform Apply
176+
timeout-minutes: 120
176177
run: |
177178
cd ${{ env.PATH_TO_INSTALLATION }}
178179
nebius iam session-management revoke --all-my-active
@@ -187,7 +188,7 @@ jobs:
187188
aws configure set region $NEBIUS_REGION
188189
aws configure set endpoint_url https://storage.$NEBIUS_REGION.nebius.cloud:443
189190
190-
go test -v -timeout 2h --tags=e2e -run TestTerraformApply ./test/e2e/...
191+
go run ./cmd/e2e apply
191192
192193
- name: K8s Cluster Info and NodeGroups
193194
if: always()
@@ -299,6 +300,7 @@ jobs:
299300

300301
- name: Terraform Destroy
301302
if: always()
303+
timeout-minutes: 30
302304
run: |
303305
cd ${{ env.PATH_TO_INSTALLATION }}
304306
source .envrc
@@ -310,7 +312,7 @@ jobs:
310312
aws configure set region $NEBIUS_REGION
311313
aws configure set endpoint_url https://storage.$NEBIUS_REGION.nebius.cloud:443
312314
313-
go test -v -timeout 30m --tags=e2e -run TestTerraformDestroy ./test/e2e/...
315+
go run ./cmd/e2e destroy
314316
315317
- name: Force cleanup compute instances on failure
316318
if: failure()

‎cmd/e2e/main.go‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
package main
2+
3+
import (
4+
"fmt"
5+
"log"
6+
"os"
7+
8+
"github.com/kelseyhightower/envconfig"
9+
10+
"nebius.ai/slurm-operator/internal/e2e"
11+
)
12+
13+
func main() {
14+
if len(os.Args) < 2 {
15+
_, _ = fmt.Fprintf(os.Stderr, "Usage: e2e <apply|destroy>\n")
16+
os.Exit(2)
17+
}
18+
19+
var cfg e2e.Config
20+
if err := envconfig.Process("", &cfg); err != nil {
21+
log.Fatalf("parse config: %v", err)
22+
}
23+
24+
var err error
25+
switch os.Args[1] {
26+
case "apply":
27+
err = e2e.Apply(cfg)
28+
case "destroy":
29+
err = e2e.Destroy(cfg)
30+
default:
31+
_, _ = fmt.Fprintf(os.Stderr, "Unknown command: %s\nUsage: e2e <apply|destroy>\n", os.Args[1])
32+
os.Exit(2)
33+
}
34+
if err != nil {
35+
log.Fatalf("%s: %v", os.Args[1], err)
36+
}
37+
}

‎go.mod‎

Lines changed: 1 addition & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ require (
66
github.com/SlinkyProject/slurm-client v0.3.1-20250801102150-043094ce1e4e
77
github.com/go-logr/logr v1.4.3
88
github.com/golang-jwt/jwt/v5 v5.3.0
9-
github.com/gruntwork-io/terratest v0.54.0
109
github.com/hashicorp/go-retryablehttp v0.7.8
1110
github.com/hashicorp/hcl/v2 v2.22.0
1211
github.com/kelseyhightower/envconfig v1.4.0
@@ -39,7 +38,6 @@ require (
3938
github.com/agext/levenshtein v1.2.3 // indirect
4039
github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect
4140
github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
42-
github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
4341
github.com/blang/semver/v4 v4.0.0 // indirect
4442
github.com/cert-manager/cert-manager v1.18.2 // indirect
4543
github.com/containers/common v0.60.4 // indirect
@@ -50,18 +48,7 @@ require (
5048
github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
5149
github.com/google/btree v1.1.3 // indirect
5250
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
53-
github.com/hashicorp/errwrap v1.1.0 // indirect
5451
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
55-
github.com/hashicorp/go-getter/v2 v2.2.3 // indirect
56-
github.com/hashicorp/go-multierror v1.1.1 // indirect
57-
github.com/hashicorp/go-safetemp v1.0.0 // indirect
58-
github.com/hashicorp/go-version v1.7.0 // indirect
59-
github.com/hashicorp/terraform-json v0.23.0 // indirect
60-
github.com/jinzhu/copier v0.4.0 // indirect
61-
github.com/klauspost/compress v1.18.0 // indirect
62-
github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326 // indirect
63-
github.com/mitchellh/go-homedir v1.1.0 // indirect
64-
github.com/mitchellh/go-testing-interface v1.14.1 // indirect
6552
github.com/mitchellh/go-wordwrap v1.0.1 // indirect
6653
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
6754
github.com/oapi-codegen/runtime v1.1.1 // indirect
@@ -76,9 +63,7 @@ require (
7663
github.com/sethvargo/go-envconfig v1.3.0 // indirect
7764
github.com/sirupsen/logrus v1.9.3 // indirect
7865
github.com/stretchr/objx v0.5.2 // indirect
79-
github.com/tmccombs/hcl2json v0.6.4 // indirect
8066
github.com/ugorji/go/codec v1.2.12 // indirect
81-
github.com/ulikunitz/xz v0.5.14 // indirect
8267
github.com/x448/float16 v0.8.4 // indirect
8368
go.opentelemetry.io/otel v1.35.0 // indirect
8469
go.opentelemetry.io/otel/trace v1.35.0 // indirect
@@ -130,7 +115,7 @@ require (
130115
gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
131116
google.golang.org/protobuf v1.36.8 // indirect
132117
gopkg.in/inf.v0 v0.9.1 // indirect
133-
gopkg.in/yaml.v3 v3.0.1
118+
gopkg.in/yaml.v3 v3.0.1 // indirect
134119
k8s.io/apiextensions-apiserver v0.34.2 // indirect
135120
k8s.io/klog/v2 v2.130.1
136121
k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect

‎go.sum‎

Lines changed: 0 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,6 @@ github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew
1111
github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4=
1212
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
1313
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
14-
github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1UJrqV3uuy861HCTo708pDMbjHHdCas=
15-
github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4=
1614
github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
1715
github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
1816
github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w=
@@ -83,31 +81,14 @@ github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaU
8381
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
8482
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
8583
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
86-
github.com/gruntwork-io/terratest v0.54.0 h1:JOVATYDpU0NAPbEkgYUP50BR2m45UGiR4dbs20sKzck=
87-
github.com/gruntwork-io/terratest v0.54.0/go.mod h1:QvwQWZMTJmJB4E0d1Uc18quQm7+X53liKKp+fJSuaKA=
88-
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
89-
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
90-
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
9184
github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
9285
github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
93-
github.com/hashicorp/go-getter/v2 v2.2.3 h1:6CVzhT0KJQHqd9b0pK3xSP0CM/Cv+bVhk+jcaRJ2pGk=
94-
github.com/hashicorp/go-getter/v2 v2.2.3/go.mod h1:hp5Yy0GMQvwWVUmwLs3ygivz1JSLI323hdIE9J9m7TY=
9586
github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
9687
github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
97-
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
98-
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
9988
github.com/hashicorp/go-retryablehttp v0.7.8 h1:ylXZWnqa7Lhqpk0L1P1LzDtGcCR0rPVUrx/c8Unxc48=
10089
github.com/hashicorp/go-retryablehttp v0.7.8/go.mod h1:rjiScheydd+CxvumBsIrFKlx3iS0jrZ7LvzFGFmuKbw=
101-
github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo=
102-
github.com/hashicorp/go-safetemp v1.0.0/go.mod h1:oaerMy3BhqiTbVye6QuFhFtIceqFoDHxNAB65b+Rj1I=
103-
github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY=
104-
github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
10590
github.com/hashicorp/hcl/v2 v2.22.0 h1:hkZ3nCtqeJsDhPRFz5EA9iwcG1hNWGePOTw6oyul12M=
10691
github.com/hashicorp/hcl/v2 v2.22.0/go.mod h1:62ZYHrXgPoX8xBnzl8QzbWq4dyDsDtfCRgIq1rbJEvA=
107-
github.com/hashicorp/terraform-json v0.23.0 h1:sniCkExU4iKtTADReHzACkk8fnpQXrdD2xoR+lppBkI=
108-
github.com/hashicorp/terraform-json v0.23.0/go.mod h1:MHdXbBAbSg0GvzuWazEGKAn/cyNfIB7mN6y7KJN6y2c=
109-
github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8=
110-
github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=
11192
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
11293
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
11394
github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE=
@@ -141,14 +122,8 @@ github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxec
141122
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
142123
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
143124
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
144-
github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326 h1:ofNAzWCcyTALn2Zv40+8XitdzCgXY6e9qvXwN9W0YXg=
145-
github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo=
146125
github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE=
147126
github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A=
148-
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
149-
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
150-
github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU=
151-
github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8=
152127
github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0=
153128
github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0=
154129
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -228,12 +203,8 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
228203
github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
229204
github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
230205
github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
231-
github.com/tmccombs/hcl2json v0.6.4 h1:/FWnzS9JCuyZ4MNwrG4vMrFrzRgsWEOVi+1AyYUVLGw=
232-
github.com/tmccombs/hcl2json v0.6.4/go.mod h1:+ppKlIW3H5nsAsZddXPy2iMyvld3SHxyjswOZhavRDk=
233206
github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE=
234207
github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
235-
github.com/ulikunitz/xz v0.5.14 h1:uv/0Bq533iFdnMHZdRBTOlaNMdb1+ZxXIlHDZHIHcvg=
236-
github.com/ulikunitz/xz v0.5.14/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
237208
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
238209
github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
239210
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

‎internal/e2e/apply.go‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
package e2e
2+
3+
import (
4+
"fmt"
5+
"log"
6+
)
7+
8+
func Apply(cfg Config) error {
9+
runner, cleanup, err := Init(cfg)
10+
if err != nil {
11+
return err
12+
}
13+
defer cleanup()
14+
15+
if err := runner.Destroy(); err != nil {
16+
log.Printf("Pre-cleanup destroy failed: %v", err)
17+
logState(runner)
18+
return fmt.Errorf("pre-cleanup destroy failed, state may contain stuck resources: %w", err)
19+
}
20+
21+
if err := runner.Apply(); err != nil {
22+
return fmt.Errorf("terraform apply: %w", err)
23+
}
24+
return nil
25+
}

test/e2e/e2e_config_test.go renamed to internal/e2e/config.go

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,7 @@
1-
//go:build e2e
2-
3-
package e2e_test
1+
package e2e
42

53
// nolint:tagalign
6-
type testConfig struct {
4+
type Config struct {
75
SoperatorVersion string `split_words:"true" required:"true"` // SOPERATOR_VERSION
86
SoperatorUnstable bool `split_words:"true" required:"true"` // SOPERATOR_UNSTABLE
97
PathToInstallation string `split_words:"true" required:"true"` // PATH_TO_INSTALLATION

‎internal/e2e/destroy.go‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
package e2e
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"log"
7+
"os"
8+
"os/exec"
9+
"strings"
10+
11+
"nebius.ai/slurm-operator/internal/e2e/tfrunner"
12+
)
13+
14+
const k8sClusterName = "soperator-e2e-test"
15+
16+
func Destroy(cfg Config) error {
17+
runner, cleanup, err := Init(cfg)
18+
if err != nil {
19+
return err
20+
}
21+
defer cleanup()
22+
23+
return destroyWithK8sRecovery(runner)
24+
}
25+
26+
func destroyWithK8sRecovery(runner *tfrunner.Runner) error {
27+
err := runner.Destroy()
28+
if err == nil {
29+
return nil
30+
}
31+
if !strings.Contains(err.Error(), "Kubernetes cluster unreachable") {
32+
return err
33+
}
34+
if !isMK8SClusterGone() {
35+
return err
36+
}
37+
log.Print("K8s cluster is confirmed gone, removing helm releases from state and retrying destroy")
38+
removeHelmReleasesFromState(runner)
39+
logState(runner)
40+
if retryErr := runner.Destroy(); retryErr != nil {
41+
return fmt.Errorf("destroy after helm release state cleanup: %w", retryErr)
42+
}
43+
log.Printf("Destroy recovered: K8s cluster %s was already gone, removed helm releases from state to unblock cleanup", k8sClusterName)
44+
return nil
45+
}
46+
47+
func removeHelmReleasesFromState(runner *tfrunner.Runner) {
48+
out, err := runner.Run("state", "list")
49+
if err != nil {
50+
log.Printf("terraform state list failed during helm release removal: %v", err)
51+
return
52+
}
53+
for _, resource := range strings.Split(out, "\n") {
54+
resource = strings.TrimSpace(resource)
55+
if resource == "" || !strings.Contains(resource, "helm_release") {
56+
continue
57+
}
58+
log.Printf("Removing %s from terraform state", resource)
59+
if _, rmErr := runner.Run("state", "rm", resource); rmErr != nil {
60+
log.Printf("terraform state rm %s failed: %v", resource, rmErr)
61+
}
62+
}
63+
}
64+
65+
func isMK8SClusterGone() bool {
66+
projectID := os.Getenv("NEBIUS_PROJECT_ID")
67+
if projectID == "" {
68+
log.Print("NEBIUS_PROJECT_ID not set, cannot verify cluster existence")
69+
return false
70+
}
71+
72+
out, err := exec.CommandContext(context.Background(),
73+
"nebius", "mk8s", "cluster", "get-by-name",
74+
"--parent-id", projectID,
75+
"--name", k8sClusterName,
76+
).CombinedOutput()
77+
if err != nil {
78+
log.Printf("mk8s cluster %s not found (get-by-name failed: %v, output: %s)", k8sClusterName, err, string(out))
79+
return true
80+
}
81+
log.Printf("mk8s cluster %s still exists", k8sClusterName)
82+
return false
83+
}

0 commit comments

Comments
 (0)