diff --git a/CLAUDE.md b/CLAUDE.md index f8c3bf6..d19991d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -227,95 +227,152 @@ type Config struct { ``` networks/ ├── ethereum/ -│ ├── helmfile.yaml.gotmpl # Main configuration -│ ├── Chart.yaml # Optional local chart -│ └── templates/ # Optional Kubernetes resources +│ ├── values.yaml.gotmpl # Configuration template with annotations +│ ├── helmfile.yaml # Deployment logic (pure Helmfile syntax) +│ ├── Chart.yaml # Optional local chart +│ └── templates/ # Optional Kubernetes resources ├── helios/ -│ └── helmfile.yaml.gotmpl +│ ├── values.yaml.gotmpl +│ └── helmfile.yaml └── aztec/ - └── helmfile.yaml.gotmpl + ├── values.yaml.gotmpl + └── helmfile.yaml ``` ### Two-Stage Templating -**Stage 1: CLI Flag Templating** (Go templates) +**Stage 1: CLI Flag Templating** (Go templates → values.yaml) + +`values.yaml.gotmpl` contains configuration fields with annotations: +```yaml +# @enum mainnet,sepolia,holesky,hoodi +# @default mainnet +# @description Blockchain network to deploy +network: {{.Network}} + +# @enum reth,geth,nethermind,besu,erigon,ethereumjs +# @default reth +executionClient: {{.ExecutionClient}} +``` + +CLI processes this template and generates `values.yaml`: ```yaml -values: - # Annotations parsed at build time for CLI flag generation - # @enum mainnet,sepolia,holesky,hoodi - # @default mainnet - # @description Blockchain network to deploy - - network: {{.Network}} - # @enum reth,geth,nethermind,besu,erigon,ethereumjs - # @default reth - executionClient: {{.ExecutionClient}} - # Namespace generated at install time - namespace: {{.Namespace}} +network: mainnet +executionClient: reth ``` -**Stage 2: Helmfile Templating** (Helm/Go templates) -- CLI populates Stage 1 templates with flag values -- Templated helmfile saved to `$CONFIG_DIR/networks///` -- Helmfile processes template using `{{ .Values.* }}` syntax +**Note**: `id` is NOT in `values.yaml` - it's passed separately via directory structure. + +**Stage 2: Helmfile Templating** (Helmfile processes values) + +`helmfile.yaml` references values using Helmfile syntax: +```yaml +releases: + - name: ethereum-pvcs + namespace: ethereum-{{ .Values.id }} # Dynamic namespace + values: + - network: '{{ .Values.network }}' + executionClient: '{{ .Values.executionClient }}' +``` + +When `helmfile sync --state-values-file values.yaml` runs: +- Reads values from `values.yaml` +- Substitutes `{{ .Values.* }}` references - Generates final Kubernetes YAML - Applies to cluster in unique namespace ### Unique Namespace Pattern **Namespace generation**: -- Pattern: `-` -- Example: `ethereum-nervous-otter`, `ethereum-happy-panda` -- Uses `github.com/dustinkirkland/golang-petname` +- Pattern: `-` +- ID can be user-specified (`--id prod`) or auto-generated (petname like `knowing-wahoo`) +- Uses `github.com/dustinkirkland/golang-petname` for auto-generation +- Examples: + - `ethereum-knowing-wahoo` (auto-generated) + - `ethereum-prod` (user-specified with `--id prod`) + - `helios-united-bison` (auto-generated) + - `aztec-staging` (user-specified) + +**ID as deployment identifier**: +- `id` is NOT in `values.yaml` or `values.yaml.gotmpl` (special case) +- Determined by directory structure: `~/.config/obol/networks///` +- CLI auto-generates petname if `--id` flag not provided +- Passed to Helmfile via `--state-values-set id=` during sync +- Helmfile enforces namespace: `namespace: {{ .Values.id }}` **Benefits**: 1. **Multiple deployments**: Run mainnet + testnet simultaneously 2. **Isolated resources**: Each deployment has dedicated CPU, memory, storage 3. **Independent lifecycle**: Update/delete one without affecting others 4. **Simple cleanup**: Delete namespace removes all resources +5. **Predictable naming**: User controls ID for production deployments **Example**: ```bash -# First deployment +# Auto-generated ID (development) obol network install ethereum --network=mainnet -# Creates: ethereum-nervous-otter namespace - -# Second deployment (different config) -obol network install ethereum --network=holesky -# Creates: ethereum-happy-panda namespace - -# Both run simultaneously, isolated from each other +# Generated deployment ID: knowing-wahoo +# Creates: ~/.config/obol/networks/ethereum/knowing-wahoo/ +# Namespace: ethereum-knowing-wahoo + +# User-specified ID (production) +obol network install ethereum --id prod --network=mainnet +# Creates: ~/.config/obol/networks/ethereum/prod/ +# Namespace: ethereum-prod + +# Multiple deployments with different configs +obol network install ethereum --id mainnet-01 +obol network install ethereum --id holesky-test --network=holesky +# Both run simultaneously, isolated in separate namespaces ``` ### Network Configuration Flow -1. **Install**: +1. **Install** (config generation only): ``` - obol network install ethereum --network=holesky --execution-client=geth + obol network install ethereum --network=holesky --execution-client=geth --id my-node ↓ - Parse embedded helmfile annotations → generate flags + Check if directory exists: ~/.config/obol/networks/ethereum/my-node/ (fail unless --force) ↓ - Collect flag values into overrides map + Parse values.yaml.gotmpl → extract field definitions + annotations (sorted by line number) ↓ - Generate unique namespace (ethereum-nervous-otter) + Collect CLI flag values into overrides map (id collected separately, not as template field) ↓ - Template Stage 1: Populate {{.Network}}, {{.ExecutionClient}}, {{.Namespace}} + Template values.yaml.gotmpl: Populate {{.Network}}, {{.ExecutionClient}} (NOT {{.Id}}) ↓ - Save to: ~/.config/obol/networks/ethereum/nervous-otter/helmfile.yaml.gotmpl + Validate YAML syntax of generated content ↓ - Run: helmfile sync -f + Write values.yaml to: ~/.config/obol/networks/ethereum/my-node/values.yaml ↓ - Helmfile templates Stage 2 and applies to cluster + Copy helmfile.yaml.gotmpl as-is (no templating) + ↓ + Copy other files (Chart.yaml, templates/) + ``` + +2. **Sync** (deployment): + ``` + obol network sync ethereum/my-node + ↓ + Extract id from directory path: "my-node" + ↓ + Run: helmfile sync --state-values-file values.yaml --state-values-set id=my-node + ↓ + Helmfile reads values.yaml + receives id via --state-values-set + ↓ + Substitutes {{ .Values.* }} in helmfile.yaml.gotmpl (including {{ .Values.id }}) + ↓ + Deploys to namespace: ethereum-my-node ``` -2. **Delete**: +3. **Delete**: ``` - obol network delete ethereum-nervous-otter + obol network delete ethereum/knowing-wahoo ↓ Delete Kubernetes namespace (removes all resources) ↓ Delete PVCs and persistent data ↓ - Remove: ~/.config/obol/networks/ethereum/nervous-otter/ + Remove: ~/.config/obol/networks/ethereum/knowing-wahoo/ ``` ## Directory Structure @@ -338,14 +395,24 @@ obol network install ethereum --network=holesky │ └── obol-frontend.yaml.gotmpl └── networks/ # Installed network deployments ├── ethereum/ - │ ├── nervous-otter/ # First ethereum deployment - │ │ └── helmfile.yaml.gotmpl - │ └── happy-panda/ # Second ethereum deployment - │ └── helmfile.yaml.gotmpl + │ ├── knowing-wahoo/ # First ethereum deployment + │ │ ├── values.yaml # Generated config (plain YAML) + │ │ ├── helmfile.yaml # Deployment logic (copied as-is) + │ │ ├── Chart.yaml + │ │ └── templates/ + │ └── prod/ # Second ethereum deployment + │ ├── values.yaml + │ ├── helmfile.yaml + │ ├── Chart.yaml + │ └── templates/ ├── helios/ - │ └── laughing-elephant/ + │ └── united-bison/ + │ ├── values.yaml + │ └── helmfile.yaml └── aztec/ - └── clever-fox/ + └── staging/ + ├── values.yaml + └── helmfile.yaml ~/.local/bin/ # Binaries ├── obol # Obol CLI @@ -359,10 +426,10 @@ obol network install ethereum --network=holesky ~/.local/share/obol/ # Persistent data └── / └── networks/ - ├── ethereum_nervous-otter/ # Blockchain data for first deployment - ├── ethereum_happy-panda/ # Blockchain data for second deployment - ├── helios_laughing-elephant/ - └── aztec_clever-fox/ + ├── ethereum_knowing-wahoo/ # Blockchain data for first deployment + ├── ethereum_prod/ # Blockchain data for second deployment + ├── helios_united-bison/ + └── aztec_staging/ ``` ### Development Layout @@ -482,9 +549,9 @@ obol network install ethereum --network=holesky ## Network Install Implementation Details -### Annotation Parser +### Template Field Parser -**Location**: `internal/network/network.go` - `ParseEmbeddedNetworkEnvVars()` +**Location**: `internal/network/parser.go` - `ParseTemplateFields()` **Annotations supported**: - `@enum`: Comma-separated valid values @@ -492,17 +559,16 @@ obol network install ethereum --network=holesky - `@description`: Help text for flag **Parsing logic**: -1. Read embedded `helmfile.yaml.gotmpl` -2. Find `values:` section -3. Extract environment variable references: `{{ env "VAR_NAME" | default "value" }}` -4. Parse annotations from comments above each value -5. Generate `EnvVar` struct with: - - Name: Environment variable name (e.g., `ETHEREUM_NETWORK`) - - FlagName: CLI flag name (lowercase, dashed, e.g., `network`) - - DefaultValue: From `default` pipe or `@default` annotation +1. Read embedded `values.yaml.gotmpl` +2. Parse Go template to extract field references (e.g., `{{.Network}}`, `{{.ExecutionClient}}`) +3. Parse annotations from comments above each field +4. Generate `TemplateField` struct with: + - Name: Template field name (e.g., `Network`, `ExecutionClient`) + - FlagName: CLI flag name (lowercase, dashed, e.g., `network`, `execution-client`) + - DefaultValue: From `@default` annotation - EnumValues: From `@enum` annotation - Description: From `@description` annotation - - Required: True if no default value + - Required: True if no `@default` annotation present ### CLI Flag Generation @@ -510,8 +576,8 @@ obol network install ethereum --network=holesky **Process**: 1. For each embedded network: - - Parse helmfile annotations - - Build `cli.Flag` for each environment variable + - Parse values template to extract template fields + - Build `cli.Flag` for each template field - Add enum validation to flag usage - Set Required based on default presence 2. Create network-specific subcommand: `obol network install ` @@ -519,33 +585,49 @@ obol network install ethereum --network=holesky 4. Register subcommand dynamically **Flag naming convention**: -- Environment variable: `ETHEREUM_EXECUTION_CLIENT` +- Template field: `ExecutionClient` - Flag name: `--execution-client` -- Transformation: Remove network prefix, lowercase, dash-separated +- Transformation: Insert hyphens before uppercase letters, lowercase ### Install Implementation **Location**: `internal/network/network.go` - `Install()` -**Current implementation** (temporary, until two-stage templating): -1. Parse embedded helmfile for environment variables -2. Display configuration to user (with overrides highlighted) -3. Create temporary directory: `/tmp/obol-network--XXXX` -4. Copy embedded network to temp directory -5. Set environment variables in process -6. Run: `helmfile -f /helmfile.yaml.gotmpl sync` -7. Helmfile processes template with environment variables -8. Deploy to cluster -9. Remove temp directory - -**Future implementation** (two-stage templating): -1. Generate unique namespace (petname) -2. Parse embedded helmfile -3. Template Stage 1: Populate `{{.Network}}`, `{{.ExecutionClient}}`, etc. with flag values -4. Save templated helmfile to: `$CONFIG_DIR/networks///` -5. Run: `helmfile sync -f ` -6. Helmfile templates Stage 2 and applies to cluster -7. User can edit saved helmfile and re-sync later +**Implementation** (two-stage templating): +1. Generate unique deployment ID (petname or user-specified via `--id`) +2. Check if deployment directory exists (fail unless `--force` flag provided) +3. Parse embedded values template to extract template fields +4. Build template data map from CLI flag overrides and defaults (NOT including `id`) +5. Display configuration to user (showing id from directory, overrides, and defaults) +6. Execute Go template on `values.yaml.gotmpl` with template data +7. Validate generated YAML syntax (catch malformed values early) +8. Write rendered `values.yaml` to: `$CONFIG_DIR/networks///values.yaml` +9. Copy network files (`helmfile.yaml.gotmpl`, `Chart.yaml`, `templates/`) to deployment directory +10. User runs `obol network sync /` to deploy +11. Sync command extracts `id` from directory path +12. Sync runs: `helmfile sync --state-values-file values.yaml --state-values-set id=` +13. Helmfile reads values.yaml, receives `id` via CLI flag, templates Stage 2 (substitutes `{{.Values.*}}`), and applies to cluster + +### Validation and Safety Features + +**Deployment Overwrite Protection**: +- Install command checks if deployment directory already exists +- Fails with clear error if directory exists: `deployment already exists: ethereum/my-node` +- User must provide `--force` or `-f` flag to explicitly overwrite +- Shows warning when overwriting: `⚠️ WARNING: Overwriting existing deployment` + +**YAML Syntax Validation**: +- After template execution, generated YAML is validated before writing to disk +- Uses `gopkg.in/yaml.v3` to parse and validate syntax +- Catches malformed values early (e.g., unquoted strings with colons) +- Error message shows the problematic content and specific syntax error +- Prevents invalid configuration from being saved or deployed + +**Deterministic Field Ordering**: +- Template fields are parsed from `values.yaml.gotmpl` using Go template AST +- Fields are sorted by line number before processing +- Ensures consistent CLI flag ordering in `--help` output +- Predictable behavior across runs and environments ## Key Implementation Patterns @@ -673,18 +755,11 @@ obol network delete ethereum- --force 1. **Relative paths in k3d config**: Will fail with Docker volume mounts 2. **Missing absolute path resolution**: k3d.yaml must have absolute paths before cluster creation 3. **Namespace collisions**: Without unique namespaces, multiple deployments will conflict -4. **Environment variable hijacking**: Current implementation uses env vars, planned migration to two-stage templating -5. **Root-owned PVCs**: Kubernetes creates PVCs as root, `-f` flag required to remove them +4. **Root-owned PVCs**: Kubernetes creates PVCs as root, `-f` flag required to remove them +5. **Special characters in values**: Unquoted YAML special chars (`:`, `[`, `{`) break syntax - caught by validation ### Future Work -**Two-stage templating migration**: -- Replace environment variable approach -- Use Go template execution for Stage 1 -- Save templated helmfile to config directory -- Enable user editing and re-sync -- Track: https://github.com/ObolNetwork/obol-stack/issues/ - **ERPC integration**: - Extract to separate helmfile - Auto-discover network endpoints diff --git a/cmd/obol/network.go b/cmd/obol/network.go index c71bd95..8c8940b 100644 --- a/cmd/obol/network.go +++ b/cmd/obol/network.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "slices" "strings" "github.com/ObolNetwork/obol-stack/internal/config" @@ -36,22 +37,27 @@ func networkCommand(cfg *config.Config) *cli.Command { }, }, { - Name: "delete", - Usage: "Remove network and clean up cluster resources", - ArgsUsage: "", - Flags: []cli.Flag{ - &cli.BoolFlag{ - Name: "force", - Aliases: []string{"f"}, - Usage: "Skip confirmation prompt", - }, + Name: "sync", + Usage: "Deploy or update network configuration to cluster", + ArgsUsage: "/ or -", + Action: func(c *cli.Context) error { + if c.NArg() == 0 { + return fmt.Errorf("deployment identifier required (e.g., ethereum/knowing-wahoo or ethereum-knowing-wahoo)") + } + deploymentIdentifier := c.Args().First() + return network.Sync(cfg, deploymentIdentifier) }, + }, + { + Name: "delete", + Usage: "Remove network deployment and clean up cluster resources", + ArgsUsage: "/ or -", Action: func(c *cli.Context) error { if c.NArg() == 0 { - return fmt.Errorf("network name required (e.g., ethereum, helios)") + return fmt.Errorf("deployment identifier required (e.g., ethereum/test-deploy or ethereum-test-deploy)") } - networkName := c.Args().First() - return network.Delete(cfg, networkName, c.Bool("force")) + deploymentIdentifier := c.Args().First() + return network.Delete(cfg, deploymentIdentifier) }, }, }, @@ -68,47 +74,62 @@ func buildNetworkInstallCommands(cfg *config.Config) []*cli.Command { var commands []*cli.Command for _, networkName := range networks { - // Parse the embedded helmfile to get env vars - envVars, err := network.ParseEmbeddedNetworkEnvVars(networkName) + // Parse the embedded values template to get fields + fields, err := network.ParseTemplateFields(networkName) if err != nil { // Skip networks we can't parse continue } - // Build flags from env vars - flags := []cli.Flag{} - for _, envVar := range envVars { + // Build flags from template fields + flags := []cli.Flag{ + // id flag is always present (special case - not parsed from template) + &cli.StringFlag{ + Name: "id", + Usage: fmt.Sprintf("Deployment identifier for namespace (e.g., 'my-node' becomes '%s-my-node', defaults to generated petname)", networkName), + Required: false, + }, + // force flag to allow overwriting existing deployments + &cli.BoolFlag{ + Name: "force", + Aliases: []string{"f"}, + Usage: "Overwrite existing deployment configuration if it already exists", + }, + } + + // Add flags from parsed template fields + for _, field := range fields { // Build usage string - usage := envVar.Description + usage := field.Description if usage == "" { - usage = fmt.Sprintf("Override %s", envVar.Name) + usage = fmt.Sprintf("Override %s", field.Name) } // Mark as required if no default value - if envVar.Required { + if field.Required { usage = "[REQUIRED] " + usage } // Add enum options if available - if len(envVar.EnumValues) > 0 { - usage += fmt.Sprintf(" [options: %s]", strings.Join(envVar.EnumValues, ", ")) + if len(field.EnumValues) > 0 { + usage += fmt.Sprintf(" [options: %s]", strings.Join(field.EnumValues, ", ")) } // Add default value - if envVar.DefaultValue != "" { - usage += fmt.Sprintf(" (default: %s)", envVar.DefaultValue) + if field.DefaultValue != "" { + usage += fmt.Sprintf(" (default: %s)", field.DefaultValue) } flags = append(flags, &cli.StringFlag{ - Name: envVar.FlagName, + Name: field.FlagName, Usage: usage, - Required: envVar.Required, + Required: field.Required, }) } // Create the network-specific install command netName := networkName // Capture for closure - netEnvVars := envVars // Capture for validation + netFields := fields // Capture for validation commands = append(commands, &cli.Command{ Name: netName, Usage: fmt.Sprintf("Install %s network", netName), @@ -116,28 +137,32 @@ func buildNetworkInstallCommands(cfg *config.Config) []*cli.Command { Action: func(c *cli.Context) error { // Collect and validate flag values overrides := make(map[string]string) - for _, envVar := range netEnvVars { - value := c.String(envVar.FlagName) + + // Collect id flag (special case - not in parsed fields) + if idValue := c.String("id"); idValue != "" { + overrides["id"] = idValue + } + + // Collect parsed template fields + for _, field := range netFields { + value := c.String(field.FlagName) if value != "" { // Validate enum constraint if defined - if len(envVar.EnumValues) > 0 { - valid := false - for _, enumVal := range envVar.EnumValues { - if value == enumVal { - valid = true - break - } - } + if len(field.EnumValues) > 0 { + valid := slices.Contains(field.EnumValues, value) if !valid { return fmt.Errorf("invalid value '%s' for --%s. Valid options: %s", - value, envVar.FlagName, strings.Join(envVar.EnumValues, ", ")) + value, field.FlagName, strings.Join(field.EnumValues, ", ")) } } - overrides[envVar.FlagName] = value + overrides[field.FlagName] = value } } - return network.Install(cfg, netName, overrides) + // Get force flag + force := c.Bool("force") + + return network.Install(cfg, netName, overrides, force) }, }) } diff --git a/go.mod b/go.mod index e242d12..ac5aa02 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,8 @@ go 1.25 require ( github.com/dustinkirkland/golang-petname v0.0.0-20240428194347-eebcea082ee0 - github.com/google/uuid v1.6.0 github.com/urfave/cli/v2 v2.27.7 + gopkg.in/yaml.v3 v3.0.1 ) require ( diff --git a/go.sum b/go.sum index d4a1559..5b3c61c 100644 --- a/go.sum +++ b/go.sum @@ -2,11 +2,13 @@ github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3 github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/dustinkirkland/golang-petname v0.0.0-20240428194347-eebcea082ee0 h1:aYo8nnk3ojoQkP5iErif5Xxv0Mo0Ga/FR5+ffl/7+Nk= github.com/dustinkirkland/golang-petname v0.0.0-20240428194347-eebcea082ee0/go.mod h1:8AuBTZBRSFqEYBPYULd+NN474/zZBLP+6WeT5S9xlAc= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/urfave/cli/v2 v2.27.7 h1:bH59vdhbjLv3LAvIu6gd0usJHgoTTPhCFib8qqOwXYU= github.com/urfave/cli/v2 v2.27.7/go.mod h1:CyNAG/xg+iAOg0N4MPGZqVmv2rCoP267496AOXUZjA4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/embed/networks/aztec/helmfile.yaml.gotmpl b/internal/embed/networks/aztec/helmfile.yaml.gotmpl index 45279d7..bf7ebbd 100644 --- a/internal/embed/networks/aztec/helmfile.yaml.gotmpl +++ b/internal/embed/networks/aztec/helmfile.yaml.gotmpl @@ -1,25 +1,10 @@ -values: - # @enum mainnet - # @description Blockchain network to deploy - - network: {{ env "AZTEC_NETWORK" | default "mainnet" }} - # @description Attester private key (hex string) - attesterPrivateKey: {{ env "AZTEC_ATTESTER_PRIVATE_KEY" }} - # @description L1 Execution RPC URL override (defaults to local erpc endpoint based on network) - l1ExecutionUrl: {{ env "AZTEC_L1_EXECUTION_URL" | default "" }} - # @description L1 Consensus RPC URL - l1ConsensusUrl: {{ env "AZTEC_L1_CONSENSUS_URL" | default "https://ethereum-beacon-api.publicnode.com" }} - # Computed value: L1 Execution URL - uses override or defaults to erpc endpoint - __l1ExecutionUrl: {{ if (env "AZTEC_L1_EXECUTION_URL") }}{{ env "AZTEC_L1_EXECUTION_URL" }}{{ else }}http://erpc.erpc.svc.cluster.local:4000/rpc/{{ env "AZTEC_NETWORK" | default "mainnet" }}{{ end }} - ---- - repositories: - name: obol url: https://obolnetwork.github.io/helm-charts/ releases: - name: aztec-sequencer - namespace: aztec + namespace: aztec-{{ .Values.id }} createNamespace: true chart: obol/aztec-node version: 0.2.0 @@ -45,20 +30,20 @@ releases: - --network - '{{ .Values.network }}' l1ExecutionUrls: - - '{{ .Values.__l1ExecutionUrl }}' + - '{{ if .Values.l1ExecutionUrl }}{{ .Values.l1ExecutionUrl }}{{ else }}http://erpc.erpc.svc.cluster.local:4000/rpc/{{ .Values.network }}{{ end }}' l1ConsensusUrls: - '{{ .Values.l1ConsensusUrl }}' resources: requests: - cpu: "4" - memory: "16Gi" + cpu: "4" + memory: "16Gi" limits: - cpu: "8" - memory: "32Gi" + cpu: "8" + memory: "32Gi" storage: dataDirectory: /data - dataStoreMapSize: "134217728" - worldStateMapSize: "134217728" + dataStoreMapSize: "134217728" + worldStateMapSize: "134217728" startupProbe: periodSeconds: 60 failureThreshold: 60 @@ -74,7 +59,7 @@ releases: httpPort: 8080 p2p: enabled: true - nodePortEnabled: false + nodePortEnabled: false port: 40400 announcePort: 40400 admin: diff --git a/internal/embed/networks/aztec/values.yaml.gotmpl b/internal/embed/networks/aztec/values.yaml.gotmpl new file mode 100644 index 0000000..07140b9 --- /dev/null +++ b/internal/embed/networks/aztec/values.yaml.gotmpl @@ -0,0 +1,18 @@ +# Configuration via CLI flags +# Template fields populated by obol CLI during network installation + +# @enum mainnet +# @default mainnet +# @description Blockchain network to deploy +network: {{.Network}} + +# @description Attester private key (hex string) +attesterPrivateKey: {{.AttesterPrivateKey}} + +# @default +# @description L1 Execution RPC URL (defaults to ERPC: http://erpc.erpc.svc.cluster.local:4000/rpc/{network}) +l1ExecutionUrl: {{.L1ExecutionUrl}} + +# @default https://ethereum-beacon-api.publicnode.com +# @description L1 Consensus RPC URL +l1ConsensusUrl: {{.L1ConsensusUrl}} diff --git a/internal/embed/networks/ethereum/helmfile.yaml.gotmpl b/internal/embed/networks/ethereum/helmfile.yaml.gotmpl index d024746..cc332ee 100644 --- a/internal/embed/networks/ethereum/helmfile.yaml.gotmpl +++ b/internal/embed/networks/ethereum/helmfile.yaml.gotmpl @@ -1,18 +1,3 @@ -# Configuration via environment variables -# Override with: ETHEREUM_NETWORK, ETHEREUM_EXECUTION_CLIENT, ETHEREUM_CONSENSUS_CLIENT -values: - # @enum mainnet,sepolia,holesky,hoodi - # @description Blockchain network to deploy - - network: {{ env "ETHEREUM_NETWORK" | default "mainnet" }} - # @enum reth,geth,nethermind,besu,erigon,ethereumjs - # @description Execution layer client - executionClient: {{ env "ETHEREUM_EXECUTION_CLIENT" | default "reth" }} - # @enum lighthouse,prysm,teku,nimbus,lodestar,grandine - # @description Consensus layer client - consensusClient: {{ env "ETHEREUM_CONSENSUS_CLIENT" | default "lighthouse" }} - ---- - repositories: - name: ethereum-helm-charts url: https://ethpandaops.github.io/ethereum-helm-charts @@ -20,12 +5,13 @@ repositories: releases: # Create PVCs first - before the Ethereum node - name: ethereum-pvcs - namespace: ethereum + namespace: ethereum-{{ .Values.id }} createNamespace: true chart: . values: # Pass only the values needed for PVC creation - - executionClient: '{{ .Values.executionClient }}' + - id: '{{ .Values.id }}' + executionClient: '{{ .Values.executionClient }}' consensusClient: '{{ .Values.consensusClient }}' network: '{{ .Values.network }}' @@ -33,7 +19,7 @@ releases: # Uses the external ethereum-helm-charts/ethereum-node umbrella chart # Depends on ethereum-pvcs to ensure PVCs exist first - name: ethereum - namespace: ethereum + namespace: ethereum-{{ .Values.id }} createNamespace: true chart: ethereum-helm-charts/ethereum-node needs: [ethereum-pvcs] @@ -54,6 +40,18 @@ releases: - {{ .Values.executionClient }}: enabled: true nameOverride: execution-{{ .Values.executionClient }}-{{ .Values.network }} + labels: + app.kubernetes.io/part-of: obol.stack + obol.stack/chain: ethereum + obol.stack/network: {{ .Values.network }} + obol.stack/type: execution + obol.stack/client: {{ .Values.executionClient }} + podLabels: + app.kubernetes.io/part-of: obol.stack + obol.stack/chain: ethereum + obol.stack/network: {{ .Values.network }} + obol.stack/type: execution + obol.stack/client: {{ .Values.executionClient }} persistence: enabled: true size: 500Gi @@ -64,9 +62,32 @@ releases: - {{ .Values.consensusClient }}: enabled: true nameOverride: consensus-{{ .Values.consensusClient }}-{{ .Values.network }} + labels: + app.kubernetes.io/part-of: obol.stack + obol.stack/chain: ethereum + obol.stack/network: {{ .Values.network }} + obol.stack/type: consensus + obol.stack/client: {{ .Values.consensusClient }} + podLabels: + app.kubernetes.io/part-of: obol.stack + obol.stack/chain: ethereum + obol.stack/network: {{ .Values.network }} + obol.stack/type: consensus + obol.stack/client: {{ .Values.consensusClient }} + extraArgs: + - --execution-endpoint=http://ethereum-execution-{{ .Values.executionClient }}-{{ .Values.network }}:8551 + - --network={{ .Values.network }} persistence: enabled: true size: 200Gi # Use existing PVC with network/client naming: consensus-{client}-{network} existingClaim: consensus-{{ .Values.consensusClient }}-{{ .Values.network }} + # Ingress for Ethereum node + - name: ethereum-ingress + namespace: ethereum-{{ .Values.id }} + chart: . + values: + - executionClient: {{ .Values.executionClient }} + consensusClient: {{ .Values.consensusClient }} + network: {{ .Values.network }} diff --git a/internal/embed/networks/ethereum/templates/ingress.yaml b/internal/embed/networks/ethereum/templates/ingress.yaml index 65d69f5..ca2a5bd 100644 --- a/internal/embed/networks/ethereum/templates/ingress.yaml +++ b/internal/embed/networks/ethereum/templates/ingress.yaml @@ -17,14 +17,14 @@ spec: pathType: ImplementationSpecific backend: service: - name: ethereum-execution + name: ethereum-execution-{{ .Values.executionClient }}-{{ .Values.network }} port: number: 8545 - path: /ethereum/beacon(/|$)(.*) pathType: ImplementationSpecific backend: service: - name: ethereum-beacon + name: ethereum-consensus-{{ .Values.consensusClient }}-{{ .Values.network }} port: number: 5052 {{- end }} diff --git a/internal/embed/networks/ethereum/templates/pvc.yaml b/internal/embed/networks/ethereum/templates/pvc.yaml index 2f05f9c..de4e018 100644 --- a/internal/embed/networks/ethereum/templates/pvc.yaml +++ b/internal/embed/networks/ethereum/templates/pvc.yaml @@ -5,7 +5,7 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: execution-{{ .Values.executionClient }}-{{ .Values.network }} - namespace: ethereum + namespace: ethereum-{{ .Values.id }} spec: accessModes: - ReadWriteOnce @@ -19,7 +19,7 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: consensus-{{ .Values.consensusClient }}-{{ .Values.network }} - namespace: ethereum + namespace: ethereum-{{ .Values.id }} spec: accessModes: - ReadWriteOnce diff --git a/internal/embed/networks/ethereum/values.yaml.gotmpl b/internal/embed/networks/ethereum/values.yaml.gotmpl new file mode 100644 index 0000000..c119259 --- /dev/null +++ b/internal/embed/networks/ethereum/values.yaml.gotmpl @@ -0,0 +1,17 @@ +# Configuration via CLI flags +# Template fields populated by obol CLI during network installation + +# @enum mainnet,sepolia,holesky,hoodi +# @default mainnet +# @description Blockchain network to deploy +network: {{.Network}} + +# @enum reth,geth,nethermind,besu,erigon,ethereumjs +# @default reth +# @description Execution layer client +executionClient: {{.ExecutionClient}} + +# @enum lighthouse,prysm,teku,nimbus,lodestar,grandine +# @default lighthouse +# @description Consensus layer client +consensusClient: {{.ConsensusClient}} diff --git a/internal/embed/networks/helios/helmfile.yaml.gotmpl b/internal/embed/networks/helios/helmfile.yaml.gotmpl index dbe4043..ff1040f 100644 --- a/internal/embed/networks/helios/helmfile.yaml.gotmpl +++ b/internal/embed/networks/helios/helmfile.yaml.gotmpl @@ -1,27 +1,13 @@ # Helios Light Client Network # Provides an Ethereum light client that can be used as an RPC endpoint -# Configuration via environment variables -# Override with: HELIOS_NETWORK, HELIOS_CONSENSUS_RPC, HELIOS_EXECUTION_RPC, HELIOS_CHECKPOINT -# @enum mainnet -# @description Ethereum network (only mainnet supported currently) -# @description Consensus RPC endpoint URL -# @description Execution RPC endpoint URL -# @description Checkpoint block root (must be first slot of epoch) -values: - - network: {{ env "HELIOS_NETWORK" | default "mainnet" }} - consensusRpc: {{ env "HELIOS_CONSENSUS_RPC" | default "http://testing.mainnet.beacon-api.nimbus.team" }} - executionRpc: {{ env "HELIOS_EXECUTION_RPC" | default "https://eth.drpc.org" }} - ---- - repositories: - name: obol url: https://obolnetwork.github.io/helm-charts/ releases: - name: helios - namespace: helios + namespace: helios-{{ .Values.id }} createNamespace: true chart: obol/helios version: 0.1.4 diff --git a/internal/embed/networks/helios/values.yaml.gotmpl b/internal/embed/networks/helios/values.yaml.gotmpl new file mode 100644 index 0000000..baed67f --- /dev/null +++ b/internal/embed/networks/helios/values.yaml.gotmpl @@ -0,0 +1,17 @@ +# Helios Light Client Network +# Provides an Ethereum light client that can be used as an RPC endpoint +# Configuration via CLI flags +# Template fields populated by obol CLI during network installation + +# @enum mainnet +# @default mainnet +# @description Ethereum network (only mainnet supported currently) +network: {{.Network}} + +# @default http://testing.mainnet.beacon-api.nimbus.team +# @description Consensus RPC endpoint URL +consensusRpc: {{.ConsensusRpc}} + +# @default https://eth.drpc.org +# @description Execution RPC endpoint URL +executionRpc: {{.ExecutionRpc}} diff --git a/internal/network/network.go b/internal/network/network.go index 80f9403..f3e1d7d 100644 --- a/internal/network/network.go +++ b/internal/network/network.go @@ -1,32 +1,20 @@ package network import ( + "bytes" "fmt" "os" "os/exec" "path/filepath" "strings" + "text/template" "github.com/ObolNetwork/obol-stack/internal/config" "github.com/ObolNetwork/obol-stack/internal/embed" + "github.com/dustinkirkland/golang-petname" + "gopkg.in/yaml.v3" ) -// TODO: Network Management System -// -// The network system manages blockchain network configurations using embedded helmfiles. -// -// Architecture: -// - Embedded networks: internal/embed/networks//helmfile.yaml -// - Installed networks: $OBOL_CONFIG_DIR/networks//helmfile.yaml -// - Each network may configure endpoints that are proxied through ERPC -// -// Implementation needed: -// 1. List() - Traverse and display available networks from internal/embed/networks -// 2. Install(cfg, network, overrides) - Copy embedded network to OBOL_CONFIG_DIR/networks and deploy via helmfile sync -// 3. Delete(cfg, network) - Remove network config and associated k8s namespaces -// -// See: plan.md for detailed design - // List displays all available networks from the embedded filesystem func List(cfg *config.Config) error { fmt.Println("Available networks:") @@ -52,92 +40,313 @@ func List(cfg *config.Config) error { return nil } -// Install deploys a network by extracting it to a temp directory and running helmfile sync -func Install(cfg *config.Config, network string, overrides map[string]string) error { +// Install creates a network configuration by executing Go templates and saving to config directory +func Install(cfg *config.Config, network string, overrides map[string]string, force bool) error { fmt.Printf("Installing network: %s\n", network) - // Parse embedded helmfile to get environment variables - envVars, err := ParseEmbeddedNetworkEnvVars(network) + // Generate deployment ID if not provided in overrides (use petname) + id, hasId := overrides["id"] + if !hasId || id == "" { + id = petname.Generate(2, "-") + overrides["id"] = id + fmt.Printf("Generated deployment ID: %s\n", id) + } else { + fmt.Printf("Using deployment ID: %s\n", id) + } + + // Check if deployment already exists + deploymentDir := filepath.Join(cfg.ConfigDir, "networks", network, id) + if _, err := os.Stat(deploymentDir); err == nil { + // Directory exists + if !force { + return fmt.Errorf("deployment already exists: %s/%s\n"+ + "Directory: %s\n"+ + "Use --force or -f to overwrite the existing configuration", network, id, deploymentDir) + } + fmt.Printf("⚠️ WARNING: Overwriting existing deployment at %s\n", deploymentDir) + } + + // Parse embedded values template to get fields + fields, err := ParseTemplateFields(network) if err != nil { - return fmt.Errorf("failed to parse embedded helmfile: %w", err) + return fmt.Errorf("failed to parse embedded values template: %w", err) } - // Display configuration and set environment variables - if len(envVars) > 0 { - fmt.Println("Configuration:") - for _, envVar := range envVars { - value := envVar.DefaultValue + // Build template data from CLI flags and defaults + templateData := make(map[string]string) + + fmt.Println("Configuration:") + fmt.Printf(" deployment id: %s (from directory structure)\n", id) - // Check if there's an override from CLI flags - if overrideValue, ok := overrides[envVar.FlagName]; ok { - value = overrideValue - fmt.Printf(" %s = %s (from --%s)\n", envVar.Name, value, envVar.FlagName) - } else { - fmt.Printf(" %s = %s (default)\n", envVar.Name, value) - } + // Process parsed fields + for _, field := range fields { + value := field.DefaultValue - // Set environment variable in process for helmfile to read - os.Setenv(envVar.Name, value) + // Check if there's an override from CLI flags + if overrideValue, ok := overrides[field.FlagName]; ok { + value = overrideValue + fmt.Printf(" %s = %s (from --%s)\n", field.Name, value, field.FlagName) + } else if field.Required && value == "" { + // Required field with no value provided + return fmt.Errorf("missing required flag: --%s", field.FlagName) + } else if value != "" { + fmt.Printf(" %s = %s (default)\n", field.Name, value) + } else { + // Optional field with empty default + fmt.Printf(" %s = (empty, optional)\n", field.Name) } + + // Add to template data using field name (e.g., "Network", "ExecutionClient") + templateData[field.Name] = value } - // Create temporary directory for network files - tmpDir, err := os.MkdirTemp("", fmt.Sprintf("obol-network-%s-*", network)) + // Read the embedded values template + valuesContent, err := embed.ReadEmbeddedNetworkFile(network, "values.yaml.gotmpl") if err != nil { - return fmt.Errorf("failed to create temp directory: %w", err) + return fmt.Errorf("failed to read embedded values: %w", err) } - defer os.RemoveAll(tmpDir) - fmt.Printf("Extracting network to temporary directory: %s\n", tmpDir) + // Parse and execute the Go template for values + tmpl, err := template.New("values").Parse(string(valuesContent)) + if err != nil { + return fmt.Errorf("failed to parse values template: %w", err) + } - // Copy embedded network to temp directory - if err := embed.CopyNetwork(network, tmpDir); err != nil { - return fmt.Errorf("failed to copy network: %w", err) + var buf bytes.Buffer + if err := tmpl.Execute(&buf, templateData); err != nil { + return fmt.Errorf("failed to execute values template: %w", err) } - // Use .yaml.gotmpl extension so helmfile processes Go templates - helmfilePath := filepath.Join(tmpDir, "helmfile.yaml.gotmpl") - fmt.Println("Deploying network via helmfile sync") + // Validate that the generated content is valid YAML + var yamlCheck interface{} + if err := yaml.Unmarshal(buf.Bytes(), &yamlCheck); err != nil { + return fmt.Errorf("generated values.yaml contains invalid YAML syntax: %w\n"+ + "This may be caused by special characters in your input values.\n"+ + "Generated content:\n%s", err, buf.String()) + } - // Get kubeconfig path - kubeconfigPath := filepath.Join(cfg.ConfigDir, "kubeconfig.yaml") + // Create deployment directory in config: networks/// + // (deploymentDir already defined earlier for existence check) + if err := os.MkdirAll(deploymentDir, 0755); err != nil { + return fmt.Errorf("failed to create deployment directory: %w", err) + } - // Build helmfile command with PATH including binDir - cmd := exec.Command("helmfile", "-f", helmfilePath, "sync") - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr + fmt.Printf("Saving configuration to: %s\n", deploymentDir) + + // Write the templated values.yaml (plain YAML, no more templating) + valuesPath := filepath.Join(deploymentDir, "values.yaml") + if err := os.WriteFile(valuesPath, buf.Bytes(), 0644); err != nil { + return fmt.Errorf("failed to write values.yaml: %w", err) + } - // Set PATH to include binDir so helmfile can be found - pathEnv := os.Getenv("PATH") - if cfg.BinDir != "" { - if !strings.Contains(pathEnv, cfg.BinDir) { - pathEnv = cfg.BinDir + string(os.PathListSeparator) + pathEnv + // Copy network files (helmfile.yaml.gotmpl, Chart.yaml, templates/, etc.) + if err := embed.CopyNetwork(network, deploymentDir); err != nil { + return fmt.Errorf("failed to copy network files: %w", err) + } + + // Remove values.yaml.gotmpl if it was copied (we already generated values.yaml) + valuesTemplatePath := filepath.Join(deploymentDir, "values.yaml.gotmpl") + os.Remove(valuesTemplatePath) // Ignore error if file doesn't exist + + fmt.Printf("\nNetwork configuration saved successfully!\n") + fmt.Printf("Deployment: %s/%s\n", network, id) + fmt.Printf("Location: %s\n", deploymentDir) + fmt.Printf("\nFiles generated:\n") + fmt.Printf(" - values.yaml: Configuration values\n") + fmt.Printf(" - helmfile.yaml.gotmpl: Deployment definition\n") + fmt.Printf("\nTo deploy, run: obol network sync %s/%s\n", network, id) + + return nil +} + +// Sync deploys or updates a network configuration to the cluster using helmfile +func Sync(cfg *config.Config, deploymentIdentifier string) error { + // Parse deployment identifier (supports both "ethereum/knowing-wahoo" and "ethereum-knowing-wahoo") + var networkName, deploymentID string + + // Try slash separator first + if strings.Contains(deploymentIdentifier, "/") { + parts := strings.SplitN(deploymentIdentifier, "/", 2) + if len(parts) != 2 { + return fmt.Errorf("invalid deployment identifier format. Use: / or -") + } + networkName = parts[0] + deploymentID = parts[1] + } else { + // Try to split by first dash that separates network from ID + // Network names are expected to be single words (ethereum, helios, aztec) + parts := strings.SplitN(deploymentIdentifier, "-", 2) + if len(parts) != 2 { + return fmt.Errorf("invalid deployment identifier format. Use: / or -") } + networkName = parts[0] + deploymentID = parts[1] + } + + fmt.Printf("Syncing deployment: %s/%s\n", networkName, deploymentID) + + // Locate deployment directory + deploymentDir := filepath.Join(cfg.ConfigDir, "networks", networkName, deploymentID) + if _, err := os.Stat(deploymentDir); os.IsNotExist(err) { + return fmt.Errorf("deployment not found: %s\nDirectory: %s", deploymentIdentifier, deploymentDir) + } + + // Check if helmfile.yaml.gotmpl or helmfile.yaml exists (prefer .gotmpl for Helmfile v1+) + helmfilePath := filepath.Join(deploymentDir, "helmfile.yaml.gotmpl") + if _, err := os.Stat(helmfilePath); os.IsNotExist(err) { + // Fallback to helmfile.yaml for backwards compatibility + helmfilePath = filepath.Join(deploymentDir, "helmfile.yaml") + if _, err := os.Stat(helmfilePath); os.IsNotExist(err) { + return fmt.Errorf("helmfile.yaml or helmfile.yaml.gotmpl not found in deployment directory: %s", deploymentDir) + } + } + + // Check if values.yaml exists + valuesPath := filepath.Join(deploymentDir, "values.yaml") + if _, err := os.Stat(valuesPath); os.IsNotExist(err) { + return fmt.Errorf("values.yaml not found in deployment directory: %s", deploymentDir) } - // Set environment with PATH and KUBECONFIG + // Check if kubeconfig exists (cluster must be running) + kubeconfigPath := filepath.Join(cfg.ConfigDir, "kubeconfig.yaml") + if _, err := os.Stat(kubeconfigPath); os.IsNotExist(err) { + return fmt.Errorf("cluster not running. Run 'obol stack up' first") + } + + // Get helmfile binary path + helmfileBinary := filepath.Join(cfg.BinDir, "helmfile") + if _, err := os.Stat(helmfileBinary); os.IsNotExist(err) { + return fmt.Errorf("helmfile not found at %s", helmfileBinary) + } + + fmt.Printf("Deployment directory: %s\n", deploymentDir) + fmt.Printf("Using: %s\n", filepath.Base(helmfilePath)) + fmt.Printf("Deployment ID: %s (from directory structure)\n", deploymentID) + fmt.Printf("Running helmfile sync...\n\n") + + // Execute helmfile sync with explicit file, state-values-file, and id from directory structure + cmd := exec.Command(helmfileBinary, "-f", helmfilePath, "sync", + "--state-values-file", valuesPath, + "--state-values-set", fmt.Sprintf("id=%s", deploymentID)) + cmd.Dir = deploymentDir // Run in deployment directory cmd.Env = append(os.Environ(), - "PATH="+pathEnv, - "KUBECONFIG="+kubeconfigPath, + fmt.Sprintf("KUBECONFIG=%s", kubeconfigPath), ) + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr if err := cmd.Run(); err != nil { return fmt.Errorf("helmfile sync failed: %w", err) } - fmt.Printf("Network %s installed successfully\n", network) + fmt.Printf("\nDeployment synced successfully!\n") + fmt.Printf("Namespace: %s-%s\n", networkName, deploymentID) + fmt.Printf("\nTo check status: obol kubectl get all -n %s-%s\n", networkName, deploymentID) + fmt.Printf("To view logs: obol kubectl logs -n %s-%s \n", networkName, deploymentID) + fmt.Printf("To access dashboard: obol k9s -n %s-%s\n", networkName, deploymentID) return nil } -// Delete removes the network configuration and cluster resources -func Delete(cfg *config.Config, network string, force bool) error { - fmt.Printf("Deleting network: %s\n", network) - fmt.Println("TODO: Implement network deletion") - fmt.Println(" 1. Remove $OBOL_CONFIG_DIR/networks/{network}") - fmt.Println(" 2. Identify and delete associated k8s namespaces") - fmt.Println(" 3. Handle ERPC re-configuration if needed") - fmt.Println(" 4. Confirm cleanup completion") +// Delete removes the network deployment configuration and cluster resources +func Delete(cfg *config.Config, deploymentIdentifier string) error { + // Parse deployment identifier (supports both "ethereum/knowing-wahoo" and "ethereum-knowing-wahoo") + var networkName, deploymentID string + + // Try slash separator first + if strings.Contains(deploymentIdentifier, "/") { + parts := strings.SplitN(deploymentIdentifier, "/", 2) + if len(parts) != 2 { + return fmt.Errorf("invalid deployment identifier format. Use: / or -") + } + networkName = parts[0] + deploymentID = parts[1] + } else { + // Try to split by first dash that separates network from ID + parts := strings.SplitN(deploymentIdentifier, "-", 2) + if len(parts) != 2 { + return fmt.Errorf("invalid deployment identifier format. Use: / or -") + } + networkName = parts[0] + deploymentID = parts[1] + } + + namespaceName := fmt.Sprintf("%s-%s", networkName, deploymentID) + deploymentDir := filepath.Join(cfg.ConfigDir, "networks", networkName, deploymentID) + + fmt.Printf("Deleting deployment: %s/%s\n", networkName, deploymentID) + fmt.Printf("Namespace: %s\n", namespaceName) + fmt.Printf("Config directory: %s\n", deploymentDir) + + // Check if config directory exists + configExists := false + if _, err := os.Stat(deploymentDir); err == nil { + configExists = true + } + + // Check if namespace exists in cluster + namespaceExists := false + kubeconfigPath := filepath.Join(cfg.ConfigDir, "kubeconfig.yaml") + if _, err := os.Stat(kubeconfigPath); err == nil { + // Cluster is running, check for namespace + kubectlBinary := filepath.Join(cfg.BinDir, "kubectl") + cmd := exec.Command(kubectlBinary, "get", "namespace", namespaceName) + cmd.Env = append(os.Environ(), fmt.Sprintf("KUBECONFIG=%s", kubeconfigPath)) + if err := cmd.Run(); err == nil { + namespaceExists = true + } + } + + // Display what will be deleted + fmt.Println("\nResources to be deleted:") + if namespaceExists { + fmt.Printf(" ✓ Kubernetes namespace: %s (including all resources)\n", namespaceName) + } else { + fmt.Printf(" - Kubernetes namespace: %s (not found)\n", namespaceName) + } + if configExists { + fmt.Printf(" ✓ Configuration directory: %s\n", deploymentDir) + } else { + fmt.Printf(" - Configuration directory: %s (not found)\n", deploymentDir) + } + + // Check if there's anything to delete + if !namespaceExists && !configExists { + return fmt.Errorf("deployment not found: %s", deploymentIdentifier) + } + + // Delete Kubernetes namespace + if namespaceExists { + fmt.Printf("\nDeleting namespace %s...\n", namespaceName) + kubectlBinary := filepath.Join(cfg.BinDir, "kubectl") + cmd := exec.Command(kubectlBinary, "delete", "namespace", namespaceName, "--force", "--grace-period=0") + cmd.Env = append(os.Environ(), fmt.Sprintf("KUBECONFIG=%s", kubeconfigPath)) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to delete namespace: %w", err) + } + fmt.Println("Namespace deleted successfully") + } + + // Delete configuration directory + if configExists { + fmt.Printf("Deleting configuration directory...\n") + if err := os.RemoveAll(deploymentDir); err != nil { + return fmt.Errorf("failed to delete config directory: %w", err) + } + fmt.Println("Configuration deleted successfully") + + // Check if parent network directory is empty and remove it + networkDir := filepath.Join(cfg.ConfigDir, "networks", networkName) + entries, err := os.ReadDir(networkDir) + if err == nil && len(entries) == 0 { + os.Remove(networkDir) // Clean up empty network directory + } + } + + fmt.Printf("\n✓ Deployment %s/%s deleted successfully!\n", networkName, deploymentID) return nil } diff --git a/internal/network/parser.go b/internal/network/parser.go index 81ead6c..3681e31 100644 --- a/internal/network/parser.go +++ b/internal/network/parser.go @@ -3,147 +3,202 @@ package network import ( "fmt" "regexp" + "sort" "strings" + "text/template" + "text/template/parse" "github.com/ObolNetwork/obol-stack/internal/embed" ) -// EnvVar represents an environment variable with its default value -type EnvVar struct { +// TemplateField represents a template field with its configuration +type TemplateField struct { Name string DefaultValue string - FlagName string // CLI flag name derived from env var name + FlagName string // CLI flag name derived from field name Description string // Human-readable description from @description EnumValues []string // Valid enum values from @enum - Required bool // Whether this env var is required (no default value) + Required bool // Whether this field is required (no default value) } -// envVarToFlagName converts an environment variable name to a CLI flag name -// Automatically strips network-specific prefixes (e.g., ETHEREUM_, AZTEC_, HELIOS_) -// Example: ETHEREUM_NETWORK -> network -// Example: AZTEC_ATTESTER_PRIVATE_KEY -> attester-private-key -func envVarToFlagName(envName string) string { - // Find the first underscore to detect network prefix pattern - // Network-specific env vars follow pattern: NETWORK_NAME_* - parts := strings.SplitN(envName, "_", 2) - if len(parts) == 2 { - // Strip the network prefix (everything before first underscore) - envName = parts[1] +// extractTemplateFields parses a Go template and extracts all field references +// Returns a map of field names to their line numbers for annotation matching +func extractTemplateFields(content string) (map[string]int, error) { + // Parse the template + tmpl, err := template.New("helmfile").Parse(content) + if err != nil { + return nil, fmt.Errorf("failed to parse template: %w", err) } - // Convert to lowercase and replace underscores with hyphens - flagName := strings.ToLower(envName) - flagName = strings.ReplaceAll(flagName, "_", "-") + fields := make(map[string]int) + lines := strings.Split(content, "\n") - return flagName -} + // Walk the template AST to find field references + var walkNodes func(node parse.Node) + walkNodes = func(node parse.Node) { + if node == nil { + return + } -// ParseEmbeddedNetworkEnvVars extracts environment variables from an embedded network helmfile -func ParseEmbeddedNetworkEnvVars(networkName string) ([]EnvVar, error) { - // Read the embedded helmfile - content, err := embed.ReadEmbeddedNetworkFile(networkName, "helmfile.yaml.gotmpl") - if err != nil { - return nil, fmt.Errorf("failed to read embedded helmfile: %w", err) + switch n := node.(type) { + case *parse.ListNode: + if n != nil { + for _, child := range n.Nodes { + walkNodes(child) + } + } + case *parse.ActionNode: + if n.Pipe != nil { + for _, cmd := range n.Pipe.Cmds { + for _, arg := range cmd.Args { + if field, ok := arg.(*parse.FieldNode); ok { + // Extract field name (e.g., .Network -> Network) + if len(field.Ident) > 0 { + fieldName := field.Ident[0] + // Find line number of this field in the content + for i, line := range lines { + if strings.Contains(line, "{{."+fieldName+"}}") { + fields[fieldName] = i + break + } + } + } + } + } + } + } + case *parse.IfNode: + walkNodes(n.List) + walkNodes(n.ElseList) + case *parse.RangeNode: + walkNodes(n.List) + walkNodes(n.ElseList) + case *parse.WithNode: + walkNodes(n.List) + walkNodes(n.ElseList) + case *parse.TemplateNode: + walkNodes(n.Pipe) + } } - // Generate expected prefix from network name (e.g., "aztec" -> "AZTEC_") - networkPrefix := strings.ToUpper(networkName) + "_" + // Walk the template tree + if tmpl.Tree != nil && tmpl.Tree.Root != nil { + walkNodes(tmpl.Tree.Root) + } - lines := strings.Split(string(content), "\n") - var envVars []EnvVar - seen := make(map[string]bool) + return fields, nil +} - // Track annotations from preceding comment lines - var currentEnum []string - var currentDesc string +// parseAnnotationsFromLines extracts annotations from comment lines preceding a field +// Returns enum values, default value, description, and whether a default was specified +func parseAnnotationsFromLines(lines []string, fieldLineNum int) ([]string, string, string, bool) { + var enumValues []string + var defaultValue string + var description string + var hasDefault bool + + // Look backwards from the field line to find annotations + for i := fieldLineNum - 1; i >= 0; i-- { + line := lines[i] + trimmed := strings.TrimSpace(line) + + // Stop if we hit a non-comment line or empty line + if !strings.HasPrefix(trimmed, "#") { + break + } - for _, line := range lines { // Parse @enum annotation if enumMatch := regexp.MustCompile(`#\s*@enum\s+(.+)`).FindStringSubmatch(line); enumMatch != nil { enumStr := strings.TrimSpace(enumMatch[1]) - currentEnum = strings.Split(enumStr, ",") - for i := range currentEnum { - currentEnum[i] = strings.TrimSpace(currentEnum[i]) + enumValues = strings.Split(enumStr, ",") + for j := range enumValues { + enumValues[j] = strings.TrimSpace(enumValues[j]) } - continue + } + + // Parse @default annotation (value is optional, may be empty string) + if defaultMatch := regexp.MustCompile(`#\s*@default\s*(.*)`).FindStringSubmatch(line); defaultMatch != nil { + defaultValue = strings.TrimSpace(defaultMatch[1]) + hasDefault = true } // Parse @description annotation if descMatch := regexp.MustCompile(`#\s*@description\s+(.+)`).FindStringSubmatch(line); descMatch != nil { - currentDesc = strings.TrimSpace(descMatch[1]) - continue + description = strings.TrimSpace(descMatch[1]) } + } - // Parse env var line with default: {{ env "VAR_NAME" | default "value" }} - reWithDefault := regexp.MustCompile(`{{\s*env\s+"([^"]+)"\s*\|\s*default\s+"([^"]*)"\s*}}`) - if envMatch := reWithDefault.FindStringSubmatch(line); envMatch != nil { - envName := envMatch[1] - defaultValue := envMatch[2] - - // Only include env vars that match the network prefix - if !strings.HasPrefix(envName, networkPrefix) { - continue - } + return enumValues, defaultValue, description, hasDefault +} - // Skip duplicates - if seen[envName] { - continue - } - seen[envName] = true - - // Convert env var name to CLI flag name (strips network prefix) - flagName := envVarToFlagName(envName) - - envVar := EnvVar{ - Name: envName, - DefaultValue: defaultValue, - FlagName: flagName, - Description: currentDesc, - EnumValues: currentEnum, - Required: false, // Has default value, so optional - } - envVars = append(envVars, envVar) +// ParseTemplateFields extracts template fields from an embedded network values file +// Uses Go template parsing to identify field references and their annotations +func ParseTemplateFields(networkName string) ([]TemplateField, error) { + // Read the embedded values template + content, err := embed.ReadEmbeddedNetworkFile(networkName, "values.yaml.gotmpl") + if err != nil { + return nil, fmt.Errorf("failed to read embedded values: %w", err) + } - // Reset annotations for next variable - currentEnum = nil - currentDesc = "" - continue - } + // Extract template fields using Go template parser + fieldMap, err := extractTemplateFields(string(content)) + if err != nil { + return nil, fmt.Errorf("failed to extract template fields: %w", err) + } - // Parse required env var line (no default): {{ env "VAR_NAME" }} - reRequired := regexp.MustCompile(`{{\s*env\s+"([^"]+)"\s*}}`) - if envMatch := reRequired.FindStringSubmatch(line); envMatch != nil { - envName := envMatch[1] + lines := strings.Split(string(content), "\n") - // Only include env vars that match the network prefix - if !strings.HasPrefix(envName, networkPrefix) { - continue - } + // Create a sorted list of field names by line number for deterministic ordering + type fieldWithLine struct { + name string + line int + } + sortedFields := make([]fieldWithLine, 0, len(fieldMap)) + for fieldName, lineNum := range fieldMap { + sortedFields = append(sortedFields, fieldWithLine{name: fieldName, line: lineNum}) + } + sort.Slice(sortedFields, func(i, j int) bool { + return sortedFields[i].line < sortedFields[j].line + }) + + // Process fields in order they appear in the file + fields := make([]TemplateField, 0, len(sortedFields)) + for _, f := range sortedFields { + enumValues, defaultValue, description, hasDefault := parseAnnotationsFromLines(lines, f.line) + + // Convert field name to CLI flag name (e.g., ExecutionClient -> execution-client) + flagName := fieldNameToFlagName(f.name) + + // Determine if field is required (no @default annotation) + required := !hasDefault + + field := TemplateField{ + Name: f.name, + DefaultValue: defaultValue, + FlagName: flagName, + Description: description, + EnumValues: enumValues, + Required: required, + } + fields = append(fields, field) + } - // Skip duplicates - if seen[envName] { - continue - } - seen[envName] = true - - // Convert env var name to CLI flag name (strips network prefix) - flagName := envVarToFlagName(envName) - - envVar := EnvVar{ - Name: envName, - DefaultValue: "", - FlagName: flagName, - Description: currentDesc, - EnumValues: currentEnum, - Required: true, // No default value, so required - } - envVars = append(envVars, envVar) + return fields, nil +} - // Reset annotations for next variable - currentEnum = nil - currentDesc = "" +// fieldNameToFlagName converts a template field name to a CLI flag name +// Example: ExecutionClient -> execution-client +// Example: Network -> network +func fieldNameToFlagName(fieldName string) string { + // Insert hyphen before uppercase letters (except first) + var result strings.Builder + for i, r := range fieldName { + if i > 0 && r >= 'A' && r <= 'Z' { + result.WriteRune('-') } + result.WriteRune(r) } - return envVars, nil + // Convert to lowercase + return strings.ToLower(result.String()) } diff --git a/obolup.sh b/obolup.sh index 94d2716..2741a53 100755 --- a/obolup.sh +++ b/obolup.sh @@ -50,9 +50,9 @@ fi # Pinned dependency versions # Update these versions to upgrade dependencies across all installations readonly KUBECTL_VERSION="1.31.0" -readonly HELM_VERSION="3.16.2" +readonly HELM_VERSION="3.19.1" readonly K3D_VERSION="5.8.3" -readonly HELMFILE_VERSION="0.169.1" +readonly HELMFILE_VERSION="1.2.2" readonly K9S_VERSION="0.32.5" readonly HELM_DIFF_VERSION="3.9.11" @@ -190,6 +190,21 @@ create_binary_symlink() { local global_path="$2" local local_path="$OBOL_BIN_DIR/$binary_name" + # Detect and prevent circular symlinks. + # This can happen if a global binary path (e.g. from a previous dev install) + # already points to the location this script is trying to manage. + if [[ -L "$global_path" ]]; then + local link_target + link_target=$(readlink "$global_path" || echo "") # Handle readlink failure + + if [[ "$link_target" == "$local_path" ]]; then + log_warn "Circular symlink detected for $binary_name." + log_warn " $global_path -> $local_path" + log_warn "Ignoring global binary to prevent loop. Will install fresh copy." + return 1 # Signal failure to prevent using this global binary + fi + fi + # Check if local path already exists if [[ -e "$local_path" || -L "$local_path" ]]; then # Check if it's already a symlink to the correct target @@ -1198,6 +1213,7 @@ configure_path() { add_to_profile "$profile" echo "" log_info "PATH updated for future sessions" + echo "" log_info "To use immediately in this session, run:" echo "" echo " export PATH=\"$OBOL_BIN_DIR:\$PATH\"" @@ -1352,4 +1368,4 @@ main() { } # Run main -main +main \ No newline at end of file