Commands generator binary (#875)

<!--- Note to EXTERNAL Contributors -->
<!-- Thanks for opening a PR! 
If it is a significant code change, please **make sure there is an open
issue** for this.
We work best with you when we have accepted the idea first before you
code. -->

<!--- For ALL Contributors 👇 -->

## What was changed
<!-- Describe what has changed in this PR -->

Make `gen-commands` and `gen-docs` a binary that can be used any YAML
input.

## Why?
<!-- Tell your future self why have you made these changes -->

Temporal wants to re-use the `commandsgen` tool for other _internal_
CLIs.

**It is not supported for users outside of Temporal.** 

## Checklist
<!--- add/delete as needed --->

1. Closes <!-- add issue number here -->

2. How was this tested:
<!--- Please describe how you tested your changes/how we can test them
-->

3. Any docs updates needed?
<!--- update README if applicable
      or point out where to update docs.temporal.io -->

Author: stephanos

.github/workflows/ci.yaml

@@ -10,10 +10,17 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest, macos-13, windows-latest, ubuntu-arm]
+        os:
+          [
+            ubuntu-latest,
+            macos-latest,
+            macos-15-intel,
+            windows-latest,
+            ubuntu-arm,
+          ]
         include:
           - os: ubuntu-latest
-            checkGenCodeTarget: true
+            checkGenCommands: true
             cloudTestTarget: true
           - os: ubuntu-arm
             runsOn: buildjet-4vcpu-ubuntu-2204-arm
@@ -51,11 +58,16 @@ jobs:
           retention-days: 14
 
       - name: Regen code, confirm unchanged
-        if: ${{ matrix.checkGenCodeTarget }}
+        if: ${{ matrix.checkGenCommands }}
         run: |
-          go run ./internal/cmd/gen-commands
+          go run ./internal/cmd/gen-commands -input internal/temporalcli/commands.yaml -pkg temporalcli -context "*CommandContext" > internal/temporalcli/commands.gen.go
           git diff --exit-code
 
+      - name: Generate docs, confirm working
+        if: ${{ matrix.checkGenCommands }}
+        run: |
+          go run ./internal/cmd/gen-docs -input internal/temporalcli/commands.yaml -output dist/docs
+
       - name: Test cloud mTLS
         if: ${{ matrix.cloudTestTarget && env.HAS_SECRETS == 'true' }}
         env:

CONTRIBUTING.md

@@ -20,18 +20,18 @@ Example to run a single test case:
 
 ## Adding/updating commands
 
-First, update [commands.yml](internal/commandsgen/commands.yml) following the rules in that file. Then to regenerate the
-[commands.gen.go](internal/commands.gen.go) file from code, simply run:
+First, update [commands.yaml](internal/temporalcli/commands.yaml) following the rules in that file. Then to regenerate the
+[commands.gen.go](internal/temporalcli/commands.gen.go) file from code, run:
 
-    go run ./internal/cmd/gen-commands
+    go run ./internal/cmd/gen-commands -input internal/temporalcli/commands.yaml -pkg temporalcli -context "*CommandContext" > internal/temporalcli/commands.gen.go
 
 This will expect every non-parent command to have a `run` method, so for new commands developers will have to implement
 `run` on the new command in a separate file before it will compile.
 
 Once a command is updated, the CI will automatically generate new docs
 and create a PR in the Documentation repo with the corresponding updates. To generate these docs locally, run:
 
-    go run ./internal/cmd/gen-docs
+    go run ./internal/cmd/gen-docs -input internal/temporalcli/commands.yaml -output dist/docs
 
 This will auto-generate a new set of docs to `dist/docs/`. If a new root command is added, a new file will be automatically generated, like `temporal activity` and `activity.mdx`.
 

Makefile

@@ -1,11 +1,14 @@
-.PHONY: all gen build fmt-imports
+.PHONY: all gen gen-docs build
 
 all: gen build
 
-gen: internal/commands.gen.go
+gen: internal/temporalcli/commands.gen.go
 
-internal/commands.gen.go: internal/commandsgen/commands.yml
-	go run ./internal/cmd/gen-commands
+internal/temporalcli/commands.gen.go: internal/temporalcli/commands.yaml
+	go run ./internal/cmd/gen-commands -input $< -pkg temporalcli -context "*CommandContext" > $@
+
+gen-docs: internal/temporalcli/commands.yaml
+	go run ./internal/cmd/gen-docs -input $< -output dist/docs
 
 build:
 	go build ./cmd/temporal

internal/cmd/gen-commands/README.md

@@ -0,0 +1,4 @@
+This package provides a CLI command generator for Temporal CLI applications.
+It is designed specifically for the needs of Temporal CLIs, and not general-purpose command-line tools.
+
+Backwards-compatibility is not guaranteed.

internal/cmd/gen-commands/main.go

@@ -1,11 +1,10 @@
 package main
 
 import (
+	"flag"
 	"fmt"
 	"log"
 	"os"
-	"path/filepath"
-	"runtime"
 
 	"github.com/temporalio/cli/internal/commandsgen"
 )
@@ -17,26 +16,42 @@ func main() {
 }
 
 func run() error {
-	// Get commands dir
-	_, file, _, _ := runtime.Caller(0)
-	genCommandsDir := filepath.Dir(file)
-	commandsDir := filepath.Join(genCommandsDir, "../../")
+	var (
+		pkg         string
+		contextType string
+		inputFile   string
+	)
+
+	flag.StringVar(&pkg, "pkg", "main", "Package name for generated code")
+	flag.StringVar(&contextType, "context", "*CommandContext", "Context type for generated code")
+	flag.StringVar(&inputFile, "input", "", "Input YAML file (required)")
+	flag.Parse()
+
+	// Read input from file
+	if inputFile == "" {
+		return fmt.Errorf("-input flag is required")
+	}
+	yamlBytes, err := os.ReadFile(inputFile)
+	if err != nil {
+		return fmt.Errorf("failed reading input: %w", err)
+	}
 
 	// Parse YAML
-	cmds, err := commandsgen.ParseCommands()
+	cmds, err := commandsgen.ParseCommands(yamlBytes)
 	if err != nil {
 		return fmt.Errorf("failed parsing YAML: %w", err)
 	}
 
 	// Generate code
-	b, err := commandsgen.GenerateCommandsCode("temporalcli", cmds)
+	b, err := commandsgen.GenerateCommandsCode(pkg, contextType, cmds)
 	if err != nil {
 		return fmt.Errorf("failed generating code: %w", err)
 	}
 
-	// Write
-	if err := os.WriteFile(filepath.Join(commandsDir, "commands.gen.go"), b, 0644); err != nil {
-		return fmt.Errorf("failed writing file: %w", err)
+	// Write output to stdout
+	if _, err = os.Stdout.Write(b); err != nil {
+		return fmt.Errorf("failed writing output: %w", err)
 	}
+
 	return nil
 }

internal/cmd/gen-docs/README.md

@@ -0,0 +1,4 @@
+This package provides a CLI command documentation generator for Temporal CLI applications.
+It is designed specifically for the needs of Temporal CLIs, and not general-purpose command-line tools.
+
+Backwards-compatibility is not guaranteed.

internal/cmd/gen-docs/main.go

@@ -1,11 +1,11 @@
 package main
 
 import (
+	"flag"
 	"fmt"
 	"log"
 	"os"
 	"path/filepath"
-	"runtime"
 
 	"github.com/temporalio/cli/internal/commandsgen"
 )
@@ -17,33 +17,46 @@ func main() {
 }
 
 func run() error {
-	// Get commands dir
-	_, file, _, _ := runtime.Caller(0)
-	genDocsDir := filepath.Dir(file)
-	docsDir := filepath.Join(genDocsDir, "../../../dist/docs/")
+	var (
+		outputDir string
+		inputFile string
+	)
 
-	err := os.MkdirAll(docsDir, os.ModePerm)
+	flag.StringVar(&inputFile, "input", "", "Input YAML file (required)")
+	flag.StringVar(&outputDir, "output", ".", "Output directory for docs")
+	flag.Parse()
+
+	// Read input from file
+	if inputFile == "" {
+		return fmt.Errorf("-input flag is required")
+	}
+	yamlBytes, err := os.ReadFile(inputFile)
 	if err != nil {
-		log.Fatalf("Error creating directory: %v", err)
+		return fmt.Errorf("failed reading input: %w", err)
+	}
+
+	// Create output directory
+	if err := os.MkdirAll(outputDir, 0755); err != nil {
+		return fmt.Errorf("failed creating output directory: %w", err)
 	}
 
 	// Parse YAML
-	cmds, err := commandsgen.ParseCommands()
+	cmds, err := commandsgen.ParseCommands(yamlBytes)
 	if err != nil {
 		return fmt.Errorf("failed parsing YAML: %w", err)
 	}
 
 	// Generate docs
-	b, err := commandsgen.GenerateDocsFiles(cmds)
+	docs, err := commandsgen.GenerateDocsFiles(cmds)
 	if err != nil {
-		return err
+		return fmt.Errorf("failed generating docs: %w", err)
 	}
 
-	// Write
-	for filename, content := range b {
-		filePath := filepath.Join(docsDir, filename+".mdx")
+	// Write files
+	for filename, content := range docs {
+		filePath := filepath.Join(outputDir, filename+".mdx")
 		if err := os.WriteFile(filePath, content, 0644); err != nil {
-			return fmt.Errorf("failed writing file: %w", err)
+			return fmt.Errorf("failed writing %s: %w", filePath, err)
 		}
 	}
 

internal/commandsgen/code.go

@@ -2,18 +2,27 @@ package commandsgen
 
 import (
 	"bytes"
+	"embed"
 	"fmt"
+	"go/ast"
 	"go/format"
+	"go/parser"
+	"go/token"
+	"io/fs"
 	"path"
 	"regexp"
 	"sort"
 	"strings"
 
-	"go.temporal.io/server/common/primitives/timestamp"
+	"github.com/temporalio/cli/internal/commandsgen/types"
 )
 
-func GenerateCommandsCode(pkg string, commands Commands) ([]byte, error) {
-	w := &codeWriter{allCommands: commands.CommandList, OptionSets: commands.OptionSets}
+//go:embed types/*.go
+var typesFS embed.FS
+
+func GenerateCommandsCode(pkg string, contextType string, commands Commands) ([]byte, error) {
+	w := &codeWriter{allCommands: commands.CommandList, OptionSets: commands.OptionSets, contextType: contextType}
+
 	// Put terminal check at top
 	w.writeLinef("var hasHighlighting = %v.IsTerminal(%v.Stdout.Fd())", w.importIsatty(), w.importPkg("os"))
 
@@ -24,13 +33,28 @@ func GenerateCommandsCode(pkg string, commands Commands) ([]byte, error) {
 		}
 	}
 
-	// Write all commands, then come back and write package and imports
+	// Write all commands
 	for _, cmd := range commands.CommandList {
 		if err := cmd.writeCode(w); err != nil {
 			return nil, fmt.Errorf("failed writing command %v: %w", cmd.FullName, err)
 		}
 	}
 
+	// Append embedded Go files from types/ (parse imports with go/ast, write code after imports)
+	err := fs.WalkDir(typesFS, "types", func(path string, d fs.DirEntry, err error) error {
+		if err != nil || d.IsDir() || strings.Contains(path, "_test.go") {
+			return err
+		}
+		src, err := typesFS.ReadFile(path)
+		if err != nil {
+			return err
+		}
+		return w.appendGoSource(string(src))
+	})
+	if err != nil {
+		return nil, err
+	}
+
 	// Write package and imports to final buf
 	var finalBuf bytes.Buffer
 	finalBuf.WriteString("// Code generated. DO NOT EDIT.\n\n")
@@ -59,6 +83,7 @@ type codeWriter struct {
 	buf         bytes.Buffer
 	allCommands []Command
 	OptionSets  []OptionSets
+	contextType string
 	// Key is short ref, value is full
 	imports map[string]string
 }
@@ -98,6 +123,36 @@ func (c *codeWriter) importPkg(pkg string) string {
 
 func (c *codeWriter) importCobra() string { return c.importPkg("github.com/spf13/cobra") }
 
+// appendGoSource parses a Go source file, registers its imports, and appends
+// everything after the import block to the output buffer.
+func (c *codeWriter) appendGoSource(src string) error {
+	fset := token.NewFileSet()
+	f, err := parser.ParseFile(fset, "", src, parser.ImportsOnly)
+	if err != nil {
+		return fmt.Errorf("failed to parse embedded source: %w", err)
+	}
+
+	// Register imports
+	for _, imp := range f.Imports {
+		// imp.Path.Value includes quotes, so trim them
+		c.importPkg(strings.Trim(imp.Path.Value, `"`))
+	}
+
+	// Find end of imports and append the rest
+	var lastImportEnd token.Pos
+	for _, decl := range f.Decls {
+		if genDecl, ok := decl.(*ast.GenDecl); ok && genDecl.Tok == token.IMPORT {
+			if genDecl.End() > lastImportEnd {
+				lastImportEnd = genDecl.End()
+			}
+		}
+	}
+
+	// Write everything after imports
+	c.buf.WriteString(src[fset.Position(lastImportEnd).Offset:])
+	return nil
+}
+
 func (c *codeWriter) importPflag() string { return c.importPkg("github.com/spf13/pflag") }
 
 func (c *codeWriter) importIsatty() string { return c.importPkg("github.com/mattn/go-isatty") }
@@ -120,8 +175,8 @@ func (o *OptionSets) writeCode(w *codeWriter) error {
 	w.writeLinef("}\n")
 
 	// write flags
-	w.writeLinef("func (v *%v) buildFlags(cctx *CommandContext, f *%v.FlagSet) {",
-		o.setStructName(), w.importPflag())
+	w.writeLinef("func (v *%v) buildFlags(cctx %s, f *%v.FlagSet) {",
+		o.setStructName(), w.contextType, w.importPflag())
 	o.writeFlagBuilding("v", "f", w)
 	w.writeLinef("}\n")
 
@@ -164,10 +219,10 @@ func (c *Command) writeCode(w *codeWriter) error {
 
 	// Constructor builds the struct and sets the flags
 	if hasParent {
-		w.writeLinef("func New%v(cctx *CommandContext, parent *%v) *%v {",
-			c.structName(), parent.structName(), c.structName())
+		w.writeLinef("func New%v(cctx %s, parent *%v) *%v {",
+			c.structName(), w.contextType, parent.structName(), c.structName())
 	} else {
-		w.writeLinef("func New%v(cctx *CommandContext) *%v {", c.structName(), c.structName())
+		w.writeLinef("func New%v(cctx %s) *%v {", c.structName(), w.contextType, c.structName())
 	}
 	w.writeLinef("var s %v", c.structName())
 	if hasParent {
@@ -328,7 +383,7 @@ func (o *Option) writeFlagBuilding(selfVar, flagVar string, w *codeWriter) error
 	case "duration":
 		flagMeth, setDefault = "Var", "0"
 		if o.Default != "" {
-			dur, err := timestamp.ParseDuration(o.Default)
+			dur, err := types.ParseDuration(o.Default)
 			if err != nil {
 				return fmt.Errorf("invalid default: %w", err)
 			}