feat: implement a dump command

It allows to dump all packages in a set either to JSON or via Go
template to be transformed into any other text.

This was used to build automatically extension catalog
in siderolabs/extensions#793

Signed-off-by: Andrey Smirnov <[email protected]>

Author: smira

cmd/bldr/cmd/dump.go

@@ -0,0 +1,108 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package cmd
+
+import (
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+	"text/template"
+
+	"github.com/Masterminds/sprig/v3"
+	"github.com/spf13/cobra"
+	"github.com/stretchr/testify/assert/yaml"
+
+	"github.com/siderolabs/bldr/internal/pkg/solver"
+)
+
+var dumpCmdFlags struct {
+	templatePath string
+	buildArgs    []string
+}
+
+// dumpCmd represents the graph command.
+var dumpCmd = &cobra.Command{
+	Use:   "dump",
+	Short: "Dump information about all packages.",
+	Long: `This command outputs the set of all packages as JSON
+output, or runs a Go template against a set of all packages
+and prints the template output.
+`,
+	Args: cobra.NoArgs,
+	Run: func(_ *cobra.Command, _ []string) {
+		context := options.GetVariables().Copy()
+
+		for _, buildArg := range dumpCmdFlags.buildArgs {
+			name, value, _ := strings.Cut(buildArg, "=")
+
+			context["BUILD_ARG_"+name] = value
+		}
+
+		loader := solver.FilesystemPackageLoader{
+			Root:    pkgRoot,
+			Context: context,
+		}
+
+		packages, err := solver.NewPackages(&loader)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		var packageSet solver.PackageSet
+
+		if options.Target != "" {
+			var graph *solver.PackageGraph
+
+			graph, err = packages.Resolve(options.Target)
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			packageSet = graph.ToSet()
+		} else {
+			packageSet = packages.ToSet()
+		}
+
+		packageSet = packageSet.Sorted()
+
+		if dumpCmdFlags.templatePath == "" {
+			if err = packageSet.DumpJSON(os.Stdout); err != nil {
+				log.Fatal(err)
+			}
+
+			return
+		}
+
+		funcs := sprig.HermeticTxtFuncMap()
+		funcs["mustFromYAML"] = mustFromYAML
+
+		tmpl, err := template.New(filepath.Base(dumpCmdFlags.templatePath)).
+			Funcs(funcs).
+			ParseFiles(dumpCmdFlags.templatePath)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		if err := packageSet.Template(os.Stdout, tmpl); err != nil {
+			log.Fatal(err)
+		}
+	},
+}
+
+func mustFromYAML(v string) (any, error) {
+	var output any
+
+	err := yaml.Unmarshal([]byte(v), &output)
+
+	return output, err
+}
+
+func init() {
+	dumpCmd.Flags().StringVarP(&options.Target, "target", "t", "", "Target image to dump, if not set - dump all stages")
+	dumpCmd.Flags().StringSliceVar(&dumpCmdFlags.buildArgs, "build-arg", nil, "Build arguments to pass similar to docker buildx")
+	dumpCmd.Flags().StringVarP(&dumpCmdFlags.templatePath, "template", "", "", "Path to Go template file to use for output formatting")
+	rootCmd.AddCommand(dumpCmd)
+}

internal/pkg/solver/filesystem_loader.go

@@ -23,15 +23,16 @@ import (
 // FilesystemPackageLoader loads packages by walking file system tree.
 type FilesystemPackageLoader struct {
 	*log.Logger
-	Context      types.Variables
-	pathContexts map[string]types.Variables
-	multiErr     *multierror.Error
-	pkgFile      *v1alpha2.Pkgfile
-	Root         string
-	absRootPath  string
-	pkgFilePaths []string
-	varFilePaths []string
-	pkgs         []*v1alpha2.Pkg
+	Context           types.Variables
+	pathContexts      map[string]types.Variables
+	multiErr          *multierror.Error
+	pkgFile           *v1alpha2.Pkgfile
+	Root              string
+	absRootPath       string
+	pkgFilePaths      []string
+	varFilePaths      []string
+	templateFilePaths []string
+	pkgs              []*v1alpha2.Pkg
 }
 
 func (fspl *FilesystemPackageLoader) walkFunc() filepath.WalkFunc {
@@ -50,11 +51,13 @@ func (fspl *FilesystemPackageLoader) walkFunc() filepath.WalkFunc {
 			return nil
 		}
 
-		switch info.Name() {
-		case constants.PkgYaml:
+		switch {
+		case info.Name() == constants.PkgYaml:
 			fspl.pkgFilePaths = append(fspl.pkgFilePaths, path)
-		case constants.VarsYaml:
+		case info.Name() == constants.VarsYaml:
 			fspl.varFilePaths = append(fspl.varFilePaths, path)
+		case strings.HasSuffix(info.Name(), constants.TemplateExt):
+			fspl.templateFilePaths = append(fspl.templateFilePaths, path)
 		}
 
 		return nil
@@ -115,6 +118,17 @@ func (fspl *FilesystemPackageLoader) Load() (*LoadResult, error) {
 			fspl.Printf("loaded pkg %q from %q", pkg.Name, path)
 			fspl.pkgs = append(fspl.pkgs, pkg)
 		}
+
+		for _, path := range fspl.templateFilePaths {
+			var pkg *v1alpha2.Pkg
+
+			if pkg, err = fspl.attachTemplate(path); err != nil {
+				fspl.Printf("error attaching template %q: %s", path, err)
+				fspl.multiErr = multierror.Append(fspl.multiErr, fmt.Errorf("error attaching template %q: %w", path, err))
+			} else {
+				fspl.Printf("attached template %q to %q", path, pkg.Name)
+			}
+		}
 	}
 
 	return &LoadResult{
@@ -197,6 +211,38 @@ func (fspl *FilesystemPackageLoader) loadPkg(path string) (*v1alpha2.Pkg, error)
 	return v1alpha2.NewPkg(filepath.Dir(basePath), path, contents, context)
 }
 
+func (fspl *FilesystemPackageLoader) attachTemplate(path string) (*v1alpha2.Pkg, error) {
+	// find the closest pkgs in relative path
+	var (
+		closestPkg  *v1alpha2.Pkg
+		shortestRel string
+	)
+
+	for _, pkg := range fspl.pkgs {
+		rel, err := filepath.Rel(pkg.BaseDir, filepath.Dir(path))
+		if err != nil || strings.HasPrefix(rel, "..") {
+			continue
+		}
+
+		if shortestRel == "" || len(rel) < len(shortestRel) {
+			closestPkg = pkg
+			shortestRel = rel
+		}
+	}
+
+	if closestPkg == nil {
+		return nil, fmt.Errorf("no suitable package found for template %q", path)
+	}
+
+	content, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+
+	// attach the template to the closest package
+	return closestPkg, closestPkg.AttachTemplatedFile(filepath.Join(shortestRel, filepath.Base(path)), content)
+}
+
 func (fspl *FilesystemPackageLoader) loadPkgfile() error {
 	f, err := os.Open(filepath.Join(fspl.Root, constants.Pkgfile))
 	if err != nil {

internal/pkg/solver/set.go

@@ -5,7 +5,11 @@
 package solver
 
 import (
+	"cmp"
+	"encoding/json"
 	"io"
+	"slices"
+	"text/template"
 
 	"github.com/emicklei/dot"
 )
@@ -23,3 +27,23 @@ func (set PackageSet) DumpDot(w io.Writer) {
 
 	g.Write(w)
 }
+
+// Sorted returns a new set which is sorted by name package set.
+func (set PackageSet) Sorted() PackageSet {
+	return slices.SortedFunc(slices.Values(set), func(a, b *PackageNode) int {
+		return cmp.Compare(a.Name, b.Name)
+	})
+}
+
+// DumpJSON dumps the package set as JSON.
+func (set PackageSet) DumpJSON(w io.Writer) error {
+	encoder := json.NewEncoder(w)
+	encoder.SetIndent("", "  ")
+
+	return encoder.Encode(set)
+}
+
+// Template dumps the package set as a Go template.
+func (set PackageSet) Template(w io.Writer, tmpl *template.Template) error {
+	return tmpl.Execute(w, set)
+}

internal/pkg/types/v1alpha2/pkg.go

@@ -19,18 +19,17 @@ import (
 
 // Pkg represents build instructions for a single package.
 type Pkg struct {
-	templatedFiles []TemplatedFile
-
-	Context      types.Variables `yaml:"-"`
-	Name         string          `yaml:"name,omitempty"`
-	Shell        Shell           `yaml:"shell,omitempty"`
-	BaseDir      string          `yaml:"-"`
-	FileName     string          `yaml:"-"`
-	Install      Install         `yaml:"install,omitempty"`
-	Dependencies Dependencies    `yaml:"dependencies,omitempty"`
-	Steps        Steps           `yaml:"steps,omitempty"`
-	Finalize     []Finalize      `yaml:"finalize,omitempty"`
-	Variant      Variant         `yaml:"variant,omitempty"`
+	TemplatedFiles []TemplatedFile `yaml:"-"`
+	Context        types.Variables `yaml:"-"`
+	Name           string          `yaml:"name,omitempty"`
+	Shell          Shell           `yaml:"shell,omitempty"`
+	BaseDir        string          `yaml:"-"`
+	FileName       string          `yaml:"-"`
+	Install        Install         `yaml:"install,omitempty"`
+	Dependencies   Dependencies    `yaml:"dependencies,omitempty"`
+	Steps          Steps           `yaml:"steps,omitempty"`
+	Finalize       []Finalize      `yaml:"finalize,omitempty"`
+	Variant        Variant         `yaml:"variant,omitempty"`
 }
 
 // NewPkg loads Pkg structure from file.

internal/pkg/types/v1alpha2/templates.go

@@ -35,7 +35,7 @@ func (p *Pkg) AttachTemplatedFile(path string, content []byte) error {
 		return fmt.Errorf("failed to template file %s: %w", path, err)
 	}
 
-	p.templatedFiles = append(p.templatedFiles, TemplatedFile{
+	p.TemplatedFiles = append(p.TemplatedFiles, TemplatedFile{
 		Path:    strings.TrimSuffix(path, constants.TemplateExt),
 		Content: buf.Bytes(),
 	})
@@ -45,5 +45,5 @@ func (p *Pkg) AttachTemplatedFile(path string, content []byte) error {
 
 // GetTemplatedFiles return a list of templated files in the package.
 func (p *Pkg) GetTemplatedFiles() []TemplatedFile {
-	return p.templatedFiles
+	return p.TemplatedFiles
 }