Assemble templates using the new basedOn setting

It allows a template to be constructed by merging values from
one or more base templates together. This merge process will
maintain all comments from both the template and the bases.

The template is assembled before an instance is created, and
only the combined template is stored as lima.yaml in the instance
directory.

There merging semantics are otherwise similar to how lima.yaml
is combined with override.yaml, defaults.yaml, and the builtin
default values.

Signed-off-by: Jan Dubois <[email protected]>

Author: jandubois

cmd/limactl/start.go

@@ -205,7 +205,9 @@ func loadOrCreateInstance(cmd *cobra.Command, args []string, createOnly bool) (*
 			return nil, err
 		}
 	}
-
+	if err := tmpl.Embed(cmd.Context()); err != nil {
+		return nil, err
+	}
 	yqExprs, err := editflags.YQExpressions(flags, true)
 	if err != nil {
 		return nil, err

cmd/limactl/template.go

@@ -4,6 +4,7 @@
 package main
 
 import (
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -52,24 +53,74 @@ var templateCopyExample = `  Template locators are local files, file://, https:/
 
 func newTemplateCopyCommand() *cobra.Command {
 	templateCopyCommand := &cobra.Command{
-		Use:     "copy TEMPLATE DEST",
+		Use:     "copy [OPTIONS] TEMPLATE DEST",
 		Short:   "Copy template",
 		Long:    "Copy a template via locator to a local file",
 		Example: templateCopyExample,
 		Args:    WrapArgsError(cobra.ExactArgs(2)),
 		RunE:    templateCopyAction,
 	}
+	templateCopyCommand.Flags().Bool("embed", false, "embed dependencies into template")
+	templateCopyCommand.Flags().Bool("fill", false, "fill defaults")
+	templateCopyCommand.Flags().Bool("verbatim", false, "don't make locators absolute")
 	return templateCopyCommand
 }
 
 func templateCopyAction(cmd *cobra.Command, args []string) error {
+	embed, err := cmd.Flags().GetBool("embed")
+	if err != nil {
+		return err
+	}
+	fill, err := cmd.Flags().GetBool("fill")
+	if err != nil {
+		return err
+	}
+	verbatim, err := cmd.Flags().GetBool("verbatim")
+	if err != nil {
+		return err
+	}
+	if embed && verbatim {
+		return errors.New("--embed and --verbatim cannot be used together")
+	}
+	if fill && verbatim {
+		return errors.New("--fill and --verbatim cannot be used together")
+	}
+
 	tmpl, err := limatmpl.Read(cmd.Context(), "", args[0])
 	if err != nil {
 		return err
 	}
 	if len(tmpl.Bytes) == 0 {
 		return fmt.Errorf("don't know how to interpret %q as a template locator", args[0])
 	}
+	if !verbatim {
+		if embed {
+			if err := tmpl.Embed(cmd.Context()); err != nil {
+				return err
+			}
+		} else {
+			if err := tmpl.UseAbsLocators(); err != nil {
+				return err
+			}
+		}
+	}
+	if fill {
+		limaDir, err := dirnames.LimaDir()
+		if err != nil {
+			return err
+		}
+		// Load() will merge the template with override.yaml and default.yaml via FillDefaults().
+		// FillDefaults() needs the potential instance directory to validate host templates using {{.Dir}}.
+		filePath := filepath.Join(limaDir, tmpl.Name+".yaml")
+		tmpl.Config, err = limayaml.Load(tmpl.Bytes, filePath)
+		if err != nil {
+			return err
+		}
+		tmpl.Bytes, err = limayaml.Marshal(tmpl.Config, false)
+		if err != nil {
+			return err
+		}
+	}
 	writer := cmd.OutOrStdout()
 	target := args[1]
 	if target != "-" {
@@ -118,8 +169,8 @@ func templateValidateAction(cmd *cobra.Command, args []string) error {
 		}
 		// Load() will merge the template with override.yaml and default.yaml via FillDefaults().
 		// FillDefaults() needs the potential instance directory to validate host templates using {{.Dir}}.
-		instDir := filepath.Join(limaDir, tmpl.Name)
-		y, err := limayaml.Load(tmpl.Bytes, instDir)
+		filePath := filepath.Join(limaDir, tmpl.Name+".yaml")
+		y, err := limayaml.Load(tmpl.Bytes, filePath)
 		if err != nil {
 			return err
 		}

pkg/limatmpl/abs.go

@@ -0,0 +1,89 @@
+package limatmpl
+
+import (
+	"errors"
+	"fmt"
+	"net/url"
+	"path/filepath"
+	"strings"
+)
+
+// UseAbsLocators will replace all relative template locators with absolute ones, so this template
+// can be stored anywhere and still reference the same base templates and files.
+func (tmpl *Template) UseAbsLocators() error {
+	err := tmpl.useAbsLocators()
+	return tmpl.ClearOnError(err)
+}
+
+func (tmpl *Template) useAbsLocators() error {
+	if err := tmpl.Unmarshal(); err != nil {
+		return err
+	}
+	basePath, err := basePath(tmpl.Locator)
+	if err != nil {
+		return err
+	}
+	for i, baseLocator := range tmpl.Config.BasedOn {
+		locator, err := absPath(baseLocator, basePath)
+		if err != nil {
+			return err
+		}
+		if i == 0 {
+			// basedOn can either be a single string, or a list of strings
+			tmpl.expr.WriteString(fmt.Sprintf("| ($a.basedOn | select(type == \"!!str\")) |= %q\n", locator))
+			tmpl.expr.WriteString(fmt.Sprintf("| ($a.basedOn | select(type == \"!!seq\") | .[0]) |= %q\n", locator))
+		} else {
+			tmpl.expr.WriteString(fmt.Sprintf("| $a.basedOn[%d] = %q\n", i, locator))
+		}
+	}
+	for i, p := range tmpl.Config.Probes {
+		if p.File != nil {
+			locator, err := absPath(*p.File, basePath)
+			if err != nil {
+				return err
+			}
+			tmpl.expr.WriteString(fmt.Sprintf("| $a.probes[%d].file = %q\n", i, locator))
+		}
+	}
+	for i, p := range tmpl.Config.Provision {
+		if p.File != nil {
+			locator, err := absPath(*p.File, basePath)
+			if err != nil {
+				return err
+			}
+			tmpl.expr.WriteString(fmt.Sprintf("| $a.provision[%d].file = %q\n", i, locator))
+		}
+	}
+	return tmpl.evalExpr()
+}
+
+// basePath returns the locator without the filename part.
+func basePath(locator string) (string, error) {
+	u, err := url.Parse(locator)
+	if err != nil || u.Scheme == "" {
+		return filepath.Abs(filepath.Dir(locator))
+	}
+	// filepath.Dir("") returns ".", which must be removed for url.JoinPath() to do the right thing later
+	return u.Scheme + "://" + strings.TrimSuffix(filepath.Dir(filepath.Join(u.Host, u.Path)), "."), nil
+}
+
+// absPath either returns the locator directly, or combines it with the basePath if the locator is a relative path.
+func absPath(locator, basePath string) (string, error) {
+	u, err := url.Parse(locator)
+	if (err == nil && u.Scheme != "") || filepath.IsAbs(locator) {
+		return locator, nil
+	}
+	switch {
+	case basePath == "":
+		return "", errors.New("basePath is empty")
+	case basePath == "-":
+		return "", errors.New("can't use relative paths when reading template from STDIN")
+	case strings.Contains(locator, "../"):
+		return "", fmt.Errorf("relative locator path %q must not contain '../' segments", locator)
+	}
+	u, err = url.Parse(basePath)
+	if err != nil {
+		return "", err
+	}
+	return u.JoinPath(locator).String(), nil
+}

pkg/limatmpl/abs_test.go

@@ -0,0 +1,199 @@
+package limatmpl
+
+import (
+	"strings"
+	"testing"
+
+	"gotest.tools/v3/assert"
+)
+
+type useAbsLocatorsTestCase struct {
+	description string
+	locator     string
+	template    string
+	expected    string
+}
+
+var useAbsLocatorsTestCases = []useAbsLocatorsTestCase{
+	{
+		"Template without basedOn or script file",
+		"template://foo",
+		`foo: bar`,
+		`foo: bar`,
+	},
+	{
+		"Single string base template",
+		"template://foo",
+		`basedOn: bar.yaml`,
+		`basedOn: template://bar.yaml`,
+	},
+	{
+		"Flow style array of one base template",
+		"template://foo",
+		`basedOn: [bar.yaml]`,
+		`basedOn: ['template://bar.yaml']`,
+	},
+	{
+		"Block style array of one base template",
+		"template://foo",
+		`
+basedOn:
+- bar.yaml
+`,
+		`
+basedOn:
+- template://bar.yaml`,
+	},
+	{
+		"Block style of four base templates",
+		"template://foo",
+		`
+basedOn:
+- bar.yaml
+- template://my
+- https://example.com/my.yaml
+- baz.yaml
+`,
+		`
+basedOn:
+- template://bar.yaml
+- template://my
+- https://example.com/my.yaml
+- template://baz.yaml
+`,
+	},
+	{
+		"Provisioning and probe scripts",
+		"template://experimental/foo",
+		`
+provision:
+- mode: user
+  file: script.sh
+probes:
+- file: probe.sh
+`,
+		`
+provision:
+- mode: user
+  file: template://experimental/script.sh
+probes:
+- file: template://experimental/probe.sh
+`,
+	},
+}
+
+func TestUseAbsLocators(t *testing.T) {
+	for _, tc := range useAbsLocatorsTestCases {
+		t.Run(tc.description, func(t *testing.T) { RunUseAbsLocatorTest(t, tc) })
+	}
+}
+
+func RunUseAbsLocatorTest(t *testing.T, tc useAbsLocatorsTestCase) {
+	tmpl := &Template{
+		Bytes:   []byte(strings.TrimSpace(tc.template)),
+		Locator: tc.locator,
+	}
+	err := tmpl.UseAbsLocators()
+	assert.NilError(t, err, tc.description)
+
+	actual := strings.TrimSpace(string(tmpl.Bytes))
+	expected := strings.TrimSpace(tc.expected)
+	assert.Equal(t, actual, expected, tc.description)
+}
+
+func TestBasePath(t *testing.T) {
+	actual, err := basePath("/foo")
+	assert.NilError(t, err)
+	assert.Equal(t, actual, "/")
+
+	actual, err = basePath("/foo/bar")
+	assert.NilError(t, err)
+	assert.Equal(t, actual, "/foo")
+
+	actual, err = basePath("template://foo")
+	assert.NilError(t, err)
+	assert.Equal(t, actual, "template://")
+
+	actual, err = basePath("template://foo/bar")
+	assert.NilError(t, err)
+	assert.Equal(t, actual, "template://foo")
+
+	actual, err = basePath("http://host/foo")
+	assert.NilError(t, err)
+	assert.Equal(t, actual, "http://host")
+
+	actual, err = basePath("http://host/foo/bar")
+	assert.NilError(t, err)
+	assert.Equal(t, actual, "http://host/foo")
+
+	actual, err = basePath("file:///foo")
+	assert.NilError(t, err)
+	assert.Equal(t, actual, "file:///")
+
+	actual, err = basePath("file:///foo/bar")
+	assert.NilError(t, err)
+	assert.Equal(t, actual, "file:///foo")
+}
+
+func TestAbsPath(t *testing.T) {
+	// If the locator is already an absolute path, it is returned unchanged (no extension appended either)
+	actual, err := absPath("/foo", "/root")
+	assert.NilError(t, err)
+	assert.Equal(t, actual, "/foo")
+
+	actual, err = absPath("template://foo", "/root")
+	assert.NilError(t, err)
+	assert.Equal(t, actual, "template://foo")
+
+	actual, err = absPath("http://host/foo", "/root")
+	assert.NilError(t, err)
+	assert.Equal(t, actual, "http://host/foo")
+
+	actual, err = absPath("file:///foo", "/root")
+	assert.NilError(t, err)
+	assert.Equal(t, actual, "file:///foo")
+
+	// Can't have relative path when reading from STDIN
+	_, err = absPath("foo", "-")
+	assert.ErrorContains(t, err, "STDIN")
+
+	// Relative paths must be underneath the basePath
+	_, err = absPath("../foo", "/root")
+	assert.ErrorContains(t, err, "'../'")
+
+	// basePath must not be empty
+	_, err = absPath("foo", "")
+	assert.ErrorContains(t, err, "empty")
+
+	_, err = absPath("./foo", "")
+	assert.ErrorContains(t, err, "empty")
+
+	// Check relative paths with all the supported schemes
+	actual, err = absPath("./foo", "/root")
+	assert.NilError(t, err)
+	assert.Equal(t, actual, "/root/foo")
+
+	actual, err = absPath("foo", "template://")
+	assert.NilError(t, err)
+	assert.Equal(t, actual, "template://foo")
+
+	actual, err = absPath("bar", "template://foo")
+	assert.NilError(t, err)
+	assert.Equal(t, actual, "template://foo/bar")
+
+	actual, err = absPath("foo", "http://host")
+	assert.NilError(t, err)
+	assert.Equal(t, actual, "http://host/foo")
+
+	actual, err = absPath("bar", "http://host/foo")
+	assert.NilError(t, err)
+	assert.Equal(t, actual, "http://host/foo/bar")
+
+	actual, err = absPath("foo", "file:///")
+	assert.NilError(t, err)
+	assert.Equal(t, actual, "file:///foo")
+
+	actual, err = absPath("bar", "file:///foo")
+	assert.NilError(t, err)
+	assert.Equal(t, actual, "file:///foo/bar")
+}