[tyson] Add tyson support experiment (#1656)

## Summary

Minimal viable implementation of tyson support. This allows user to
opt-in via feature flag and a `devbox.tson` config. It has no effect for
non-opt

Proof of concept of how this can be used:
https://github.com/jetpack-io/frontend/pull/504 (warning private repo)

To simplify this implementation, it is not possible to use any devbox
command that writes to `devbox.json`. It will fail with an error. We
will also show an error if both configs are present.

Tyson allows more programatic logic in our configs. Specifically:

* Allows importing other configs which adds composability.
* Allows importing of other package manager files (e.g. `package.json`,
`composer.json`, etc which allows you to mimic and enhance their
functionality)
* Allows you to customize hooks, packages, and scripts using host
environment variables. ([Proof of
concept](jetify-com/axiom#3532))

## How was it tested?

* Tested in the proof of concept link above.
* CICD

Author: mikeland73

internal/boxcli/featureflag/feature.go

@@ -8,6 +8,7 @@ import (
 	"strconv"
 	"testing"
 
+	"go.jetpack.io/devbox/internal/build"
 	"go.jetpack.io/devbox/internal/debug"
 	"go.jetpack.io/devbox/internal/envir"
 )
@@ -61,6 +62,13 @@ func (f *feature) Enabled() bool {
 	return f.enabled
 }
 
+func (f *feature) EnableOnDev() *feature {
+	if build.IsDev {
+		f.enabled = true
+	}
+	return f
+}
+
 func (f *feature) EnableForTest(t *testing.T) {
 	t.Setenv(envir.DevboxFeaturePrefix+f.name, "1")
 }

internal/boxcli/featureflag/tyson.go

@@ -0,0 +1,3 @@
+package featureflag
+
+var TySON = disable("TYSON").EnableOnDev()

internal/boxcli/multi/multi.go

@@ -6,6 +6,7 @@ import (
 
 	"go.jetpack.io/devbox"
 	"go.jetpack.io/devbox/internal/debug"
+	"go.jetpack.io/devbox/internal/devconfig"
 	"go.jetpack.io/devbox/internal/impl/devopt"
 )
 
@@ -20,7 +21,7 @@ func Open(opts *devopt.Opts) ([]devbox.Devbox, error) {
 				return err
 			}
 
-			if !dirEntry.IsDir() && filepath.Base(path) == "devbox.json" {
+			if !dirEntry.IsDir() && devconfig.IsConfigName(filepath.Base(path)) {
 				optsCopy := *opts
 				optsCopy.Dir = path
 				box, err := devbox.Open(&optsCopy)

internal/devconfig/config.go

@@ -16,12 +16,21 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/tailscale/hujson"
+	"go.jetpack.io/devbox/internal/boxcli/featureflag"
 	"go.jetpack.io/devbox/internal/boxcli/usererr"
 	"go.jetpack.io/devbox/internal/cuecfg"
 	"go.jetpack.io/devbox/internal/impl/shellcmd"
 )
 
-const DefaultName = "devbox.json"
+const (
+	defaultName      = "devbox.json"
+	defaultTySONName = "devbox.tson"
+)
+
+const (
+	jsonFormat = iota
+	tsonFormat
+)
 
 // Config defines a devbox environment as JSON.
 type Config struct {
@@ -48,7 +57,8 @@ type Config struct {
 	// This is a similar format to nix inputs
 	Include []string `json:"include,omitempty"`
 
-	ast *configAST
+	ast    *configAST
+	format int
 }
 
 type shellConfig struct {
@@ -128,7 +138,10 @@ func (c *Config) InitHook() *shellcmd.Commands {
 
 // SaveTo writes the config to a file.
 func (c *Config) SaveTo(path string) error {
-	return os.WriteFile(filepath.Join(path, DefaultName), c.Bytes(), 0o644)
+	if c.format != jsonFormat {
+		return errors.New("cannot save config to non-json format")
+	}
+	return os.WriteFile(filepath.Join(path, defaultName), c.Bytes(), 0o644)
 }
 
 // Load reads a devbox config file, and validates it.
@@ -224,3 +237,15 @@ func ValidateNixpkg(cfg *Config) error {
 	}
 	return nil
 }
+
+func IsConfigName(name string) bool {
+	return slices.Contains(ValidConfigNames(), name)
+}
+
+func ValidConfigNames() []string {
+	names := []string{defaultName}
+	if featureflag.TySON.Enabled() {
+		names = append(names, defaultTySONName)
+	}
+	return names
+}

internal/devconfig/config_test.go

@@ -602,7 +602,7 @@ func TestDefault(t *testing.T) {
 	if err != nil {
 		t.Fatal("got save error:", err)
 	}
-	out, err := Load(filepath.Join(path, "devbox.json"))
+	out, err := Load(filepath.Join(path, defaultName))
 	if err != nil {
 		t.Fatal("got load error:", err)
 	}

internal/devconfig/init.go

@@ -4,20 +4,25 @@
 package devconfig
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"io"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"strings"
 
 	"github.com/fatih/color"
 
+	"go.jetpack.io/devbox/internal/boxcli/featureflag"
+	"go.jetpack.io/devbox/internal/devpkg/pkgtype"
+	"go.jetpack.io/devbox/internal/fileutil"
 	"go.jetpack.io/devbox/internal/initrec"
 )
 
 func Init(dir string, writer io.Writer) (created bool, err error) {
-	created, err = initConfigFile(filepath.Join(dir, DefaultName))
+	created, err = initConfigFile(filepath.Join(dir, defaultName))
 	if err != nil || !created {
 		return created, err
 	}
@@ -62,3 +67,43 @@ func initConfigFile(path string) (created bool, err error) {
 	}
 	return true, nil
 }
+
+func Open(projectDir string) (*Config, error) {
+	cfgPath := filepath.Join(projectDir, defaultName)
+
+	if !featureflag.TySON.Enabled() {
+		return Load(cfgPath)
+	}
+
+	tysonCfgPath := filepath.Join(projectDir, defaultTySONName)
+
+	// If tyson config exists use it. Otherwise fallback to json config.
+	// In the future we may error out if both configs exist, but for now support
+	// both while we experiment with tyson support.
+	if fileutil.Exists(tysonCfgPath) {
+		paths, err := pkgtype.RunXClient().Install(context.TODO(), "jetpack-io/tyson")
+		if err != nil {
+			return nil, err
+		}
+		binPath := filepath.Join(paths[0], "tyson")
+		tmpFile, err := os.CreateTemp("", "*.json")
+		if err != nil {
+			return nil, err
+		}
+		cmd := exec.Command(binPath, "eval", tysonCfgPath)
+		cmd.Stderr = os.Stderr
+		cmd.Stdout = tmpFile
+		if err = cmd.Run(); err != nil {
+			return nil, err
+		}
+		cfgPath = tmpFile.Name()
+		config, err := Load(cfgPath)
+		if err != nil {
+			return nil, err
+		}
+		config.format = tsonFormat
+		return config, nil
+	}
+
+	return Load(cfgPath)
+}

internal/impl/devbox.go

@@ -78,9 +78,8 @@ func Open(opts *devopt.Opts) (*Devbox, error) {
 	if err != nil {
 		return nil, err
 	}
-	cfgPath := filepath.Join(projectDir, devconfig.DefaultName)
 
-	cfg, err := devconfig.Load(cfgPath)
+	cfg, err := devconfig.Open(projectDir)
 	if err != nil {
 		return nil, errors.WithStack(err)
 	}

internal/impl/dir.go

@@ -43,7 +43,7 @@ func findProjectDirAtPath(absPath string) (string, error) {
 
 	switch mode := fi.Mode(); {
 	case mode.IsDir():
-		if !fileutil.Exists(filepath.Join(absPath, devconfig.DefaultName)) {
+		if !configExistsIn(absPath) {
 			return "", missingConfigError(absPath, false /*didCheckParents*/)
 		}
 		return absPath, nil
@@ -63,13 +63,13 @@ func findProjectDirFromParentDirSearch(
 	cur := absPath
 	// Search parent directories for a devbox.json
 	for cur != root {
-		debug.Log("finding %s in dir: %s\n", devconfig.DefaultName, cur)
-		if fileutil.Exists(filepath.Join(cur, devconfig.DefaultName)) {
+		debug.Log("finding devbox config in dir: %s\n", cur)
+		if configExistsIn(cur) {
 			return cur, nil
 		}
 		cur = filepath.Dir(cur)
 	}
-	if fileutil.Exists(filepath.Join(cur, devconfig.DefaultName)) {
+	if configExistsIn(cur) {
 		return cur, nil
 	}
 	return "", missingConfigError(absPath, true /*didCheckParents*/)
@@ -107,3 +107,12 @@ func missingConfigError(path string, didCheckParents bool) error {
 		parentDirCheckAddendum,
 	)
 }
+
+func configExistsIn(path string) bool {
+	for _, name := range devconfig.ValidConfigNames() {
+		if fileutil.Exists(filepath.Join(path, name)) {
+			return true
+		}
+	}
+	return false
+}

internal/impl/dir_test.go

@@ -53,7 +53,7 @@ func TestFindProjectDirFromParentDirSearch(t *testing.T) {
 			err = os.MkdirAll(filepath.Join(root, testCase.allDirs), 0o777)
 			assert.NoError(err)
 
-			absProjectPath, err := filepath.Abs(filepath.Join(root, testCase.projectDir, devconfig.DefaultName))
+			absProjectPath, err := filepath.Abs(filepath.Join(root, testCase.projectDir, devconfig.ValidConfigNames()[0]))
 			assert.NoError(err)
 			err = os.WriteFile(absProjectPath, []byte("{}"), 0o666)
 			assert.NoError(err)
@@ -97,14 +97,14 @@ func TestFindParentDirAtPath(t *testing.T) {
 			name:        "flag_path_is_file_has_config",
 			allDirs:     "a/b/c",
 			projectDir:  "a/b",
-			flagPath:    "a/b/" + devconfig.DefaultName,
+			flagPath:    "a/b/" + devconfig.ValidConfigNames()[0],
 			expectError: false,
 		},
 		{
 			name:        "flag_path_is_file_missing_config",
 			allDirs:     "a/b/c",
 			projectDir:  "", // missing config
-			flagPath:    "a/b/" + devconfig.DefaultName,
+			flagPath:    "a/b/" + devconfig.ValidConfigNames()[0],
 			expectError: true,
 		},
 	}
@@ -121,7 +121,7 @@ func TestFindParentDirAtPath(t *testing.T) {
 
 			var absProjectPath string
 			if testCase.projectDir != "" {
-				absProjectPath, err = filepath.Abs(filepath.Join(root, testCase.projectDir, devconfig.DefaultName))
+				absProjectPath, err = filepath.Abs(filepath.Join(root, testCase.projectDir, devconfig.ValidConfigNames()[0]))
 				assert.NoError(err)
 				err = os.WriteFile(absProjectPath, []byte("{}"), 0o666)
 				assert.NoError(err)

internal/pullbox/files.go

@@ -62,7 +62,7 @@ func profileIsNotEmpty(path string) (bool, error) {
 		return false, errors.WithStack(err)
 	}
 	for _, entry := range entries {
-		if entry.Name() != devconfig.DefaultName ||
+		if !devconfig.IsConfigName(entry.Name()) ||
 			isModifiedConfig(filepath.Join(path, entry.Name())) {
 			return true, nil
 		}
@@ -71,7 +71,7 @@ func profileIsNotEmpty(path string) (bool, error) {
 }
 
 func isModifiedConfig(path string) bool {
-	if filepath.Base(path) == devconfig.DefaultName {
+	if devconfig.IsConfigName(filepath.Base(path)) {
 		cfg, err := devconfig.Load(path)
 		if err != nil {
 			return false