[pkgcfg] Create new pkgckg that can load local config (#253)

## Summary

Creates a new package that allows loading of local configs. This will
allow for rapid testing and iteration of package configuration before we
read them remotely.

## How was it tested?

Created dummy go config and ran:

```
DEVBOX_FEATURE_PKG_CONFIG=1 DEVBOX_LOCAL_PKG_CONFIG=~/dev/devbox ./dist/devbox shell
echo $FOO
./.devbox/test-file.sh
```

Author: mikeland73

boxcli/featureflag/feature.go

@@ -3,6 +3,8 @@ package featureflag
 import (
 	"os"
 	"strconv"
+
+	"go.jetpack.io/devbox/debug"
 )
 
 type feature struct {
@@ -29,6 +31,7 @@ func (f *feature) Enabled() bool {
 		return false
 	}
 	if on, _ := strconv.ParseBool(os.Getenv("DEVBOX_FEATURE_" + f.name)); on {
+		debug.Log("Feature %q enabled via environment variable.", f.name)
 		return true
 	}
 	return f.enabled

devbox.go

@@ -16,10 +16,12 @@ import (
 	"github.com/fatih/color"
 	"github.com/pkg/errors"
 	"github.com/samber/lo"
+	"go.jetpack.io/devbox/boxcli/featureflag"
 	"go.jetpack.io/devbox/cuecfg"
 	"go.jetpack.io/devbox/debug"
 	"go.jetpack.io/devbox/docker"
 	"go.jetpack.io/devbox/nix"
+	"go.jetpack.io/devbox/pkgcfg"
 	"go.jetpack.io/devbox/planner"
 	"go.jetpack.io/devbox/planner/plansdk"
 	"golang.org/x/exp/slices"
@@ -200,11 +202,22 @@ func (d *Devbox) Shell() error {
 	}
 
 	nixShellFilePath := filepath.Join(d.srcDir, ".devbox/gen/shell.nix")
-	shell, err := nix.DetectShell(
+
+	opts := []nix.ShellOption{
 		nix.WithPlanInitHook(strings.Join(plan.ShellInitHook, "\n")),
 		nix.WithProfile(profileDir),
 		nix.WithHistoryFile(filepath.Join(d.srcDir, shellHistoryFile)),
-	)
+	}
+
+	if featureflag.Get(featureflag.PKGConfig).Enabled() {
+		env, err := pkgcfg.Env(plan.DevPackages)
+		if err != nil {
+			return err
+		}
+		opts = append(opts, nix.WithEnvVariables(env))
+	}
+
+	shell, err := nix.DetectShell(opts...)
 	if err != nil {
 		// Fall back to using a plain Nix shell.
 		shell = &nix.Shell{}

generate.go

@@ -13,7 +13,9 @@ import (
 	"text/template"
 
 	"github.com/pkg/errors"
+	"go.jetpack.io/devbox/boxcli/featureflag"
 	"go.jetpack.io/devbox/debug"
+	"go.jetpack.io/devbox/pkgcfg"
 	"go.jetpack.io/devbox/planner/plansdk"
 )
 
@@ -48,6 +50,14 @@ func generateForShell(rootPath string, plan *plansdk.ShellPlan) error {
 		}
 	}
 
+	if featureflag.Get(featureflag.PKGConfig).Enabled() {
+		for _, pkg := range plan.DevPackages {
+			if err := pkgcfg.CreateFiles(pkg, rootPath); err != nil {
+				return err
+			}
+		}
+	}
+
 	return nil
 }
 

nix/shell.go

@@ -37,6 +37,7 @@ const (
 type Shell struct {
 	name            name
 	binPath         string
+	env             []string
 	userShellrcPath string
 	planInitHook    string
 
@@ -114,6 +115,14 @@ func WithHistoryFile(historyFile string) ShellOption {
 	}
 }
 
+func WithEnvVariables(envVariables map[string]string) ShellOption {
+	return func(s *Shell) {
+		for k, v := range envVariables {
+			s.env = append(s.env, fmt.Sprintf("%s=%s", k, v))
+		}
+	}
+}
+
 // rcfilePath returns the absolute path for an rcfile, which is usually in the
 // user's home directory. It doesn't guarantee that the file exists.
 func rcfilePath(basename string) string {
@@ -134,21 +143,23 @@ func (s *Shell) Run(nixShellFilePath string) error {
 	// directories that are incompatible.
 	parentPath := cleanEnvPath(os.Getenv("PATH"), nixProfileDirs)
 
-	env := append(
-		os.Environ(),
+	env := append(s.env, os.Environ()...)
+	env = append(
+		env,
 		"PARENT_PATH="+parentPath,
 		"NIX_PROFILES="+strings.Join(nixProfileDirs, " "),
 
 		// Prevent the user's shellrc from re-sourcing nix-daemon.sh
 		// inside the devbox shell.
 		"__ETC_PROFILE_NIX_SOURCED=1",
 	)
+	debug.Log("Running nix-shell with environment: %v", env)
 
 	// Launch a fallback shell if we couldn't find the path to the user's
 	// default shell.
 	if s.binPath == "" {
 		cmd := exec.Command("nix-shell", "--pure")
-		cmd.Args = append(cmd.Args, toKeepArgs(env)...)
+		cmd.Args = append(cmd.Args, toKeepArgs(env, buildAllowList(s.env))...)
 		cmd.Args = append(cmd.Args, nixShellFilePath)
 		cmd.Env = env
 		cmd.Stdin = os.Stdin
@@ -160,7 +171,7 @@ func (s *Shell) Run(nixShellFilePath string) error {
 	}
 
 	cmd := exec.Command("nix-shell", "--command", s.execCommand(), "--pure")
-	cmd.Args = append(cmd.Args, toKeepArgs(env)...)
+	cmd.Args = append(cmd.Args, toKeepArgs(env, buildAllowList(s.env))...)
 	cmd.Args = append(cmd.Args, nixShellFilePath)
 	cmd.Env = env
 	cmd.Stdin = os.Stdin
@@ -374,16 +385,25 @@ var envToKeep = map[string]bool{
 	"SSL_CERT_FILE":             true, // The path to non-Nix SSL certificates (used by some Nix and non-Nix programs).
 }
 
+func buildAllowList(allowList []string) map[string]bool {
+	for _, kv := range allowList {
+		key, _, _ := strings.Cut(kv, "=")
+		envToKeep[key] = true
+	}
+	return envToKeep
+}
+
 // toKeepArgs takes a slice of environment variables in key=value format and
 // builds a slice of "--keep" arguments that tell nix-shell which ones to
 // keep.
 //
-// See envToKeep for the full set of kept environment variables.
-func toKeepArgs(env []string) []string {
-	args := make([]string, 0, len(envToKeep)*2)
+// See envToKeep for the full set of permanent kept environment variables.
+// We also --keep any variables set by package configuration.
+func toKeepArgs(env []string, allowList map[string]bool) []string {
+	args := make([]string, 0, len(allowList)*2)
 	for _, kv := range env {
 		key, _, _ := strings.Cut(kv, "=")
-		if envToKeep[key] {
+		if allowList[key] {
 			args = append(args, "--keep", key)
 		}
 	}

pkgcfg/pkgcfg.go

@@ -0,0 +1,80 @@
+package pkgcfg
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+
+	"github.com/pkg/errors"
+	"go.jetpack.io/devbox/debug"
+)
+
+const localPkgConfigPath = "DEVBOX_LOCAL_PKG_CONFIG"
+
+type config struct {
+	Name        string            `json:"name"`
+	Version     string            `json:"version"`
+	CreateFiles map[string]string `json:"create_files"`
+	Env         map[string]string `json:"env"`
+}
+
+func get(pkg string) (*config, error) {
+	if configPath := os.Getenv(localPkgConfigPath); configPath != "" {
+		debug.Log("Using local package config at %q", configPath)
+		return getLocalConfig(configPath, pkg)
+	}
+	return &config{}, nil
+}
+
+func getLocalConfig(configPath, pkg string) (*config, error) {
+	configPath = filepath.Join(configPath, pkg+".json")
+	if _, err := os.Stat(configPath); errors.Is(err, os.ErrNotExist) {
+		// We don't need config for all packages and that's fine
+		return &config{}, nil
+	}
+	debug.Log("Reading local package config at %q", configPath)
+	content, err := os.ReadFile(configPath)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+	cfg := &config{}
+	if err = json.Unmarshal(content, cfg); err != nil {
+		return nil, errors.WithStack(err)
+	}
+	return cfg, nil
+}
+
+func CreateFiles(pkg, basePath string) error {
+	cfg, err := get(pkg)
+	if err != nil {
+		return err
+	}
+	for name, contentPath := range cfg.CreateFiles {
+		filePath := filepath.Join(basePath, name)
+		if _, err := os.Stat(filePath); err == nil {
+			continue
+		}
+		content, err := os.ReadFile(filepath.Join(basePath, contentPath))
+		if err != nil {
+			return errors.WithStack(err)
+		}
+		if err := os.WriteFile(filePath, content, 0744); err != nil {
+			return errors.WithStack(err)
+		}
+	}
+	return nil
+}
+
+func Env(pkgs []string) (map[string]string, error) {
+	env := map[string]string{}
+	for _, pkg := range pkgs {
+		cfg, err := get(pkg)
+		if err != nil {
+			return nil, err
+		}
+		for k, v := range cfg.Env {
+			env[k] = v
+		}
+	}
+	return env, nil
+}