Allow merging identical source files from different pallets to the same target (#307)

ethanjli · web-flow · commit 6e25b3d180c7 · 2024-09-22T23:22:07.000Z
* Allow merging identical source files from different pallets to the same target

* Bump changelog version for v0.8.0-alpha.2 prerelease

* Use two-space indent for generating bundle manifest files

* Show info about overridden required pallets in bundle manifest files

* Report file imports &amp; provenance of imported files in bundle manifests

* Attach version strings to overriding pallets/repos without dirty changes, when inferrable via git

* Show info about transitive pallet requirements in bundle manifest

* Bump changelog version for v0.8.0-alpha.2 prerelease
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## Unreleased
+## 0.8.0-alpha.2 - 2024-09-22
 
 ### Added
 
@@ -18,6 +18,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - (cli) Added a `[dev] plt locate-plt-file` command to print the actual filesystem path of the specified file in the specified pallet required by the local/development pallet.
 - (cli) Added a `[dev] plt show-plt-file` command to print the contents of the specified file in the specified pallet required by the local/development pallet.
 - (cli) Added a `cache rm-dl` command to delete the cache of downloaded files for export.
+- (cli) The bundle manifest's `includes` section's description of required pallets now reports when required pallets were overridden.
+- (cli) The bundle manifest's `includes` section's description of required pallets now recursively shows information about transitively-required pallets (but does not show information about file import groups in those transitively-required pallets).
+- (cli) The bundle manifest's `includes` section's description of required pallets now shows the results (as target file path -> source file path mappings) of evaluating each file import group attached to their respective required pallets.
+- (cli) The bundle manifest now has an `imports` section which describes the provenance of each imported file, as a list of how the file has been transitively imported across pallets (with pallets farther down the list being depeer in the transitive import chain).
 
 ### Changed
 
diff --git a/cmd/forklift/dev/plt/pallets.go b/cmd/forklift/dev/plt/pallets.go
@@ -133,6 +133,12 @@ func loadReplacementPallets(fsPaths []string) (replacements []*forklift.FSPallet
 		if len(externalPallets) == 0 {
 			return nil, errors.Errorf("no replacement pallets found in path %s", replacementPath)
 		}
+		for _, pallet := range externalPallets {
+			version, clean := fcli.CheckGitRepoVersion(pallet.FS.Path())
+			if clean {
+				pallet.Version = version
+			}
+		}
 		replacements = append(replacements, externalPallets...)
 	}
 	return replacements, nil
@@ -203,6 +209,12 @@ func loadReplacementRepos(
 		if len(externalRepos) == 0 {
 			return nil, errors.Errorf("no replacement repos found in path %s", replacementPath)
 		}
+		for _, repo := range externalRepos {
+			version, clean := fcli.CheckGitRepoVersion(repo.FS.Path())
+			if clean {
+				repo.Version = version
+			}
+		}
 		replacements = append(replacements, externalRepos...)
 	}
 	return replacements, nil
diff --git a/cmd/forklift/main.go b/cmd/forklift/main.go
@@ -101,7 +101,7 @@ const (
 	bundleMinVersion = "v0.7.0"
 	// newBundleVersion is the Forklift version reported in new staged pallet bundles made by Forklift.
 	// Older versions of the Forklift tool cannot use such bundles.
-	newBundleVersion = "v0.7.0"
+	newBundleVersion = "v0.8.0-alpha.2"
 	// newStageStoreVersion is the Forklift version reported in a stage store initialized by Forklift.
 	// Older versions of the Forklift tool cannot use the state store.
 	newStageStoreVersion = "v0.7.0"
diff --git a/internal/app/forklift/bundles-models.go b/internal/app/forklift/bundles-models.go
@@ -51,11 +51,16 @@ type BundleManifest struct {
 	Pallet BundlePallet `yaml:"pallet"`
 	// Includes describes repos and pallets used to define the bundle's package deployments.
 	Includes BundleInclusions `yaml:"includes,omitempty"`
-	// Deploys describes deployments provided by the bundle. Keys are names of deployments.
-	Deploys map[string]DeplDef `yaml:"deploys,omitempty"`
+	// Imports lists the files imported from required pallets and the fully-qualified paths of those
+	// source files (relative to their respective source pallets). Keys are the target paths of the
+	// files, while values are lists showing the chain of provenance of the respective files (with
+	// the deepest ancestor at the end of each list).
+	Imports map[string][]string `yaml:"imports,omitempty"`
 	// Downloads lists the URLs of files and OCI images downloaded for export by the bundle's
 	// deployments. Keys are names of the bundle's deployments which export downloaded files.
 	Downloads map[string][]string `yaml:"downloads,omitempty"`
+	// Deploys describes deployments provided by the bundle. Keys are names of deployments.
+	Deploys map[string]DeplDef `yaml:"deploys,omitempty"`
 	// Exports lists the target paths of file exports provided by the bundle's deployments. Keys are
 	// names of the bundle's deployments which provide file exports.
 	Exports map[string][]string `yaml:"exports,omitempty"`
@@ -90,6 +95,13 @@ type BundlePalletInclusion struct {
 	// Override describes the pallet used to override the required pallet, if an override was
 	// specified for the pallet when building the bundled pallet.
 	Override BundleInclusionOverride `yaml:"override,omitempty"`
+	// Includes describes pallets used to define the pallet, omitting information about file imports.
+	Includes map[string]BundlePalletInclusion `yaml:"includes,omitempty"`
+	// Imports lists the files imported from the pallet, organized by import group. Keys are the names
+	// of the import groups, and values are the results of evaluating the respective import groups -
+	// i.e. maps whose keys are target file paths (where the files are imported to) and whose values
+	// are source file paths (where the files are imported from).
+	Imports map[string]map[string]string `yaml:"imports,omitempty"`
 }
 
 // BundleRepoInclusion describes a package repository used to build the bundled pallet.
diff --git a/internal/app/forklift/bundles.go b/internal/app/forklift/bundles.go
@@ -2,6 +2,7 @@ package forklift
 
 import (
 	"archive/tar"
+	"bytes"
 	"compress/gzip"
 	"fmt"
 	"io"
@@ -60,13 +61,16 @@ func LoadFSBundle(fsys core.PathedFS, subdirPath string) (b *FSBundle, err error
 }
 
 func (b *FSBundle) WriteManifestFile() error {
-	marshaled, err := yaml.Marshal(b.Manifest)
-	if err != nil {
+	buf := bytes.Buffer{}
+	encoder := yaml.NewEncoder(&buf)
+	const yamlIndent = 2
+	encoder.SetIndent(yamlIndent)
+	if err := encoder.Encode(b.Manifest); err != nil {
 		return errors.Wrapf(err, "couldn't marshal bundle manifest")
 	}
 	outputPath := filepath.FromSlash(path.Join(b.FS.Path(), BundleManifestFile))
 	const perm = 0o644 // owner rw, group r, public r
-	if err := os.WriteFile(outputPath, marshaled, perm); err != nil {
+	if err := os.WriteFile(outputPath, buf.Bytes(), perm); err != nil {
 		return errors.Wrapf(err, "couldn't save bundle manifest to %s", outputPath)
 	}
 	return nil
diff --git a/internal/app/forklift/cli/staging.go b/internal/app/forklift/cli/staging.go
@@ -6,6 +6,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"strings"
 
 	dct "github.com/compose-spec/compose-go/v2/types"
 	"github.com/pkg/errors"
@@ -73,24 +74,24 @@ type StagingCaches struct {
 }
 
 func StagePallet(
-	indent int, pallet *forklift.FSPallet, stageStore *forklift.FSStageStore, caches StagingCaches,
+	indent int, merged *forklift.FSPallet, stageStore *forklift.FSStageStore, caches StagingCaches,
 	exportPath string, versions StagingVersions,
 	skipImageCaching, parallel, ignoreToolVersion bool,
 ) (index int, err error) {
-	if _, isMerged := pallet.FS.(*forklift.MergeFS); isMerged {
+	if _, isMerged := merged.FS.(*forklift.MergeFS); isMerged {
 		return 0, errors.Errorf("the pallet provided for staging should not be a merged pallet!")
 	}
 
-	pallet, repoCacheWithMerged, err := CacheStagingReqs(
-		0, pallet, caches.Mirrors, caches.Pallets, caches.Repos, caches.Downloads, false, parallel,
+	merged, repoCacheWithMerged, err := CacheStagingReqs(
+		0, merged, caches.Mirrors, caches.Pallets, caches.Repos, caches.Downloads, false, parallel,
 	)
 	if err != nil {
 		return 0, errors.Wrap(err, "couldn't cache requirements for staging the pallet")
 	}
 	// Note: we must have all requirements in the cache before we can check their compatibility with
 	// the Forklift tool version
 	if err = CheckDeepCompat(
-		pallet, caches.Pallets, repoCacheWithMerged, versions.Core, ignoreToolVersion,
+		merged, caches.Pallets, repoCacheWithMerged, versions.Core, ignoreToolVersion,
 	); err != nil {
 		return 0, err
 	}
@@ -102,10 +103,10 @@ func StagePallet(
 	}
 	fmt.Printf("Bundling pallet as stage %d for staged application...\n", index)
 	if err = buildBundle(
-		pallet, repoCacheWithMerged, caches.Downloads,
+		merged, caches.Pallets, repoCacheWithMerged, caches.Downloads,
 		versions.NewBundle, path.Join(stageStore.FS.Path(), fmt.Sprintf("%d", index)),
 	); err != nil {
-		return index, errors.Wrapf(err, "couldn't bundle pallet %s as stage %d", pallet.Path(), index)
+		return index, errors.Wrapf(err, "couldn't bundle pallet %s as stage %d", merged.Path(), index)
 	}
 	if err = SetNextStagedBundle(
 		indent, stageStore, index, exportPath, versions.Core.Tool, versions.MinSupportedBundle,
@@ -119,19 +120,18 @@ func StagePallet(
 }
 
 func buildBundle(
-	pallet *forklift.FSPallet,
-	repoCache forklift.PathedRepoCache, dlCache *forklift.FSDownloadCache,
+	merged *forklift.FSPallet,
+	palletCache forklift.PathedPalletCache, repoCache forklift.PathedRepoCache,
+	dlCache *forklift.FSDownloadCache,
 	forkliftVersion, outputPath string,
 ) (err error) {
 	outputBundle := forklift.NewFSBundle(outputPath)
-	// TODO: once we can overlay pallets, save the result of overlaying the pallets to a `overlay`
-	// subdir
-	outputBundle.Manifest, err = newBundleManifest(pallet, repoCache, forkliftVersion)
+	outputBundle.Manifest, err = newBundleManifest(merged, palletCache, repoCache, forkliftVersion)
 	if err != nil {
 		return errors.Wrapf(err, "couldn't create bundle manifest for %s", outputBundle.FS.Path())
 	}
 
-	depls, _, err := Check(0, pallet, repoCache)
+	depls, _, err := Check(0, merged, repoCache)
 	if err != nil {
 		return errors.Wrap(err, "couldn't ensure pallet validity")
 	}
@@ -141,8 +141,8 @@ func buildBundle(
 		}
 	}
 
-	if err := outputBundle.SetBundledPallet(pallet); err != nil {
-		return errors.Wrapf(err, "couldn't write pallet %s into bundle", pallet.Def.Pallet.Path)
+	if err := outputBundle.SetBundledPallet(merged); err != nil {
+		return errors.Wrapf(err, "couldn't write pallet %s into bundle", merged.Def.Pallet.Path)
 	}
 	if err = outputBundle.WriteRepoDefFile(); err != nil {
 		return errors.Wrap(err, "couldn't write repo declaration into bundle")
@@ -157,13 +157,15 @@ func buildBundle(
 }
 
 func newBundleManifest(
-	pallet *forklift.FSPallet, repoCache forklift.PathedRepoCache, forkliftVersion string,
+	merged *forklift.FSPallet,
+	palletCache forklift.PathedPalletCache, repoCache forklift.PathedRepoCache,
+	forkliftVersion string,
 ) (forklift.BundleManifest, error) {
 	desc := forklift.BundleManifest{
 		ForkliftVersion: forkliftVersion,
 		Pallet: forklift.BundlePallet{
-			Path:        pallet.Path(),
-			Description: pallet.Def.Pallet.Description,
+			Path:        merged.Path(),
+			Description: merged.Def.Pallet.Description,
 		},
 		Includes: forklift.BundleInclusions{
 			Pallets: make(map[string]forklift.BundlePalletInclusion),
@@ -173,30 +175,45 @@ func newBundleManifest(
 		Downloads: make(map[string][]string),
 		Exports:   make(map[string][]string),
 	}
-	desc.Pallet.Version, desc.Pallet.Clean = checkGitRepoVersion(pallet.FS.Path())
-	palletReqs, err := pallet.LoadFSPalletReqs("**")
+	desc.Pallet.Version, desc.Pallet.Clean = CheckGitRepoVersion(merged.FS.Path())
+	palletReqs, err := merged.LoadFSPalletReqs("**")
 	if err != nil {
-		return desc, errors.Wrapf(err, "couldn't determine pallets required by pallet %s", pallet.Path())
+		return desc, errors.Wrapf(err, "couldn't determine pallets required by pallet %s", merged.Path())
 	}
-	// TODO: once we can overlay pallets, the description of pallet & repo inclusions should probably
-	// be made from the result of overlaying. We could also describe pre-overlay requirements from the
-	// bundled pallet, in desc.Pallet.Requires.
 	for _, req := range palletReqs {
-		inclusion := forklift.BundlePalletInclusion{Req: req.PalletReq}
-		// TODO: also check for overridden pallets
-		desc.Includes.Pallets[req.RequiredPath] = inclusion
+		if desc.Includes.Pallets[req.RequiredPath], err = newBundlePalletInclusion(
+			merged, req, palletCache, true,
+		); err != nil {
+			return desc, errors.Wrapf(
+				err, "couldn't generate description of requirement for pallet %s", req.RequiredPath,
+			)
+		}
 	}
-	repoReqs, err := pallet.LoadFSRepoReqs("**")
+	repoReqs, err := merged.LoadFSRepoReqs("**")
 	if err != nil {
-		return desc, errors.Wrapf(err, "couldn't determine repos required by pallet %s", pallet.Path())
+		return desc, errors.Wrapf(err, "couldn't determine repos required by pallet %s", merged.Path())
 	}
 	for _, req := range repoReqs {
 		desc.Includes.Repos[req.RequiredPath] = newBundleRepoInclusion(req, repoCache)
 	}
+	if mergeFS, ok := merged.FS.(*forklift.MergeFS); ok {
+		imports, err := mergeFS.ListImports()
+		if err != nil {
+			return desc, errors.Wrapf(err, "couldn't list pallet file import groups")
+		}
+		desc.Imports = make(map[string][]string)
+		for target, sourceRef := range imports {
+			sources := make([]string, 0, len(sourceRef.Sources))
+			for _, source := range sourceRef.Sources {
+				sources = append(sources, path.Join(source, sourceRef.Path))
+			}
+			desc.Imports[target] = sources
+		}
+	}
 	return desc, nil
 }
 
-func checkGitRepoVersion(palletPath string) (version string, clean bool) {
+func CheckGitRepoVersion(palletPath string) (version string, clean bool) {
 	gitRepo, err := git.Open(filepath.FromSlash(palletPath))
 	if err != nil {
 		return "", false
@@ -220,6 +237,102 @@ func checkGitRepoVersion(palletPath string) (version string, clean bool) {
 	return versionString, status.IsClean()
 }
 
+func newBundlePalletInclusion(
+	pallet *forklift.FSPallet, req *forklift.FSPalletReq, palletCache forklift.PathedPalletCache,
+	describeImports bool,
+) (inclusion forklift.BundlePalletInclusion, err error) {
+	inclusion = forklift.BundlePalletInclusion{
+		Req:      req.PalletReq,
+		Includes: make(map[string]forklift.BundlePalletInclusion),
+	}
+	for {
+		if palletCache == nil {
+			break
+		}
+		layeredCache, ok := palletCache.(*forklift.LayeredPalletCache)
+		if !ok {
+			break
+		}
+		overlay := layeredCache.Overlay
+		if overlay == nil {
+			palletCache = layeredCache.Underlay
+			continue
+		}
+
+		if loaded, err := overlay.LoadFSPallet(req.RequiredPath, req.VersionLock.Version); err == nil {
+			// i.e. the pallet was overridden
+			inclusion.Override.Path = loaded.FS.Path()
+			inclusion.Override.Version, inclusion.Override.Clean = CheckGitRepoVersion(loaded.FS.Path())
+			break
+		}
+		palletCache = layeredCache.Underlay
+	}
+
+	loaded, err := palletCache.LoadFSPallet(req.RequiredPath, req.VersionLock.Version)
+	if err != nil {
+		return inclusion, errors.Wrapf(err, "couldn't load pallet %s", req.RequiredPath)
+	}
+	palletReqs, err := loaded.LoadFSPalletReqs("**")
+	if err != nil {
+		return inclusion, errors.Wrapf(
+			err, "couldn't determine pallets required by pallet %s", loaded.Path(),
+		)
+	}
+	for _, req := range palletReqs {
+		if inclusion.Includes[req.RequiredPath], err = newBundlePalletInclusion(
+			loaded, req, palletCache, false,
+		); err != nil {
+			return inclusion, errors.Wrapf(
+				err, "couldn't generate description of transitive requirement for pallet %s", loaded.Path(),
+			)
+		}
+	}
+
+	if !describeImports {
+		return inclusion, nil
+	}
+	if inclusion.Imports, err = describePalletImports(pallet, req, palletCache); err != nil {
+		return inclusion, errors.Wrapf(err, "couldn't describe file imports for %s", req.RequiredPath)
+	}
+	return inclusion, nil
+}
+
+func describePalletImports(
+	pallet *forklift.FSPallet, req *forklift.FSPalletReq, palletCache forklift.PathedPalletCache,
+) (fileMappings map[string]map[string]string, err error) {
+	imports, err := pallet.LoadImports(path.Join(req.RequiredPath, "**/*"))
+	if err != nil {
+		return nil, errors.Wrap(err, "couldn't load file import groups")
+	}
+	allResolved, err := forklift.ResolveImports(pallet, palletCache, imports)
+	if err != nil {
+		return nil, errors.Wrap(err, "couldn't resolve file import groups")
+	}
+	requiredPallets := make(map[string]*forklift.FSPallet) // pallet path -> pallet
+	for _, resolved := range allResolved {
+		requiredPallets[resolved.Pallet.Path()] = resolved.Pallet
+	}
+	for palletPath, requiredPallet := range requiredPallets {
+		if requiredPallets[palletPath], err = forklift.MergeFSPallet(
+			requiredPallet, palletCache, nil,
+		); err != nil {
+			return nil, errors.Wrapf(
+				err, "couldn't compute merged pallet for required pallet %s", palletPath,
+			)
+		}
+	}
+
+	fileMappings = make(map[string]map[string]string)
+	for _, resolved := range allResolved {
+		resolved.Pallet = requiredPallets[req.RequiredPath]
+		importName := strings.TrimPrefix(resolved.Name, req.RequiredPath+"/")
+		if fileMappings[importName], err = resolved.Evaluate(palletCache); err != nil {
+			return nil, errors.Wrapf(err, "couldn't evaluate file import group %s", importName)
+		}
+	}
+	return fileMappings, nil
+}
+
 func newBundleRepoInclusion(
 	req *forklift.FSRepoReq, repoCache forklift.PathedRepoCache,
 ) forklift.BundleRepoInclusion {
@@ -238,11 +351,10 @@ func newBundleRepoInclusion(
 			continue
 		}
 
-		if repo, err := overlay.LoadFSRepo(
-			req.RequiredPath, req.VersionLock.Version,
-		); err == nil { // i.e. the repo was overridden
-			inclusion.Override.Path = repo.FS.Path()
-			inclusion.Override.Version, inclusion.Override.Clean = checkGitRepoVersion(repo.FS.Path())
+		if loaded, err := overlay.LoadFSRepo(req.RequiredPath, req.VersionLock.Version); err == nil {
+			// i.e. the repo was overridden
+			inclusion.Override.Path = loaded.FS.Path()
+			inclusion.Override.Version, inclusion.Override.Clean = CheckGitRepoVersion(loaded.FS.Path())
 			return inclusion
 		}
 		repoCache = layeredCache.Underlay
diff --git a/internal/app/forklift/pallets-merging.go b/internal/app/forklift/pallets-merging.go
