chore: Adding VFS testing (#5490)

* chore: Adding VFS testing

* chore: Adding VFS integration in `providercache`

* fix: Refactoring to route integration with `afero` through `internal/vfs` so that it's easier to swap out should we need to in the future.

* fix: Cleaning up names

* fix: renaming `provider_cache.go` to `providercache.go`

* fix: Removing unnecessary indirection for this check

* chore: Addressing review comments

* fix: Cleanup

* fix: Fixing error handlings

* fix: Cleaner APIs for managing structs

* fix: Cleanup of `FileExists`

* fix: Cleanup of builder pattern and unzip logic

* fix: Adding a little testing

* fix: Cleaning up implementation of zip decompressor

* fix: Adding reasonable limits

* fix: Addressing review feedback

Author: yhakbar

go.mod

@@ -94,6 +94,7 @@ require (
 	github.com/gobwas/glob v0.2.3
 	github.com/invopop/jsonschema v0.13.0
 	github.com/mattn/go-shellwords v1.0.12
+	github.com/spf13/afero v1.15.0
 	github.com/testcontainers/testcontainers-go v0.40.0
 	github.com/wI2L/jsondiff v0.7.0
 	github.com/xeipuuv/gojsonschema v1.2.0

go.sum

@@ -981,6 +981,8 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1
 github.com/smartystreets/goconvey v0.0.0-20180222194500-ef6db91d284a/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
+github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I=
+github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg=
 github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
 github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo=
 github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=

internal/providercache/provider_cache.go internal/providercache/providercache.gointernal/providercache/provider_cache.go renamed to internal/providercache/providercache.go

@@ -27,6 +27,7 @@ import (
 	"github.com/gruntwork-io/terragrunt/internal/tf/cliconfig"
 	"github.com/gruntwork-io/terragrunt/internal/tf/getproviders"
 	"github.com/gruntwork-io/terragrunt/internal/util"
+	"github.com/gruntwork-io/terragrunt/internal/vfs"
 	"github.com/gruntwork-io/terragrunt/pkg/log"
 	"github.com/gruntwork-io/terragrunt/pkg/options"
 )
@@ -78,23 +79,46 @@ type ProviderCache struct {
 	*cache.Server
 	cliCfg          *cliconfig.Config
 	providerService *services.ProviderService
+	fs              vfs.FS
 }
 
-func InitServer(l log.Logger, opts *options.TerragruntOptions) (*ProviderCache, error) {
+// NewProviderCache creates a new ProviderCache with sensible defaults.
+// Use builder methods like WithFS() to customize the configuration.
+func NewProviderCache() *ProviderCache {
+	return &ProviderCache{
+		fs: vfs.NewOSFS(),
+	}
+}
+
+// WithFS sets the filesystem for file operations and returns the ProviderCache
+// for method chaining. If not called, defaults to the real OS filesystem.
+func (pc *ProviderCache) WithFS(fs vfs.FS) *ProviderCache {
+	pc.fs = fs
+	return pc
+}
+
+// FS returns the configured filesystem.
+func (pc *ProviderCache) FS() vfs.FS {
+	return pc.fs
+}
+
+// Init initializes the ProviderCache with the given logger and options.
+// Call this after configuring the ProviderCache with builder methods.
+func (pc *ProviderCache) Init(l log.Logger, opts *options.TerragruntOptions) error {
 	// ProviderCacheDir has the same file structure as terraform plugin_cache_dir.
 	// https://developer.hashicorp.com/terraform/cli/config/config-file#provider-plugin-cache
 	if opts.ProviderCacheDir == "" {
 		cacheDir, err := util.GetCacheDir()
 		if err != nil {
-			return nil, err
+			return fmt.Errorf("failed to get cache directory: %w", err)
 		}
 
 		opts.ProviderCacheDir = filepath.Join(cacheDir, "providers")
 	}
 
 	var err error
 	if opts.ProviderCacheDir, err = filepath.Abs(opts.ProviderCacheDir); err != nil {
-		return nil, errors.New(err)
+		return errors.New(err)
 	}
 
 	if opts.ProviderCacheToken == "" {
@@ -105,25 +129,26 @@ func InitServer(l log.Logger, opts *options.TerragruntOptions) (*ProviderCache,
 		opts.ProviderCacheToken = fmt.Sprintf("%s:%s", APIKeyAuth, opts.ProviderCacheToken)
 	}
 
-	cliCfg, err := cliconfig.LoadUserConfig()
+	// Pass filesystem to LoadUserConfig
+	cliCfg, err := cliconfig.LoadUserConfig(cliconfig.WithFS(pc.FS()))
 	if err != nil {
-		return nil, err
+		return err
 	}
 
 	userProviderDir, err := cliconfig.UserProviderDir()
 	if err != nil {
-		return nil, err
+		return err
 	}
 
-	providerService := services.NewProviderService(opts.ProviderCacheDir, userProviderDir, cliCfg.CredentialsSource(), l)
+	providerService := services.NewProviderService(opts.ProviderCacheDir, userProviderDir, cliCfg.CredentialsSource(), l, services.WithFS(pc.FS()))
 	proxyProviderHandler := handlers.NewProxyProviderHandler(l, cliCfg.CredentialsSource())
 
 	providerHandlers, err := handlers.NewProviderHandlers(cliCfg, l, opts.ProviderCacheRegistryNames)
 	if err != nil {
-		return nil, errors.Errorf("creating provider handlers failed: %w", err)
+		return errors.Errorf("creating provider handlers failed: %w", err)
 	}
 
-	cache := cache.NewServer(
+	cacheServer := cache.NewServer(
 		cache.WithHostname(opts.ProviderCacheHostname),
 		cache.WithPort(opts.ProviderCachePort),
 		cache.WithToken(opts.ProviderCacheToken),
@@ -134,18 +159,29 @@ func InitServer(l log.Logger, opts *options.TerragruntOptions) (*ProviderCache,
 		cache.WithLogger(l),
 	)
 
-	return &ProviderCache{
-		Server:          cache,
-		cliCfg:          cliCfg,
-		providerService: providerService,
-	}, nil
+	pc.Server = cacheServer
+	pc.cliCfg = cliCfg
+	pc.providerService = providerService
+
+	return nil
+}
+
+// InitServer creates and initializes a new ProviderCache with the given logger and options.
+// This is a convenience function that combines NewProviderCache() and Init().
+func InitServer(l log.Logger, opts *options.TerragruntOptions) (*ProviderCache, error) {
+	pc := NewProviderCache()
+	if err := pc.Init(l, opts); err != nil {
+		return nil, err
+	}
+
+	return pc, nil
 }
 
 // TerraformCommandHook warms up the providers cache, creates `.terraform.lock.hcl` and runs the `tofu/terraform init`
 // command with using this cache. Used as a hook function that is called after running the target tofu/terraform command.
 // For example, if the target command is `tofu plan`, it will be intercepted before it is run in the `/shell` package,
 // then control will be passed to this function to init the working directory using cached providers.
-func (cache *ProviderCache) TerraformCommandHook(
+func (pc *ProviderCache) TerraformCommandHook(
 	ctx context.Context,
 	l log.Logger,
 	opts *options.TerragruntOptions,
@@ -182,18 +218,18 @@ func (cache *ProviderCache) TerraformCommandHook(
 
 	env := providerCacheEnvironment(opts, cliConfigFilename)
 
-	if output, err := cache.warmUpCache(ctx, l, opts, cliConfigFilename, args, env); err != nil {
+	if output, err := pc.warmUpCache(ctx, l, opts, cliConfigFilename, args, env); err != nil {
 		return output, err
 	}
 
 	if skipRunTargetCommand {
 		return &util.CmdOutput{}, nil
 	}
 
-	return cache.runTerraformWithCache(ctx, l, opts, cliConfigFilename, args, env)
+	return pc.runTerraformWithCache(ctx, l, opts, cliConfigFilename, args, env)
 }
 
-func (cache *ProviderCache) warmUpCache(
+func (pc *ProviderCache) warmUpCache(
 	ctx context.Context,
 	l log.Logger,
 	opts *options.TerragruntOptions,
@@ -207,7 +243,7 @@ func (cache *ProviderCache) warmUpCache(
 	)
 
 	// Create terraform cli config file that enables provider caching and does not use provider cache dir
-	if err := cache.createLocalCLIConfig(ctx, opts, cliConfigFilename, cacheRequestID); err != nil {
+	if err := pc.createLocalCLIConfig(ctx, opts, cliConfigFilename, cacheRequestID); err != nil {
 		return nil, err
 	}
 
@@ -222,7 +258,7 @@ func (cache *ProviderCache) warmUpCache(
 		}
 	}
 
-	caches, err := cache.providerService.WaitForCacheReady(cacheRequestID)
+	caches, err := pc.providerService.WaitForCacheReady(cacheRequestID)
 	if err != nil {
 		return nil, err
 	}
@@ -265,7 +301,7 @@ func (cache *ProviderCache) warmUpCache(
 	return nil, err
 }
 
-func (cache *ProviderCache) runTerraformWithCache(
+func (pc *ProviderCache) runTerraformWithCache(
 	ctx context.Context,
 	l log.Logger,
 	opts *options.TerragruntOptions,
@@ -274,7 +310,7 @@ func (cache *ProviderCache) runTerraformWithCache(
 	env map[string]string,
 ) (*util.CmdOutput, error) {
 	// Create terraform cli config file that uses provider cache dir
-	if err := cache.createLocalCLIConfig(ctx, opts, cliConfigFilename, ""); err != nil {
+	if err := pc.createLocalCLIConfig(ctx, opts, cliConfigFilename, ""); err != nil {
 		return nil, err
 	}
 
@@ -318,8 +354,8 @@ func (cache *ProviderCache) runTerraformWithCache(
 // It creates two types of configuration depending on the `cacheRequestID` variable set.
 // 1. If `cacheRequestID` is set, `terraform init` does _not_ use the provider cache directory, the cache server creates a cache for requested providers and returns HTTP status 423. Since for each module we create the CLI config, using `cacheRequestID` we have the opportunity later retrieve from the cache server exactly those cached providers that were requested by `terraform init` using this configuration.
 // 2. If `cacheRequestID` is empty, 'terraform init` uses provider cache directory, the cache server acts as a proxy.
-func (cache *ProviderCache) createLocalCLIConfig(ctx context.Context, opts *options.TerragruntOptions, filename string, cacheRequestID string) error {
-	cfg := cache.cliCfg.Clone()
+func (pc *ProviderCache) createLocalCLIConfig(ctx context.Context, opts *options.TerragruntOptions, filename string, cacheRequestID string) error {
+	cfg := pc.cliCfg.Clone()
 	cfg.PluginCacheDir = ""
 
 	// Filter registries based on OpenTofu or Terraform implementation to avoid contacting unnecessary registries
@@ -333,13 +369,13 @@ func (cache *ProviderCache) createLocalCLIConfig(ctx context.Context, opts *opti
 	for _, registryName := range filteredRegistryNames {
 		providerInstallationIncludes = append(providerInstallationIncludes, registryName+"/*/*")
 
-		apiURLs, err := cache.DiscoveryURL(ctx, registryName)
+		apiURLs, err := pc.DiscoveryURL(ctx, registryName)
 		if err != nil {
 			return err
 		}
 
 		cfg.AddHost(registryName, map[string]string{
-			"providers.v1": fmt.Sprintf("%s/%s/%s/", cache.ProviderController.URL(), cacheRequestID, registryName),
+			"providers.v1": fmt.Sprintf("%s/%s/%s/", pc.ProviderController.URL(), cacheRequestID, registryName),
 			// Since Terragrunt Provider Cache only caches providers, we need to route module requests to the original registry.
 			"modules.v1": ResolveModulesURL(registryName, apiURLs.ModulesV1),
 		})
@@ -357,8 +393,17 @@ func (cache *ProviderCache) createLocalCLIConfig(ctx context.Context, opts *opti
 		cliconfig.NewProviderInstallationDirect(nil, nil),
 	)
 
-	if cfgDir := filepath.Dir(filename); !util.FileExists(cfgDir) {
-		if err := os.MkdirAll(cfgDir, os.ModePerm); err != nil {
+	// Use VFS for directory operations
+	fs := pc.FS()
+	cfgDir := filepath.Dir(filename)
+
+	cfgDirExists, err := vfs.FileExists(fs, cfgDir)
+	if err != nil {
+		return errors.New(err)
+	}
+
+	if !cfgDirExists {
+		if err := fs.MkdirAll(cfgDir, os.ModePerm); err != nil {
 			return errors.New(err)
 		}
 	}

…nal/providercache/provider_cache_test.go …rnal/providercache/providercache_test.gointernal/providercache/provider_cache_test.go renamed to internal/providercache/providercache_test.go internal/providercache/provider_cache_test.go renamed to internal/providercache/providercache_test.go

@@ -17,6 +17,7 @@ import (
 	"github.com/gruntwork-io/terragrunt/internal/tf/cache/handlers"
 	"github.com/gruntwork-io/terragrunt/internal/tf/cache/services"
 	"github.com/gruntwork-io/terragrunt/internal/tf/cliconfig"
+	"github.com/gruntwork-io/terragrunt/internal/vfs"
 	"github.com/gruntwork-io/terragrunt/pkg/options"
 	"github.com/gruntwork-io/terragrunt/test/helpers"
 	"github.com/gruntwork-io/terragrunt/test/helpers/logger"
@@ -269,42 +270,62 @@ func TestProviderCache(t *testing.T) {
 	}
 }
 
-func TestProviderCacheWithProviderCacheDir(t *testing.T) {
-	// testing.T can Setenv, but can't Unsetenv
-	unsetEnv := func(t *testing.T, v string) {
-		t.Helper()
-
-		// let testing.T do the recovery and work around t.Parallel()
-		t.Setenv(v, "")
-		require.NoError(t, os.Unsetenv(v))
-	}
+func TestProviderCacheHomeless(t *testing.T) {
+	cacheDir := helpers.TmpDirWOSymlinks(t)
 
-	t.Run("Homeless", func(t *testing.T) { //nolint:paralleltest
-		cacheDir := helpers.TmpDirWOSymlinks(t)
+	t.Setenv("HOME", "")
+	require.NoError(t, os.Unsetenv("HOME"))
 
-		unsetEnv(t, "HOME")
-		unsetEnv(t, "XDG_CACHE_HOME")
+	t.Setenv("XDG_CACHE_HOME", "")
+	require.NoError(t, os.Unsetenv("XDG_CACHE_HOME"))
 
-		_, err := providercache.InitServer(logger.CreateLogger(), &options.TerragruntOptions{
-			ProviderCacheDir: cacheDir,
-		})
-		require.NoError(t, err, "ProviderCache shouldn't read HOME environment variable")
+	_, err := providercache.InitServer(logger.CreateLogger(), &options.TerragruntOptions{
+		ProviderCacheDir: cacheDir,
 	})
+	require.NoError(t, err, "ProviderCache shouldn't read HOME environment variable")
+}
+
+func TestProviderCacheWithProviderCacheDir(t *testing.T) {
+	t.Parallel()
 
 	t.Run("NoNewDirectoriesAtHOME", func(t *testing.T) {
-		home := helpers.TmpDirWOSymlinks(t)
-		cacheDir := helpers.TmpDirWOSymlinks(t)
+		t.Parallel()
+
+		// Use in-memory filesystem to isolate file operations from the real filesystem.
+		// This ensures InitServer doesn't create any directories on the real filesystem
+		// since all file operations are routed through the VFS.
+		memFs := vfs.NewMemMapFS()
+		cacheDir := "/test/provider-cache"
+
+		server := providercache.NewProviderCache().WithFS(memFs)
+		err := server.Init(
+			logger.CreateLogger(),
+			&options.TerragruntOptions{
+				ProviderCacheDir: cacheDir,
+			},
+		)
+		require.NoError(t, err)
+
+		// With VFS, all file operations go through the in-memory filesystem,
+		// so no directories should be created on the real filesystem at all.
+		// We can verify the VFS is being used by checking it's not empty or
+		// by the fact that no errors occurred despite using fake paths.
+	})
 
-		t.Setenv("HOME", home)
+	t.Run("InitServerWithVFS", func(t *testing.T) {
+		t.Parallel()
 
-		_, err := providercache.InitServer(logger.CreateLogger(), &options.TerragruntOptions{
-			ProviderCacheDir: cacheDir,
-		})
-		require.NoError(t, err)
+		memFs := vfs.NewMemMapFS()
+		cacheDir := "/vfs/provider-cache"
 
-		// Cache server shouldn't create any directory at $HOME when ProviderCacheDir is specified
-		entries, err := os.ReadDir(home)
+		server := providercache.NewProviderCache().WithFS(memFs)
+		err := server.Init(
+			logger.CreateLogger(),
+			&options.TerragruntOptions{
+				ProviderCacheDir: cacheDir,
+			},
+		)
 		require.NoError(t, err)
-		require.Empty(t, entries, "No new directories should be created at $HOME")
+		require.NotNil(t, server, "Init should return a valid server when using VFS")
 	})
 }