Parallelize import download/read/unmarshal work while preserving ordered merge semantics

Replace global dependency download locking with per-cache locking and in-flight dedupe
Add regression coverage for concurrent imports, merge order, disabled imports, and shared git subpaths
Make URL dependency downloads context-aware and document download synchronization behavior

Signed-off-by: Ryan Swanson <ryan.swanson@loft.sh>

Author: zerbitx

e2e/tests/imports/imports.go

@@ -139,6 +139,30 @@ var _ = DevSpaceDescribe("imports", func() {
 		framework.ExpectError(err)
 	})
 
+	ginkgo.It("should import multiple git subpaths from the same repository", func() {
+		tempDir, err := framework.CopyToTempDir("tests/imports/testdata/git-subpaths")
+		framework.ExpectNoError(err)
+		defer framework.CleanupTempDir(initialDir, tempDir)
+
+		configBuffer := &bytes.Buffer{}
+		printCmd := &cmd.PrintCmd{
+			GlobalFlags: &flags.GlobalFlags{},
+			Out:         configBuffer,
+			SkipInfo:    true,
+		}
+
+		err = printCmd.Run(f)
+		framework.ExpectNoError(err)
+
+		latestConfig := &latest.Config{}
+		err = yaml.Unmarshal(configBuffer.Bytes(), latestConfig)
+		framework.ExpectNoError(err)
+
+		framework.ExpectEqual(latestConfig.Pipelines["pipeline-a"].Run, "echo \"pipeline-a\"")
+		framework.ExpectEqual(latestConfig.Pipelines["pipeline-b"].Run, "echo \"pipeline-b\"")
+		framework.ExpectEqual(latestConfig.Pipelines["pipeline-c"].Run, "echo \"pipeline-c\"")
+	})
+
 	ginkgo.It("should import correctly with localRegistry", func() {
 		tempDir, err := framework.CopyToTempDir("tests/imports/testdata/localregistry")
 		framework.ExpectNoError(err)

e2e/tests/imports/testdata/git-subpaths/devspace.yaml

@@ -0,0 +1,13 @@
+version: v2beta1
+name: git-subpath-imports
+
+imports:
+  - git: https://github.com/loft-sh/e2e-test-dependency.git
+    branch: main
+    subPath: subA
+  - git: https://github.com/loft-sh/e2e-test-dependency.git
+    branch: main
+    subPath: subB
+  - git: https://github.com/loft-sh/e2e-test-dependency.git
+    branch: main
+    subPath: subC

pkg/devspace/config/loader/imports.go

@@ -8,11 +8,13 @@ import (
 
 	"github.com/loft-sh/devspace/pkg/devspace/config/loader/variable"
 	"github.com/loft-sh/devspace/pkg/devspace/config/versions"
+	"github.com/loft-sh/devspace/pkg/devspace/config/versions/latest"
 	"github.com/loft-sh/devspace/pkg/devspace/config/versions/util"
 	dependencyutil "github.com/loft-sh/devspace/pkg/devspace/dependency/util"
 	"github.com/loft-sh/devspace/pkg/util/log"
 	"github.com/loft-sh/devspace/pkg/util/yamlutil"
 	"github.com/pkg/errors"
+	"golang.org/x/sync/errgroup"
 )
 
 var ImportSections = []string{
@@ -31,6 +33,14 @@ var ImportSections = []string{
 	"localRegistry",
 }
 
+const maxConcurrentImportDownloads = 8
+
+type resolvedImport struct {
+	ConfigPath string
+	Data       map[string]interface{}
+	Disabled   bool
+}
+
 func ResolveImports(ctx context.Context, resolver variable.Resolver, basePath string, rawData map[string]interface{}, log log.Logger) (map[string]interface{}, error) {
 	// initially reload variables
 	err := reloadVariables(resolver, rawData, log)
@@ -59,40 +69,26 @@ func ResolveImports(ctx context.Context, resolver variable.Resolver, basePath st
 		return nil, err
 	}
 
+	resolvedImports, err := loadImports(ctx, basePath, imports.Imports, version, log)
+	if err != nil {
+		return nil, err
+	}
+
 	mergedMap := map[string]interface{}{}
 	err = util.Convert(rawData, mergedMap)
 	if err != nil {
 		return nil, err
 	}
 
 	// load imports
-	for _, i := range imports.Imports {
-		if i.Enabled != nil && !*i.Enabled {
+	for _, resolvedImport := range resolvedImports {
+		if resolvedImport.Disabled {
 			continue
+		} else if resolvedImport.Data == nil {
+			return nil, errors.Errorf("resolved import is missing data")
 		}
 
-		configPath, err := dependencyutil.DownloadDependency(ctx, basePath, &i.SourceConfig, log)
-		if err != nil {
-			return nil, errors.Wrap(err, "resolve import")
-		}
-
-		fileContent, err := os.ReadFile(configPath)
-		if err != nil {
-			return nil, errors.Wrap(err, "read import config")
-		}
-
-		importData := map[string]interface{}{}
-		err = yamlutil.Unmarshal(fileContent, &importData)
-		if err != nil {
-			return nil, err
-		}
-
-		configVersion, ok := importData["version"].(string)
-		if !ok {
-			return nil, fmt.Errorf("version is missing in import config %s", configPath)
-		} else if version != configVersion {
-			return nil, fmt.Errorf("import mismatch %s != %s. Import %s has different version than currently used devspace.yaml, please make sure the versions match between an import and the devspace.yaml using it", version, configVersion, configPath)
-		}
+		importData := resolvedImport.Data
 
 		// merge sections
 		for _, section := range ImportSections {
@@ -130,16 +126,71 @@ func ResolveImports(ctx context.Context, resolver variable.Resolver, basePath st
 		// resolve the import imports
 		if importData["imports"] != nil {
 			mergedMap["imports"] = importData["imports"]
+
+			// resolve imports
+			mergedMap, err = ResolveImports(ctx, resolver, filepath.Dir(resolvedImport.ConfigPath), mergedMap, log)
+			if err != nil {
+				return nil, err
+			}
 		} else {
 			delete(mergedMap, "imports")
-		}
 
-		// resolve imports
-		mergedMap, err = ResolveImports(ctx, resolver, filepath.Dir(configPath), mergedMap, log)
-		if err != nil {
-			return nil, err
+			err = reloadVariables(resolver, mergedMap, log)
+			if err != nil {
+				return nil, err
+			}
 		}
 	}
 
 	return mergedMap, nil
 }
+
+func loadImports(ctx context.Context, basePath string, imports []latest.Import, version string, log log.Logger) ([]resolvedImport, error) {
+	resolvedImports := make([]resolvedImport, len(imports))
+
+	eg, ctx := errgroup.WithContext(ctx)
+	eg.SetLimit(maxConcurrentImportDownloads)
+	for index := range imports {
+		importConfig := imports[index]
+		if importConfig.Enabled != nil && !*importConfig.Enabled {
+			resolvedImports[index] = resolvedImport{Disabled: true}
+			continue
+		}
+
+		eg.Go(func() error {
+			configPath, err := dependencyutil.DownloadDependency(ctx, basePath, &importConfig.SourceConfig, log)
+			if err != nil {
+				return errors.Wrap(err, "resolve import")
+			}
+
+			fileContent, err := os.ReadFile(configPath)
+			if err != nil {
+				return errors.Wrap(err, "read import config")
+			}
+
+			importData := map[string]interface{}{}
+			err = yamlutil.Unmarshal(fileContent, &importData)
+			if err != nil {
+				return err
+			}
+
+			configVersion, ok := importData["version"].(string)
+			if !ok {
+				return fmt.Errorf("version is missing in import config %s", configPath)
+			} else if version != configVersion {
+				return fmt.Errorf("import mismatch %s != %s. Import %s has different version than currently used devspace.yaml, please make sure the versions match between an import and the devspace.yaml using it", version, configVersion, configPath)
+			}
+
+			resolvedImports[index] = resolvedImport{
+				ConfigPath: configPath,
+				Data:       importData,
+			}
+			return nil
+		})
+	}
+	if err := eg.Wait(); err != nil {
+		return nil, err
+	}
+
+	return resolvedImports, nil
+}