feat(tfsearch): TF-26847 TF-27288: Added: dynamic schema for provider block

Author: anubhav-goel

internal/features/search/decoder/path_reader.go

@@ -87,10 +87,12 @@ func searchPathContext(record *state.SearchRecord, stateReader CombinedReader) (
 	sm.SetStateReader(stateReader)
 
 	meta := &tfsearch.Meta{
-		Path:      record.Path(),
-		Lists:     record.Meta.Lists,
-		Variables: record.Meta.Variables,
-		Filenames: record.Meta.Filenames,
+		Path:                 record.Path(),
+		Lists:                record.Meta.Lists,
+		Variables:            record.Meta.Variables,
+		Filenames:            record.Meta.Filenames,
+		ProviderReferences:   record.Meta.ProviderReferences,
+		ProviderRequirements: record.Meta.ProviderRequirements,
 	}
 
 	mergedSchema, err := sm.SchemaForSearch(meta)

internal/features/search/events.go

@@ -10,10 +10,8 @@ import (
 
 	lsctx "github.com/hashicorp/terraform-ls/internal/context"
 	"github.com/hashicorp/terraform-ls/internal/document"
-	"github.com/hashicorp/terraform-ls/internal/eventbus"
 	"github.com/hashicorp/terraform-ls/internal/features/search/ast"
 	"github.com/hashicorp/terraform-ls/internal/features/search/jobs"
-	"github.com/hashicorp/terraform-ls/internal/features/search/state"
 	"github.com/hashicorp/terraform-ls/internal/job"
 	"github.com/hashicorp/terraform-ls/internal/lsp"
 	"github.com/hashicorp/terraform-ls/internal/protocol"
@@ -190,7 +188,7 @@ func (f *SearchFeature) decodeSearch(ctx context.Context, dir document.DirHandle
 	metaId, err := f.stateStore.JobStore.EnqueueJob(ctx, job.Job{
 		Dir: dir,
 		Func: func(ctx context.Context) error {
-			return jobs.LoadSearchMetadata(ctx, f.store, path)
+			return jobs.LoadSearchMetadata(ctx, f.store, f.moduleFeature, f.logger, path)
 		},
 		Type:        operation.OpTypeLoadSearchMetadata.String(),
 		DependsOn:   job.IDs{parseId},
@@ -202,25 +200,13 @@ func (f *SearchFeature) decodeSearch(ctx context.Context, dir document.DirHandle
 				f.logger.Printf("loading module metadata returned error: %s", jobErr)
 			}
 
-			smIds, err := loadSearchModuleSources(ctx, f.store, f.bus, path)
-			if err != nil {
-				return deferIds, err
-			}
-			deferIds = append(deferIds, smIds...)
-
-			// while we now have the job ids in here, depending on the metaId job is not enough
-			// to await these spawned jobs, so we will need to move all depending jobs to this function
-			// as well. e.g. LoadSearchComponentSources, PreloadEmbeddedSchema (because future ref collection jobs depend on it), etc.
-			// we might just move all in here for simplicity
-
 			// Reference collection jobs will depend on this one, so we move it here in advance
 			eSchemaId, err := f.stateStore.JobStore.EnqueueJob(ctx, job.Job{
 				Dir: dir,
 				Func: func(ctx context.Context) error {
 					return jobs.PreloadEmbeddedSchema(ctx, f.logger, schemas.FS,
 						f.store, f.stateStore.ProviderSchemas, path)
 				},
-				// DependsOn: none required, since we are inside
 				Type:        operation.OpTypeSearchPreloadEmbeddedSchema.String(),
 				IgnoreState: ignoreState,
 			})
@@ -235,7 +221,7 @@ func (f *SearchFeature) decodeSearch(ctx context.Context, dir document.DirHandle
 					return jobs.DecodeReferenceTargets(ctx, f.store, f.moduleFeature, f.rootFeature, path)
 				},
 				Type:        operation.OpTypeDecodeReferenceTargets.String(),
-				DependsOn:   append(job.IDs{eSchemaId}, smIds...),
+				DependsOn:   job.IDs{eSchemaId},
 				IgnoreState: ignoreState,
 			})
 			if err != nil {
@@ -249,7 +235,7 @@ func (f *SearchFeature) decodeSearch(ctx context.Context, dir document.DirHandle
 					return jobs.DecodeReferenceOrigins(ctx, f.store, f.moduleFeature, f.rootFeature, path)
 				},
 				Type:        operation.OpTypeDecodeReferenceOrigins.String(),
-				DependsOn:   append(job.IDs{eSchemaId}, smIds...),
+				DependsOn:   job.IDs{eSchemaId},
 				IgnoreState: ignoreState,
 			})
 			if err != nil {
@@ -298,16 +284,3 @@ func (f *SearchFeature) removeIndexedSearch(rawPath string) {
 		return
 	}
 }
-
-func loadSearchModuleSources(ctx context.Context, testStore *state.SearchStore, bus *eventbus.EventBus, testPath string) (job.IDs, error) {
-	ids := make(job.IDs, 0)
-
-	_, err := testStore.SearchRecordByPath(testPath)
-	if err != nil {
-		return ids, err
-	}
-
-	// TODO load the adjacent Terraform module (usually ./)
-
-	return ids, nil
-}

internal/features/search/jobs/metadata.go

@@ -5,20 +5,23 @@ package jobs
 
 import (
 	"context"
+	"log"
 
 	"github.com/hashicorp/terraform-ls/internal/document"
 	"github.com/hashicorp/terraform-ls/internal/features/search/ast"
+	searchDecoder "github.com/hashicorp/terraform-ls/internal/features/search/decoder"
 	"github.com/hashicorp/terraform-ls/internal/features/search/state"
 	"github.com/hashicorp/terraform-ls/internal/job"
 	globalAst "github.com/hashicorp/terraform-ls/internal/terraform/ast"
 	"github.com/hashicorp/terraform-ls/internal/terraform/module/operation"
 	earlydecoder "github.com/hashicorp/terraform-schema/earlydecoder/search"
+	tfsearch "github.com/hashicorp/terraform-schema/search"
 )
 
 // LoadSearchMetadata loads data about the search in a version-independent
 // way that enables us to decode the rest of the configuration,
 // e.g. by knowing provider versions, etc.
-func LoadSearchMetadata(ctx context.Context, searchStore *state.SearchStore, searchPath string) error {
+func LoadSearchMetadata(ctx context.Context, searchStore *state.SearchStore, moduleFeature searchDecoder.ModuleReader, logger *log.Logger, searchPath string) error {
 	record, err := searchStore.SearchRecordByPath(searchPath)
 	if err != nil {
 		return err
@@ -38,6 +41,11 @@ func LoadSearchMetadata(ctx context.Context, searchStore *state.SearchStore, sea
 
 	meta, diags := earlydecoder.LoadSearch(record.Path(), record.ParsedFiles.AsMap())
 
+	err = loadSearchModuleSources(meta, moduleFeature, searchPath)
+	if err != nil {
+		logger.Printf("loading search module sources returned error: %s", err)
+	}
+
 	var mErr error
 	sErr := searchStore.UpdateMetadata(searchPath, meta, mErr)
 	if sErr != nil {
@@ -72,3 +80,29 @@ func LoadSearchMetadata(ctx context.Context, searchStore *state.SearchStore, sea
 
 	return mErr
 }
+
+func loadSearchModuleSources(searchMeta *tfsearch.Meta, moduleFeature searchDecoder.ModuleReader, path string) error {
+	// load metadata from the adjacent Terraform module
+	modMeta, err := moduleFeature.LocalModuleMeta(path)
+	if err != nil {
+		return err
+	}
+
+	if modMeta != nil {
+		// Convert from module provider references to search provider references
+		for moduleProviderRef, provider := range modMeta.ProviderReferences {
+			searchProviderRef := tfsearch.ProviderRef{
+				LocalName: moduleProviderRef.LocalName,
+				Alias:     moduleProviderRef.Alias,
+			}
+			searchMeta.ProviderReferences[searchProviderRef] = provider
+		}
+
+		// Copy provider requirements
+		for provider, constraints := range modMeta.ProviderRequirements {
+			searchMeta.ProviderRequirements[provider] = constraints
+		}
+	}
+
+	return nil
+}

internal/features/search/jobs/schema.go

@@ -33,29 +33,27 @@ func PreloadEmbeddedSchema(ctx context.Context, logger *log.Logger, fs fs.ReadDi
 	}
 	defer searchStore.SetPreloadEmbeddedSchemaState(searchPath, operation.OpStateLoaded)
 
-	// TODO: get providerrequirements from *.tf files
-	/*
-		pReqs := make(map[tfaddr.Provider]version.Constraints, len(record.Meta.ProviderRequirements))
-		for _, req := range record.Meta.ProviderRequirements {
-			pReqs[req.Source] = req.VersionConstraints
-		}
+	pReqs, err := searchStore.ProviderRequirementsForModule(searchPath)
+	if err != nil {
+		return err
+	}
+
+	missingReqs, err := schemaStore.MissingSchemas(pReqs)
+	if err != nil {
+		return err
+	}
+
+	if len(missingReqs) == 0 {
+		// avoid preloading any schemas if we already have all
+		return nil
+	}
 
-		missingReqs, err := schemaStore.MissingSchemas(pReqs)
+	for _, pAddr := range missingReqs {
+		err := globalState.PreloadSchemaForProviderAddr(ctx, pAddr, fs, schemaStore, logger)
 		if err != nil {
 			return err
 		}
-
-		if len(missingReqs) == 0 {
-			// avoid preloading any schemas if we already have all
-			return nil
-		}
-
-		for _, pAddr := range missingReqs {
-			err := globalState.PreloadSchemaForProviderAddr(ctx, pAddr, fs, schemaStore, logger)
-			if err != nil {
-				return err
-			}
-		} */
+	}
 
 	return nil
 

internal/features/search/state/search_meta.go

@@ -4,6 +4,7 @@
 package state
 
 import (
+	tfaddr "github.com/hashicorp/terraform-registry-address"
 	tfsearch "github.com/hashicorp/terraform-schema/search"
 )
 
@@ -14,6 +15,9 @@ type SearchMetadata struct {
 
 	Lists     map[string]tfsearch.List
 	Variables map[string]tfsearch.Variable
+
+	ProviderReferences   map[tfsearch.ProviderRef]tfaddr.Provider
+	ProviderRequirements tfsearch.ProviderRequirements
 }
 
 func (sm SearchMetadata) Copy() SearchMetadata {
@@ -35,5 +39,20 @@ func (sm SearchMetadata) Copy() SearchMetadata {
 		}
 	}
 
+	if sm.ProviderReferences != nil {
+		newSm.ProviderReferences = make(map[tfsearch.ProviderRef]tfaddr.Provider, len(sm.ProviderReferences))
+		for ref, provider := range sm.ProviderReferences {
+			newSm.ProviderReferences[ref] = provider
+		}
+	}
+
+	if sm.ProviderRequirements != nil {
+		newSm.ProviderRequirements = make(tfsearch.ProviderRequirements, len(sm.ProviderRequirements))
+		for provider, vc := range sm.ProviderRequirements {
+			// version.Constraints is never mutated in this context
+			newSm.ProviderRequirements[provider] = vc
+		}
+	}
+
 	return newSm
 }

internal/features/search/state/search_store.go

@@ -296,9 +296,11 @@ func (s *SearchStore) UpdateMetadata(path string, meta *tfsearch.Meta, mErr erro
 
 	record := oldRecord.Copy()
 	record.Meta = SearchMetadata{
-		Lists:     meta.Lists,
-		Variables: meta.Variables,
-		Filenames: meta.Filenames,
+		Lists:                meta.Lists,
+		Variables:            meta.Variables,
+		Filenames:            meta.Filenames,
+		ProviderReferences:   meta.ProviderReferences,
+		ProviderRequirements: meta.ProviderRequirements,
 	}
 	record.MetaErr = mErr
 
@@ -518,3 +520,26 @@ func (s *SearchStore) queueRecordChange(oldRecord, newRecord *SearchRecord) erro
 func (s *SearchStore) ProviderSchema(modPath string, addr tfaddr.Provider, vc version.Constraints) (*tfschema.ProviderSchema, error) {
 	return s.providerSchemasStore.ProviderSchema(modPath, addr, vc)
 }
+
+func (s *SearchStore) ProviderRequirementsForModule(modPath string) (tfsearch.ProviderRequirements, error) {
+	return s.providerRequirementsForModule(modPath, 0)
+}
+
+func (s *SearchStore) providerRequirementsForModule(searchPath string, level int) (tfsearch.ProviderRequirements, error) {
+	mod, err := s.SearchRecordByPath(searchPath)
+	if err != nil {
+		// It's possible that the configuration contains a module with an
+		// invalid local source, so we just ignore it if it can't be found.
+		// This allows us to still return provider requirements for other modules
+		return tfsearch.ProviderRequirements{}, nil
+	}
+
+	level++
+
+	requirements := make(tfsearch.ProviderRequirements, 0)
+	for k, v := range mod.Meta.ProviderRequirements {
+		requirements[k] = v
+	}
+
+	return requirements, nil
+}