Refactor Magician build to be more modular (#15505)

Author: ScottSuarez

mmv1/api/custom_errors.go

@@ -0,0 +1,15 @@
+package api
+
+import "fmt"
+
+// ErrProductVersionNotFound is returned when a product doesn't exist
+// at the specified version or any lower version.
+type ErrProductVersionNotFound struct {
+	ProductName string
+	Version     string
+}
+
+// Error implements the error interface.
+func (e *ErrProductVersionNotFound) Error() string {
+	return fmt.Sprintf("%s does not have a '%s' version", e.ProductName, e.Version)
+}

mmv1/api/product.go

@@ -14,10 +14,14 @@
 package api
 
 import (
+	"errors"
 	"fmt"
 	"log"
+	"os"
+	"path/filepath"
 	"reflect"
 	"regexp"
+	"sort"
 	"strings"
 	"unicode"
 
@@ -34,6 +38,9 @@ type Product struct {
 	// Example inputs: "Compute", "AccessContextManager"
 	Name string
 
+	// This is the name of the package path relative to mmv1 root repo
+	PackagePath string
+
 	// original value of :name before the provider override happens
 	// same as :name if not overridden in provider
 	ApiName string `yaml:"api_name,omitempty"`
@@ -77,6 +84,164 @@ type Product struct {
 	Compiler string `yaml:"-"`
 }
 
+// Load compiles a product with all its resources from the given path and optional overrides
+// This loads the product configuration and all its resources into memory without generating any files
+func (p *Product) Load(productName string, version string, overrideDirectory string) error {
+	productYamlPath := filepath.Join(productName, "product.yaml")
+
+	var productOverridePath string
+	if overrideDirectory != "" {
+		productOverridePath = filepath.Join(overrideDirectory, productName, "product.yaml")
+	}
+
+	_, baseProductErr := os.Stat(productYamlPath)
+	baseProductExists := !errors.Is(baseProductErr, os.ErrNotExist)
+
+	_, overrideProductErr := os.Stat(productOverridePath)
+	overrideProductExists := !errors.Is(overrideProductErr, os.ErrNotExist)
+
+	if !(baseProductExists || overrideProductExists) {
+		return fmt.Errorf("%s does not contain a product.yaml file", productName)
+	}
+
+	// Compile the product configuration
+	if overrideProductExists {
+		if baseProductExists {
+			Compile(productYamlPath, p, overrideDirectory)
+			overrideApiProduct := &Product{}
+			Compile(productOverridePath, overrideApiProduct, overrideDirectory)
+			Merge(reflect.ValueOf(p).Elem(), reflect.ValueOf(*overrideApiProduct), version)
+		} else {
+			Compile(productOverridePath, p, overrideDirectory)
+		}
+	} else {
+		Compile(productYamlPath, p, overrideDirectory)
+	}
+
+	// Check if product exists at the requested version
+	if !p.ExistsAtVersionOrLower(version) {
+		return &ErrProductVersionNotFound{ProductName: productName, Version: version}
+	}
+
+	// Compile all resources
+	resources, err := p.loadResources(productName, version, overrideDirectory)
+	if err != nil {
+		return err
+	}
+
+	p.Objects = resources
+	p.PackagePath = productName
+	p.Validate()
+
+	return nil
+}
+
+// loadResources loads all resources for a product
+func (p *Product) loadResources(productName string, version string, overrideDirectory string) ([]*Resource, error) {
+	var resources []*Resource = make([]*Resource, 0)
+
+	// Get base resource files
+	resourceFiles, err := filepath.Glob(fmt.Sprintf("%s/*", productName))
+	if err != nil {
+		return nil, fmt.Errorf("cannot get resource files: %v", err)
+	}
+
+	// Compile base resources (skip those that will be merged with overrides)
+	for _, resourceYamlPath := range resourceFiles {
+		if filepath.Base(resourceYamlPath) == "product.yaml" || filepath.Ext(resourceYamlPath) != ".yaml" {
+			continue
+		}
+
+		// Skip if resource will be merged in the override loop
+		if overrideDirectory != "" {
+			resourceOverridePath := filepath.Join(overrideDirectory, resourceYamlPath)
+			_, overrideResourceErr := os.Stat(resourceOverridePath)
+			if !errors.Is(overrideResourceErr, os.ErrNotExist) {
+				continue
+			}
+		}
+
+		resource := p.loadResource(resourceYamlPath, "", version, overrideDirectory)
+		resources = append(resources, resource)
+	}
+
+	// Compile override resources
+	if overrideDirectory != "" {
+		resources, err = p.reconcileOverrideResources(productName, version, overrideDirectory, resources)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return resources, nil
+}
+
+// reconcileOverrideResources handles resolution of override resources
+func (p *Product) reconcileOverrideResources(productName string, version string, overrideDirectory string, resources []*Resource) ([]*Resource, error) {
+	productOverridePath := filepath.Join(overrideDirectory, productName, "product.yaml")
+	productOverrideDir := filepath.Dir(productOverridePath)
+
+	overrideFiles, err := filepath.Glob(fmt.Sprintf("%s/*", productOverrideDir))
+	if err != nil {
+		return nil, fmt.Errorf("cannot get override files: %v", err)
+	}
+
+	for _, overrideYamlPath := range overrideFiles {
+		if filepath.Base(overrideYamlPath) == "product.yaml" || filepath.Ext(overrideYamlPath) != ".yaml" {
+			continue
+		}
+
+		baseResourcePath := filepath.Join(productName, filepath.Base(overrideYamlPath))
+		resource := p.loadResource(baseResourcePath, overrideYamlPath, version, overrideDirectory)
+		resources = append(resources, resource)
+	}
+
+	// Sort resources by name for consistent output
+	sort.Slice(resources, func(i, j int) bool {
+		return resources[i].Name < resources[j].Name
+	})
+
+	return resources, nil
+}
+
+// loadResource loads a single resource with optional override
+func (p *Product) loadResource(baseResourcePath string, overrideResourcePath string, version string, overrideDirectory string) *Resource {
+	resource := &Resource{}
+
+	// Check if base resource exists
+	_, baseResourceErr := os.Stat(baseResourcePath)
+	baseResourceExists := !errors.Is(baseResourceErr, os.ErrNotExist)
+
+	if overrideResourcePath != "" {
+		if baseResourceExists {
+			// Merge base and override
+			Compile(baseResourcePath, resource, overrideDirectory)
+			overrideResource := &Resource{}
+			Compile(overrideResourcePath, overrideResource, overrideDirectory)
+			Merge(reflect.ValueOf(resource).Elem(), reflect.ValueOf(*overrideResource), version)
+			resource.SourceYamlFile = baseResourcePath
+		} else {
+			// Override only
+			Compile(overrideResourcePath, resource, overrideDirectory)
+		}
+	} else {
+		// Base only
+		Compile(baseResourcePath, resource, overrideDirectory)
+		resource.SourceYamlFile = baseResourcePath
+	}
+
+	// Set resource defaults and validate
+	resource.TargetVersionName = version
+	// SetDefault before AddExtraFields to ensure relevant metadata is available on existing fields
+	resource.SetDefault(p)
+	resource.Properties = resource.AddExtraFields(resource.PropertiesWithExcluded(), nil)
+	// SetDefault after AddExtraFields to ensure relevant metadata is available for the newly generated fields
+	resource.SetDefault(p)
+	resource.Validate()
+
+	return resource
+}
+
 func (p *Product) UnmarshalYAML(unmarshal func(any) error) error {
 	type productAlias Product
 	aliasObj := (*productAlias)(p)

mmv1/loader/loader.go

@@ -0,0 +1,137 @@
+package loader
+
+import (
+	"errors"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"sync"
+
+	"github.com/GoogleCloudPlatform/magic-modules/mmv1/api"
+	"golang.org/x/exp/slices"
+)
+
+type Loader struct {
+	OverrideDirectory string
+	Version           string
+}
+
+func (l *Loader) LoadProducts() map[string]*api.Product {
+	if l.Version == "" {
+		log.Printf("No version specified, assuming ga")
+		l.Version = "ga"
+	}
+
+	var allProductFiles []string = make([]string, 0)
+
+	files, err := filepath.Glob("products/**/product.yaml")
+	if err != nil {
+		panic(err)
+	}
+	for _, filePath := range files {
+		dir := filepath.Dir(filePath)
+		allProductFiles = append(allProductFiles, fmt.Sprintf("products/%s", filepath.Base(dir)))
+	}
+
+	if l.OverrideDirectory != "" {
+		log.Printf("Using override directory %s", l.OverrideDirectory)
+
+		// Normalize override dir to a path that is relative to the magic-modules directory
+		// This is needed for templates that concatenate pwd + override dir + path
+		if filepath.IsAbs(l.OverrideDirectory) {
+			wd, err := os.Getwd()
+			if err != nil {
+				panic(err)
+			}
+			l.OverrideDirectory, err = filepath.Rel(wd, l.OverrideDirectory)
+			log.Printf("Override directory normalized to relative path %s", l.OverrideDirectory)
+		}
+
+		overrideFiles, err := filepath.Glob(fmt.Sprintf("%s/products/**/product.yaml", l.OverrideDirectory))
+		if err != nil {
+			panic(err)
+		}
+		for _, filePath := range overrideFiles {
+			product, err := filepath.Rel(l.OverrideDirectory, filePath)
+			if err != nil {
+				panic(err)
+			}
+			dir := filepath.Dir(product)
+			productFile := fmt.Sprintf("products/%s", filepath.Base(dir))
+			if !slices.Contains(allProductFiles, productFile) {
+				allProductFiles = append(allProductFiles, productFile)
+			}
+		}
+	}
+
+	return l.batchLoadProducts(allProductFiles)
+}
+
+func (l *Loader) batchLoadProducts(productNames []string) map[string]*api.Product {
+	products := make(map[string]*api.Product)
+
+	// Create result type for clarity
+	type loadResult struct {
+		name    string
+		product *api.Product
+		err     error
+	}
+
+	// Buffered channel to prevent goroutine blocking
+	productChan := make(chan loadResult, len(productNames))
+
+	// Use WaitGroup for proper synchronization
+	var wg sync.WaitGroup
+
+	// Launch all goroutines
+	for _, productName := range productNames {
+		wg.Add(1)
+		go func(name string) {
+			defer wg.Done()
+
+			product, err := l.loadProductOnly(name)
+			productChan <- loadResult{
+				name:    name,
+				product: product,
+				err:     err,
+			}
+		}(productName)
+	}
+
+	wg.Wait()
+	close(productChan)
+
+	// Collect results as they complete
+	loadFailureCount := 0
+	for result := range productChan {
+		if result.err != nil {
+			// Check if the error is the specific "version not found" error
+			var versionErr *api.ErrProductVersionNotFound
+			if errors.As(result.err, &versionErr) {
+				continue
+			}
+
+			loadFailureCount++
+			log.Printf("Error loading %s: %v", result.name, result.err)
+			continue
+		}
+		products[result.name] = result.product
+	}
+	if loadFailureCount > 0 {
+		log.Fatalf("Failed to load %d products", loadFailureCount)
+	}
+
+	return products
+}
+
+// loadProductOnly is a standalone function to just load a product without generation
+// This can be used when you only need to load and validate a product configuration
+func (l *Loader) loadProductOnly(productName string) (*api.Product, error) {
+	product := &api.Product{}
+	err := product.Load(productName, l.Version, l.OverrideDirectory)
+	if err != nil {
+		return nil, err
+	}
+	return product, nil
+}