chore(internal/postprocessor): remove .repo-metadata-full.json processing from postprocessor main (googleapis#13129)

Add new manifest command to postprocessor for updating .repo-metadata-full.json

Author: quartzmo

internal/postprocessor/README.md

@@ -82,6 +82,28 @@ To add a new module, add the directory name of the module to `modules` in
 `google-cloud-go/internal/postprocessor/config.yaml`. Please maintain
 alphabetical ordering of the module names.
 
+## Updating the Repo Metadata Manifest
+
+The `manifest` command regenerates the `internal/.repo-metadata-full.json` file.
+This file contains metadata about all of the modules in the repository.
+
+There are two modes of operation:
+1. **Full Regeneration (default):** This mode regenerates the entire file from scratch based on the existing `config.yaml` and `.OwlBot.yaml` files. It should be run anytime there are changes to those configuration files. To prevent accidental data loss, this mode automatically preserves existing entries for modules listed in the `skip-module-scan-paths` section of the configuration.
+2. **Targeted Update (`-modules` flag):** This mode allows you to add or refresh one or more specific modules without regenerating the entire file. This is useful for adding a newly-generated module to the manifest.
+
+To run a full regeneration, from the **repository root**, run the following:
+
+```
+go run ./internal/postprocessor manifest -googleapis-dir=$GOOGLEAPIS
+```
+
+To add or refresh only the `accessapproval` and `asset` modules:
+```
+go run ./internal/postprocessor manifest -googleapis-dir=$GOOGLEAPIS -modules=accessapproval,asset
+```
+
+Note: `$GOOGLEAPIS` should be an absolute path to a local clone of `googleapis/googleapis`.
+
 ## Validating your config changes
 
 The `validate` command is run as a presubmit on changes to either the

internal/postprocessor/main.go

@@ -67,6 +67,9 @@ var (
 		"build":    true,
 		"ci":       true,
 	}
+
+	manifestCmd *flag.FlagSet
+	modulesFlag string
 )
 
 var (
@@ -78,13 +81,20 @@ var (
 	internalVersionTmpl string
 )
 
+func init() {
+	manifestCmd = flag.NewFlagSet("manifest", flag.ExitOnError)
+	manifestCmd.StringVar(&modulesFlag, "modules", "", "Comma-separated list of module names to update. If empty, regenerates entire manifest based on OwlBot config files.")
+}
+
 func main() {
 	clientRoot := flag.String("client-root", "/workspace/google-cloud-go", "Path to clients.")
 	googleapisDir := flag.String("googleapis-dir", "", "Path to googleapis/googleapis repo.")
 	directories := flag.String("dirs", "", "Comma-separated list of module names to run (not paths).")
 	branchOverride := flag.String("branch", "", "The branch that should be processed by this code")
 	githubUsername := flag.String("gh-user", "googleapis", "GitHub username where repo lives.")
 	prFilepath := flag.String("pr-file", "/workspace/new_pull_request_text.txt", "Path at which to write text file if changing PR title or body.")
+	manifestCmd.StringVar(clientRoot, "client-root", "/workspace/google-cloud-go", "Path to clients.")
+	manifestCmd.StringVar(googleapisDir, "googleapis-dir", "", "Path to googleapis/googleapis repo.")
 
 	if len(os.Args) > 1 {
 		switch os.Args[1] {
@@ -95,6 +105,36 @@ func main() {
 			}
 			log.Println("Validation complete.")
 			return
+		case "manifest":
+			manifestCmd.Parse(os.Args[2:])
+			log.Println("Starting manifest generation.")
+			if *googleapisDir == "" {
+				log.Println("creating temp dir")
+				tmpDir, err := os.MkdirTemp("", "update-postprocessor")
+				if err != nil {
+					log.Fatal(err)
+				}
+				defer os.RemoveAll(tmpDir)
+
+				log.Printf("working out %s\n", tmpDir)
+				*googleapisDir = filepath.Join(tmpDir, "googleapis")
+
+				if err := DeepClone("https://github.com/googleapis/googleapis", *googleapisDir); err != nil {
+					log.Fatal(err)
+				}
+			}
+			p := &postProcessor{
+				googleapisDir:  *googleapisDir,
+				googleCloudDir: *clientRoot,
+			}
+			if err := p.loadConfig(); err != nil {
+				log.Fatal(err)
+			}
+			if err := p.UpdateManifest(modulesFlag); err != nil {
+				log.Fatal(err)
+			}
+			log.Println("Manifest generation complete.")
+			return
 		}
 	}
 	flag.Parse()
@@ -174,7 +214,7 @@ func (p *postProcessor) run(ctx context.Context) error {
 		return nil
 	}
 
-	manifest, err := p.Manifest()
+	manifest, err := p.readManifest()
 	if err != nil {
 		return err
 	}
@@ -228,6 +268,9 @@ func (p *postProcessor) InitializeNewModules(manifest map[string]ManifestEntry)
 			}
 			// serviceImportPath here should be a valid ImportPath from a MicrogenGapicConfigs
 			apiName := manifest[serviceImportPath].Description
+			if apiName == "" {
+				return fmt.Errorf("no ManifestEntry.Description found for serviceImportPath %s. Cannot generate min required files", serviceImportPath)
+			}
 			if err := p.generateMinReqFilesNewMod(moduleName, modulePath, importPath, apiName); err != nil {
 				return err
 			}

internal/postprocessor/manifest.go

@@ -16,11 +16,13 @@ package main
 
 import (
 	"bufio"
+	"bytes"
 	"encoding/json"
 	"fmt"
 	"log"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
 
 	"cloud.google.com/go/internal/postprocessor/execv/gocmd"
@@ -51,15 +53,167 @@ const (
 	otherLibraryType       libraryType = "OTHER"
 )
 
-// Manifest writes a manifest file with info about all of the confs.
-func (p *postProcessor) Manifest() (map[string]ManifestEntry, error) {
-	log.Println("updating gapic manifest")
-	entries := map[string]ManifestEntry{} // Key is the package name.
-	f, err := os.Create(filepath.Join(p.googleCloudDir, "internal", ".repo-metadata-full.json"))
+func (p *postProcessor) readManifest() (map[string]ManifestEntry, error) {
+	log.Println("reading gapic manifest")
+	// Read existing manifest to preserve entries from skipped modules.
+	entries := map[string]ManifestEntry{}
+	manifestPath := filepath.Join(p.googleCloudDir, "internal", ".repo-metadata-full.json")
+	b, err := os.ReadFile(manifestPath)
 	if err != nil {
 		return nil, err
 	}
+	if err := json.Unmarshal(b, &entries); err != nil {
+		return nil, err
+	}
+	return entries, nil
+}
+
+// UpdateManifest regenerates or updates the `internal/.repo-metadata-full.json` file.
+// There are two modes of operation:
+//  1. **Full Regeneration (default):** This mode regenerates the entire file from scratch
+//     based on the existing `config.yaml` and `.OwlBot.yaml` files. To prevent accidental
+//     data loss, this mode automatically preserves existing entries for modules listed in
+//     the `skip-module-scan-paths` section of the configuration.
+//  2. **Targeted Update (non-empty `modules` param):** This mode allows you to add or
+//     refresh one or more specific modules without regenerating the entire file. This i
+//     useful for adding a newly-generated module to the manifest.
+func (p *postProcessor) UpdateManifest(modules string) error {
+	entries := make(map[string]ManifestEntry)
+	manifestPath := filepath.Join(p.googleCloudDir, "internal", ".repo-metadata-full.json")
+	if modules == "" {
+		log.Println("updating gapic manifest")
+		var err error
+		entries, err = p.generateManifestEntries()
+		if err != nil {
+			return err
+		}
+	} else {
+		log.Println("updating gapic manifest for", modules)
+		b, err := os.ReadFile(manifestPath)
+		if err != nil && !os.IsNotExist(err) {
+			return err
+		}
+		if err == nil {
+			if err := json.Unmarshal(b, &entries); err != nil {
+				return err
+			}
+		}
+		modList := strings.Split(modules, ",")
+		for _, mod := range modList {
+			modFound := false
+			for inputDir, li := range p.config.GoogleapisToImportPath {
+				if !strings.Contains(li.ImportPath, mod) {
+					continue
+				}
+				if li.ServiceConfig == "" {
+					continue
+				}
+				yamlPath := filepath.Join(p.googleapisDir, inputDir, li.ServiceConfig)
+				yamlFile, err := os.Open(yamlPath)
+				if err != nil {
+					return err
+				}
+				yamlConfig := struct {
+					Title    string `yaml:"title"` // We only need the title and name.
+					NameFull string `yaml:"name"`  // We only need the title and name.
+				}{}
+				if err := yaml.NewDecoder(yamlFile).Decode(&yamlConfig); err != nil {
+					return fmt.Errorf("decode: %v", err)
+				}
+				docURL, err := docURL(p.googleCloudDir, li.ImportPath, li.RelPath)
+				if err != nil {
+					return fmt.Errorf("unable to build docs URL: %v", err)
+				}
+
+				releaseLevel, err := releaseLevel(p.googleCloudDir, li)
+				if err != nil {
+					return fmt.Errorf("unable to calculate release level for %v: %v", inputDir, err)
+				}
+
+				apiShortname, err := apiShortname(yamlConfig.NameFull)
+				if err != nil {
+					return fmt.Errorf("unable to determine api_shortname from %v: %v", yamlConfig.NameFull, err)
+				}
+
+				entry := ManifestEntry{
+					APIShortname:        apiShortname,
+					DistributionName:    li.ImportPath,
+					Description:         yamlConfig.Title,
+					Language:            "go",
+					ClientLibraryType:   "generated",
+					ClientDocumentation: docURL,
+					ReleaseLevel:        releaseLevel,
+					LibraryType:         gapicAutoLibraryType,
+				}
+				entries[li.ImportPath] = entry
+				modFound = true
+			}
+			if !modFound {
+				return fmt.Errorf("configuration not found for %q", mod)
+			}
+		}
+	}
+
+	// Sort and write the manifest file.
+	keys := make([]string, 0, len(entries))
+	for k := range entries {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	f, err := os.Create(manifestPath)
+	if err != nil {
+		return err
+	}
 	defer f.Close()
+
+	buf := &bytes.Buffer{}
+	buf.WriteString("{\n")
+
+	for i, key := range keys {
+		entry := entries[key]
+		// The prefix for MarshalIndent should be "  " to indent the members of
+		// the entry, and the indent should be "  " for sub-members.
+		entryBytes, err := json.MarshalIndent(entry, "  ", "  ")
+		if err != nil {
+			return err
+		}
+
+		s := string(entryBytes)
+		firstBrace := strings.Index(s, "{")
+		// Write the key and then the marshaled entry starting from the brace.
+		buf.WriteString(fmt.Sprintf("  %q: ", key))
+		buf.WriteString(s[firstBrace:])
+
+		if i < len(keys)-1 {
+			buf.WriteString(",")
+		}
+		buf.WriteString("\n")
+	}
+
+	buf.WriteString("}\n")
+	if _, err := f.Write(buf.Bytes()); err != nil {
+		return err
+	}
+	return nil
+}
+
+// generateManifestEntries gathers info about all of the confs.
+func (p *postProcessor) generateManifestEntries() (map[string]ManifestEntry, error) {
+	// Read existing manifest to preserve entries from skipped modules.
+	existingEntries := map[string]ManifestEntry{}
+	manifestPath := filepath.Join(p.googleCloudDir, "internal", ".repo-metadata-full.json")
+	b, err := os.ReadFile(manifestPath)
+	if err != nil && !os.IsNotExist(err) {
+		return nil, err
+	}
+	if err == nil {
+		if err := json.Unmarshal(b, &existingEntries); err != nil {
+			return nil, err
+		}
+	}
+
+	entries := map[string]ManifestEntry{} // Key is the package name.
 	for _, manual := range p.config.ManualClientInfo {
 		entries[manual.DistributionName] = *manual
 	}
@@ -106,11 +260,30 @@ func (p *postProcessor) Manifest() (map[string]ManifestEntry, error) {
 		}
 		entries[li.ImportPath] = entry
 	}
+
+	// Add back entries from skipped modules that were in the old manifest.
+	for distName, entry := range existingEntries {
+		isSkipped := false
+		for _, skippedPath := range p.config.SkipModuleScanPaths {
+			prefix := "cloud.google.com/go/" + skippedPath
+			// Add a trailing slash to prefix to avoid matching substrings, e.g.
+			// `.../go/a` matching `.../go/abc`. Except when the distName is
+			// exactly the prefix without a slash.
+			if strings.HasPrefix(distName, prefix+"/") || distName == prefix {
+				isSkipped = true
+				break
+			}
+		}
+		if isSkipped {
+			if _, ok := entries[distName]; !ok {
+				entries[distName] = entry
+			}
+		}
+	}
+
 	// Remove base module entry
 	delete(entries, "")
-	enc := json.NewEncoder(f)
-	enc.SetIndent("", "  ")
-	return entries, enc.Encode(entries)
+	return entries, nil
 }
 
 // Name is of form secretmanager.googleapis.com api_shortname