feat(internal/librarian): add allowlist (#3177)

Create an allowlist of libraries that we expected to be generated across
all client libraries, starting with Rust. Use this list to determine
libraries that should be generated using default values when running
`librarian generate --all`.

Fixes #3076

Author: julieqiu

internal/librarian/generate.go

@@ -21,7 +21,6 @@ import (
 	"io/fs"
 	"os"
 	"path/filepath"
-	"strings"
 
 	"github.com/googleapis/librarian/internal/config"
 	"github.com/googleapis/librarian/internal/fetch"
@@ -79,6 +78,16 @@ func runGenerate(ctx context.Context, all bool, libraryName string) error {
 }
 
 func generateAll(ctx context.Context, cfg *config.Config) error {
+	googleapisDir, err := fetchGoogleapisDir(ctx, cfg.Sources)
+	if err != nil {
+		return err
+	}
+
+	libraries, err := deriveDefaultLibraries(cfg, googleapisDir)
+	if err != nil {
+		return err
+	}
+	cfg.Libraries = append(cfg.Libraries, libraries...)
 	for _, lib := range cfg.Libraries {
 		if err := generateLibrary(ctx, cfg, lib.Name); err != nil {
 			return err
@@ -87,6 +96,77 @@ func generateAll(ctx context.Context, cfg *config.Config) error {
 	return nil
 }
 
+// deriveDefaultLibraries finds libraries for allowed channels that are not
+// explicitly configured in librarian.yaml.
+//
+// For each allowed channel without configuration, it derives default values
+// for the library name and output path. If the output directory exists, the
+// library is added for generation. Channels whose output directories do not
+// exist in the librarian.yaml but should be generated are returned.
+func deriveDefaultLibraries(cfg *config.Config, googleapisDir string) ([]*config.Library, error) {
+	if cfg.Default == nil {
+		return nil, nil
+	}
+
+	configured := make(map[string]bool)
+	for _, lib := range cfg.Libraries {
+		for _, ch := range lib.Channels {
+			configured[ch.Path] = true
+		}
+	}
+
+	var derived []*config.Library
+	for channel := range serviceconfig.Allowlist {
+		if configured[channel] {
+			continue
+		}
+		name := defaultLibraryName(cfg.Language, channel)
+		output := defaultOutput(cfg.Language, channel, cfg.Default.Output)
+		if !dirExists(output) {
+			continue
+		}
+		sc, err := serviceconfig.Find(googleapisDir, channel)
+		if err != nil {
+			return nil, err
+		}
+		derived = append(derived, &config.Library{
+			Name:   name,
+			Output: output,
+			Channels: []*config.Channel{{
+				Path:          channel,
+				ServiceConfig: sc,
+			}},
+		})
+	}
+	return derived, nil
+}
+
+func defaultLibraryName(language, channel string) string {
+	switch language {
+	case "rust":
+		return rust.DefaultLibraryName(channel)
+	default:
+		return channel
+	}
+}
+
+func defaultOutput(language, channel, defaultOut string) string {
+	switch language {
+	case "rust":
+		return rust.DefaultOutput(channel, defaultOut)
+	default:
+		return defaultOut
+	}
+}
+
+func dirExists(path string) bool {
+	info, err := os.Stat(path)
+	if err != nil {
+		return false
+	}
+	return info.IsDir()
+}
+
 func generateLibrary(ctx context.Context, cfg *config.Config, libraryName string) error {
 	googleapisDir, err := fetchGoogleapisDir(ctx, cfg.Sources)
 	if err != nil {
@@ -98,7 +178,10 @@ func generateLibrary(ctx context.Context, cfg *config.Config, libraryName string
 				fmt.Printf("⊘ Skipping %s (skip_generate is set)\n", lib.Name)
 				return nil
 			}
-			lib = prepareLibrary(cfg.Language, lib, cfg.Default)
+			lib, err := prepareLibrary(cfg.Language, lib, cfg.Default)
+			if err != nil {
+				return err
+			}
 			for _, api := range lib.Channels {
 				if api.ServiceConfig == "" {
 					serviceConfig, err := serviceconfig.Find(googleapisDir, api.Path)
@@ -117,26 +200,14 @@ func generateLibrary(ctx context.Context, cfg *config.Config, libraryName string
 // prepareLibrary applies language-specific derivations and fills defaults.
 // For Rust libraries without an explicit output path, it derives the output
 // from the first channel path before applying defaults.
-func prepareLibrary(language string, lib *config.Library, defaults *config.Default) *config.Library {
-	// TODO(https://github.com/googleapis/librarian/issues/2966):
-	// refactor so that the switch statement logic is in one place
-	if language == "rust" && lib.Output == "" && len(lib.Channels) > 0 {
-		lib.Output = deriveDefaultRustOutput(lib.Channels[0].Path, defaults.Output)
+func prepareLibrary(language string, lib *config.Library, defaults *config.Default) (*config.Library, error) {
+	if lib.Output == "" {
+		if len(lib.Channels) == 0 {
+			return nil, fmt.Errorf("library %q has no channels, cannot determine default output", lib.Name)
+		}
+		lib.Output = defaultOutput(language, lib.Channels[0].Path, defaults.Output)
 	}
-	return fillDefaults(lib, defaults)
-}
-
-// deriveDefaultRustOutput returns the output path for a Rust library. If the
-// library has an explicit output path that differs from the default, it returns
-// that path. Otherwise, it derives the output from the first channel path by
-// stripping the "google/" prefix and joining with the default output. For
-// example, the default output for google/cloud/secretmanager/v1 is
-// src/generated/cloud/secretmanager/v1.
-//
-// TODO(https://github.com/googleapis/librarian/issues/2966): refactor and move
-// to internal/rust package.
-func deriveDefaultRustOutput(channel, defaultOutput string) string {
-	return filepath.Join(defaultOutput, strings.TrimPrefix(channel, "google/"))
+	return fillDefaults(lib, defaults), nil
 }
 
 func generate(ctx context.Context, language string, library *config.Library, sources *config.Sources) error {
@@ -147,7 +218,7 @@ func generate(ctx context.Context, language string, library *config.Library, sou
 	case "rust":
 		keep := append(library.Keep, "Cargo.toml")
 		if err := cleanOutput(library.Output, keep); err != nil {
-			return err
+			return fmt.Errorf("library %s: %w", library.Name, err)
 		}
 		err = rust.Generate(ctx, library, sources)
 	default:

internal/librarian/generate_test.go

@@ -223,6 +223,7 @@ func TestPrepareLibrary(t *testing.T) {
 		output   string
 		channels []*config.Channel
 		want     string
+		wantErr  bool
 	}{
 		{
 			name:     "empty output derives path from channel",
@@ -244,10 +245,10 @@ func TestPrepareLibrary(t *testing.T) {
 			want:     "src/generated",
 		},
 		{
-			name:     "rust with no channels uses default",
+			name:     "rust with no channels returns error",
 			language: "rust",
 			channels: nil,
-			want:     "src/generated",
+			wantErr:  true,
 		},
 	} {
 		t.Run(test.name, func(t *testing.T) {
@@ -259,45 +260,23 @@ func TestPrepareLibrary(t *testing.T) {
 			defaults := &config.Default{
 				Output: "src/generated",
 			}
-			got := prepareLibrary(test.language, lib, defaults)
+			got, err := prepareLibrary(test.language, lib, defaults)
+			if test.wantErr {
+				if err == nil {
+					t.Fatal("expected error, got nil")
+				}
+				return
+			}
+			if err != nil {
+				t.Fatal(err)
+			}
 			if got.Output != test.want {
 				t.Errorf("got output %q, want %q", got.Output, test.want)
 			}
 		})
 	}
 }
 
-func TestDeriveDefaultRustOutput(t *testing.T) {
-	for _, test := range []struct {
-		name    string
-		channel string
-		want    string
-	}{
-		{
-			name:    "standard google cloud path",
-			channel: "google/cloud/secretmanager/v1",
-			want:    "src/generated/cloud/secretmanager/v1",
-		},
-		{
-			name:    "nested api path",
-			channel: "google/cloud/aiplatform/v1beta1",
-			want:    "src/generated/cloud/aiplatform/v1beta1",
-		},
-		{
-			name:    "non-cloud path",
-			channel: "google/iam/v1",
-			want:    "src/generated/iam/v1",
-		},
-	} {
-		t.Run(test.name, func(t *testing.T) {
-			got := deriveDefaultRustOutput(test.channel, "src/generated")
-			if got != test.want {
-				t.Errorf("got %q, want %q", got, test.want)
-			}
-		})
-	}
-}
-
 func TestCleanOutput(t *testing.T) {
 	for _, test := range []struct {
 		name    string
@@ -384,3 +363,67 @@ func TestCleanOutput(t *testing.T) {
 		})
 	}
 }
+
+func TestDeriveDefaultLibrariesSkipsConfigured(t *testing.T) {
+	cfg := &config.Config{
+		Language: "rust",
+		Default:  &config.Default{Output: t.TempDir()},
+		Libraries: []*config.Library{{
+			Name:     "secretmanager",
+			Channels: []*config.Channel{{Path: "google/cloud/secretmanager/v1"}},
+		}},
+	}
+	derived, err := deriveDefaultLibraries(cfg, t.TempDir())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(derived) != 0 {
+		t.Errorf("got %d derived libraries, want 0", len(derived))
+	}
+}
+
+func TestDeriveDefaultLibrariesWithOutputDir(t *testing.T) {
+	outputDir := t.TempDir()
+	googleapisDir := t.TempDir()
+
+	writeServiceConfig(t, googleapisDir, "google/cloud/speech/v2", "speech_v2.yaml")
+	if err := os.MkdirAll(filepath.Join(outputDir, "cloud/speech/v2"), 0755); err != nil {
+		t.Fatal(err)
+	}
+
+	cfg := &config.Config{
+		Language: "rust",
+		Default:  &config.Default{Output: outputDir},
+	}
+	derived, err := deriveDefaultLibraries(cfg, googleapisDir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(derived) != 1 {
+		t.Fatalf("got %d derived libraries, want 1", len(derived))
+	}
+
+	want := &config.Library{
+		Name:   "google-cloud-speech-v2",
+		Output: filepath.Join(outputDir, "cloud/speech/v2"),
+		Channels: []*config.Channel{{
+			Path:          "google/cloud/speech/v2",
+			ServiceConfig: "google/cloud/speech/v2/speech_v2.yaml",
+		}},
+	}
+	if diff := cmp.Diff(want, derived[0]); diff != "" {
+		t.Errorf("mismatch (-want +got):\n%s", diff)
+	}
+}
+
+func writeServiceConfig(t *testing.T, googleapisDir, channel, filename string) {
+	t.Helper()
+	dir := filepath.Join(googleapisDir, channel)
+	if err := os.MkdirAll(dir, 0755); err != nil {
+		t.Fatal(err)
+	}
+	content := "type: google.api.Service\nname: test.googleapis.com\n"
+	if err := os.WriteFile(filepath.Join(dir, filename), []byte(content), 0644); err != nil {
+		t.Fatal(err)
+	}
+}

internal/librarian/internal/rust/generate.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"fmt"
 	"path/filepath"
+	"strings"
 
 	"github.com/googleapis/librarian/internal/command"
 	"github.com/googleapis/librarian/internal/config"
@@ -70,3 +71,16 @@ func sourceDir(ctx context.Context, source *config.Source, repo string) (string,
 	}
 	return fetch.RepoDir(ctx, repo, source.Commit, source.SHA256)
 }
+
+// DefaultLibraryName derives a library name from a channel path.
+// For example: google/cloud/secretmanager/v1 -> google-cloud-secretmanager-v1.
+func DefaultLibraryName(channel string) string {
+	return strings.ReplaceAll(channel, "/", "-")
+}
+
+// DefaultOutput derives an output path from a channel path and default output.
+// For example: google/cloud/secretmanager/v1 with default src/generated/
+// returns src/generated/cloud/secretmanager/v1.
+func DefaultOutput(channel, defaultOutput string) string {
+	return filepath.Join(defaultOutput, strings.TrimPrefix(channel, "google/"))
+}