feat(librariangen): generate grpc stubs and resource helpers

Author: meltsufin

internal/librariangen/generate/generator.go

@@ -78,26 +78,25 @@ func Generate(ctx context.Context, cfg *Config) error {
 		return fmt.Errorf("librariangen: invalid configuration: %w", err)
 	}
 	slog.Debug("librariangen: generate command started")
-	defer cleanupIntermediateFiles(cfg.OutputDir)
+	outputConfig := &protoc.OutputConfig{
+		GAPICDir: filepath.Join(cfg.OutputDir, "gapic"),
+		GRPCDir:  filepath.Join(cfg.OutputDir, "grpc"),
+		ProtoDir: filepath.Join(cfg.OutputDir, "proto"),
+	}
+	defer cleanupIntermediateFiles(outputConfig)
 
 	generateReq, err := readGenerateReq(cfg.LibrarianDir)
 	if err != nil {
 		return fmt.Errorf("librariangen: failed to read request: %w", err)
 	}
 
-	if err := invokeProtoc(ctx, cfg, generateReq); err != nil {
+	if err := invokeProtoc(ctx, cfg, generateReq, outputConfig); err != nil {
 		return fmt.Errorf("librariangen: gapic generation failed: %w", err)
 	}
 
-	// Unzip the generated zip file.
-	zipPath := filepath.Join(cfg.OutputDir, "java_gapic.zip")
-	if err := unzip(zipPath, cfg.OutputDir); err != nil {
-		return fmt.Errorf("librariangen: failed to unzip %s: %w", zipPath, err)
-	}
-
-	// Unzip the inner temp-codegen.srcjar.
-	srcjarPath := filepath.Join(cfg.OutputDir, "temp-codegen.srcjar")
-	srcjarDest := filepath.Join(cfg.OutputDir, "java_gapic_srcjar")
+	// Unzip the temp-codegen.srcjar.
+	srcjarPath := filepath.Join(outputConfig.GAPICDir, "temp-codegen.srcjar")
+	srcjarDest := outputConfig.GAPICDir
 	if err := unzip(srcjarPath, srcjarDest); err != nil {
 		return fmt.Errorf("librariangen: failed to unzip %s: %w", srcjarPath, err)
 	}
@@ -113,18 +112,26 @@ func Generate(ctx context.Context, cfg *Config) error {
 // invokeProtoc handles the protoc GAPIC generation logic for the 'generate' CLI command.
 // It reads a request file, and for each API specified, it invokes protoc
 // to generate the client library. It returns the module path and the path to the service YAML.
-func invokeProtoc(ctx context.Context, cfg *Config, generateReq *message.Library) error {
+func invokeProtoc(ctx context.Context, cfg *Config, generateReq *message.Library, outputConfig *protoc.OutputConfig) error {
 	for _, api := range generateReq.APIs {
 		apiServiceDir := filepath.Join(cfg.SourceDir, api.Path)
 		slog.Info("processing api", "service_dir", apiServiceDir)
 		bazelConfig, err := bazelParse(apiServiceDir)
 		if err != nil {
 			return fmt.Errorf("librariangen: failed to parse BUILD.bazel for %s: %w", apiServiceDir, err)
 		}
-		args, err := protocBuild(apiServiceDir, bazelConfig, cfg.SourceDir, cfg.OutputDir)
+		args, err := protocBuild(apiServiceDir, bazelConfig, cfg.SourceDir, outputConfig)
 		if err != nil {
 			return fmt.Errorf("librariangen: failed to build protoc command for api %q in library %q: %w", api.Path, generateReq.ID, err)
 		}
+
+		// Create protoc output directories.
+		for _, dir := range []string{outputConfig.ProtoDir, outputConfig.GRPCDir, outputConfig.GAPICDir} {
+			if err := os.MkdirAll(dir, 0755); err != nil {
+				return err
+			}
+		}
+
 		if err := execvRun(ctx, args, cfg.OutputDir); err != nil {
 			return fmt.Errorf("librariangen: protoc failed for api %q in library %q: %w", api.Path, generateReq.ID, err)
 		}
@@ -168,29 +175,43 @@ func restructureOutput(outputDir, libraryID string) error {
 	slog.Debug("librariangen: restructuring output directory", "dir", outputDir)
 
 	// Define source and destination directories.
-	gapicSrcDir := filepath.Join(outputDir, "java_gapic_srcjar", "src", "main", "java")
-	gapicTestDir := filepath.Join(outputDir, "java_gapic_srcjar", "src", "test", "java")
-	protoSrcDir := filepath.Join(outputDir, "com")
-	samplesDir := filepath.Join(outputDir, "java_gapic_srcjar", "samples", "snippets")
-
+	gapicSrcDir := filepath.Join(outputDir, "gapic", "src", "main", "java")
+	gapicTestDir := filepath.Join(outputDir, "gapic", "src", "test", "java")
+	protoSrcDir := filepath.Join(outputDir, "proto")
+	resourceNameSrcDir := filepath.Join(outputDir, "gapic", "proto", "src", "main", "java")
+	grpcSrcDir := filepath.Join(outputDir, "grpc")
+	samplesDir := filepath.Join(outputDir, "gapic", "samples", "snippets")
+
+	// TODO(meltsufin): currently we assume we have a single API variant v1
 	gapicDestDir := filepath.Join(outputDir, fmt.Sprintf("google-cloud-%s", libraryID), "src", "main", "java")
 	gapicTestDestDir := filepath.Join(outputDir, fmt.Sprintf("google-cloud-%s", libraryID), "src", "test", "java")
 	protoDestDir := filepath.Join(outputDir, fmt.Sprintf("proto-google-cloud-%s-v1", libraryID), "src", "main", "java")
+	resourceNameDestDir := filepath.Join(outputDir, fmt.Sprintf("proto-google-cloud-%s-v1", libraryID), "src", "main", "java")
+	grpcDestDir := filepath.Join(outputDir, fmt.Sprintf("grpc-google-cloud-%s-v1", libraryID), "src", "main", "java")
 	samplesDestDir := filepath.Join(outputDir, "samples", "snippets")
 
 	// Create destination directories.
-	destDirs := []string{gapicDestDir, gapicTestDestDir, protoDestDir, samplesDestDir}
+	destDirs := []string{gapicDestDir, gapicTestDestDir, protoDestDir, samplesDestDir, grpcDestDir}
 	for _, dir := range destDirs {
 		if err := os.MkdirAll(dir, 0755); err != nil {
 			return err
 		}
 	}
 
-	// Move files.
+	// The resource name directory is not created if there are no resource names
+	// to generate. We create it here to avoid errors later.
+	if _, err := os.Stat(resourceNameSrcDir); os.IsNotExist(err) {
+		if err := os.MkdirAll(resourceNameSrcDir, 0755); err != nil {
+			return err
+		}
+	}
+
+	// Move files that won't have conflicts.
 	moves := map[string]string{
 		gapicSrcDir:  gapicDestDir,
 		gapicTestDir: gapicTestDestDir,
 		protoSrcDir:  protoDestDir,
+		grpcSrcDir:  grpcDestDir,
 		samplesDir:   samplesDestDir,
 	}
 	for src, dest := range moves {
@@ -199,19 +220,46 @@ func restructureOutput(outputDir, libraryID string) error {
 		}
 	}
 
+	// Merge the resource name files into the proto destination.
+	if err := copyAndMerge(resourceNameSrcDir, resourceNameDestDir); err != nil {
+		return err
+	}
+
 	return nil
 }
 
-func cleanupIntermediateFiles(outputDir string) {
-	slog.Debug("librariangen: cleaning up intermediate files", "dir", outputDir)
-	filesToRemove := []string{
-		"java_gapic_srcjar",
-		"com",
-		"java_gapic.zip",
-		"temp-codegen.srcjar",
+// copyAndMerge recursively copies the contents of src to dest, merging directories.
+func copyAndMerge(src, dest string) error {
+	entries, err := os.ReadDir(src)
+	if os.IsNotExist(err) {
+		return nil
+	}
+	if err != nil {
+		return err
 	}
-	for _, file := range filesToRemove {
-		path := filepath.Join(outputDir, file)
+
+	for _, entry := range entries {
+		srcPath := filepath.Join(src, entry.Name())
+		destPath := filepath.Join(dest, entry.Name())
+		if entry.IsDir() {
+			if err := os.MkdirAll(destPath, 0755); err != nil {
+				return err
+			}
+			if err := copyAndMerge(srcPath, destPath); err != nil {
+				return err
+			}
+		} else {
+			if err := os.Rename(srcPath, destPath); err != nil {
+				return fmt.Errorf("librariangen: failed to move %s to %s: %w", srcPath, destPath, err)
+			}
+		}
+	}
+	return nil
+}
+
+func cleanupIntermediateFiles(outputConfig *protoc.OutputConfig) {
+	slog.Debug("librariangen: cleaning up intermediate files")
+	for _, path := range []string{outputConfig.GAPICDir, outputConfig.GRPCDir, outputConfig.ProtoDir} {
 		if err := os.RemoveAll(path); err != nil {
 			slog.Error("librariangen: failed to clean up intermediate file", "path", path, "error", err)
 		}

internal/librariangen/generate/generator_test.go

@@ -24,6 +24,8 @@ import (
 	"path/filepath"
 	"strings"
 	"testing"
+
+	"cloud.google.com/java/internal/librariangen/protoc"
 )
 
 // testEnv encapsulates a temporary test environment.
@@ -110,40 +112,15 @@ func createFakeZip(t *testing.T, path string) {
 	zipWriter := zip.NewWriter(newZipFile)
 	defer zipWriter.Close()
 
-	// Create a temporary empty zip file to be included in the main zip file.
-	tmpfile, err := os.CreateTemp("", "temp-zip-*.zip")
-	if err != nil {
-		t.Fatalf("failed to create temp file: %v", err)
-	}
-	defer os.Remove(tmpfile.Name())
-
-	tempZipWriter := zip.NewWriter(tmpfile)
-	// Add the src/main/java directory to the inner zip file.
-	_, err = tempZipWriter.Create("src/main/java/")
+	// Add the src/main/java directory to the zip file.
+	_, err = zipWriter.Create("src/main/java/")
 	if err != nil {
 		t.Fatalf("failed to create directory in zip: %v", err)
 	}
-	_, err = tempZipWriter.Create("src/test/java/")
+	_, err = zipWriter.Create("src/test/java/")
 	if err != nil {
 		t.Fatalf("failed to create directory in zip: %v", err)
 	}
-	tempZipWriter.Close()
-
-	// Read the content of the temporary zip file.
-	zipBytes, err := os.ReadFile(tmpfile.Name())
-	if err != nil {
-		t.Fatalf("failed to read temp zip file: %v", err)
-	}
-
-	// Add the temporary zip file to the main zip file as temp-codegen.srcjar.
-	w, err := zipWriter.Create("temp-codegen.srcjar")
-	if err != nil {
-		t.Fatalf("failed to create empty file in zip: %v", err)
-	}
-	_, err = w.Write(zipBytes)
-	if err != nil {
-		t.Fatalf("failed to write content to zip: %v", err)
-	}
 }
 
 func TestGenerate(t *testing.T) {
@@ -255,12 +232,19 @@ java_gapic_library(
 				}
 				if tt.protocErr == nil && tt.name != "unzip fails" {
 					// Simulate protoc creating the zip file.
-					createFakeZip(t, filepath.Join(e.outputDir, "java_gapic.zip"))
+					zipPath := filepath.Join(e.outputDir, "gapic", "temp-codegen.srcjar")
+					if err := os.MkdirAll(filepath.Dir(zipPath), 0755); err != nil {
+						t.Fatalf("failed to create directory: %v", err)
+					}
+					createFakeZip(t, zipPath)
 					// Create the directory that is expected by restructureOutput.
-					if err := os.MkdirAll(filepath.Join(e.outputDir, "com"), 0755); err != nil {
+					if err := os.MkdirAll(filepath.Join(e.outputDir, "gapic", "src", "main", "java"), 0755); err != nil {
+						t.Fatalf("failed to create directory: %v", err)
+					}
+					if err := os.MkdirAll(filepath.Join(e.outputDir, "gapic", "src", "test", "java"), 0755); err != nil {
 						t.Fatalf("failed to create directory: %v", err)
 					}
-					if err := os.MkdirAll(filepath.Join(e.outputDir, "java_gapic_srcjar", "samples", "snippets"), 0755); err != nil {
+					if err := os.MkdirAll(filepath.Join(e.outputDir, "gapic", "samples", "snippets"), 0755); err != nil {
 						t.Fatalf("failed to create directory: %v", err)
 					}
 				}
@@ -348,48 +332,45 @@ func TestConfig_Validate(t *testing.T) {
 func TestRestructureOutput(t *testing.T) {
 	e := newTestEnv(t)
 	defer e.cleanup(t)
-	// Create dummy files and directories to be restructured.
-	if err := os.MkdirAll(filepath.Join(e.outputDir, "java_gapic_srcjar", "src", "main", "java", "com"), 0755); err != nil {
-		t.Fatal(err)
-	}
-	if err := os.WriteFile(filepath.Join(e.outputDir, "java_gapic_srcjar", "src", "main", "java", "com", "foo.java"), nil, 0644); err != nil {
-		t.Fatal(err)
-	}
-	if err := os.MkdirAll(filepath.Join(e.outputDir, "java_gapic_srcjar", "src", "test", "java", "com"), 0755); err != nil {
-		t.Fatal(err)
-	}
-	if err := os.WriteFile(filepath.Join(e.outputDir, "java_gapic_srcjar", "src", "test", "java", "com", "foo_test.java"), nil, 0644); err != nil {
-		t.Fatal(err)
-	}
-	if err := os.MkdirAll(filepath.Join(e.outputDir, "com"), 0755); err != nil {
-		t.Fatal(err)
-	}
-	if err := os.WriteFile(filepath.Join(e.outputDir, "com", "bar.proto"), nil, 0644); err != nil {
-		t.Fatal(err)
-	}
-	if err := os.MkdirAll(filepath.Join(e.outputDir, "java_gapic_srcjar", "samples", "snippets", "com"), 0755); err != nil {
-		t.Fatal(err)
-	}
-	if err := os.WriteFile(filepath.Join(e.outputDir, "java_gapic_srcjar", "samples", "snippets", "com", "baz.java"), nil, 0644); err != nil {
-		t.Fatal(err)
+
+	// 1. Setup: Create all the source directories and dummy files.
+	sourceFiles := map[string]string{
+		"gapic/src/main/java/com/google/foo.java":           "",
+		"gapic/src/test/java/com/google/foo_test.java":      "",
+		"proto/com/google/bar.proto":                        "",
+		"grpc/com/google/bar_grpc.java":                     "",
+		"gapic/samples/snippets/com/google/baz.java":        "",
+		"gapic/proto/src/main/java/com/google/resname.java": "",
+	}
+	for path, content := range sourceFiles {
+		fullPath := filepath.Join(e.outputDir, path)
+		if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil {
+			t.Fatalf("failed to create source directory for %s: %v", path, err)
+		}
+		if err := os.WriteFile(fullPath, []byte(content), 0644); err != nil {
+			t.Fatalf("failed to write source file for %s: %v", path, err)
+		}
 	}
 
+	// 2. Execute: Call the function under test.
 	if err := restructureOutput(e.outputDir, "my-library"); err != nil {
 		t.Fatalf("restructureOutput() failed: %v", err)
 	}
 
-	// Check that the files were moved to the correct locations.
-	if _, err := os.Stat(filepath.Join(e.outputDir, "google-cloud-my-library", "src", "main", "java", "com", "foo.java")); err != nil {
-		t.Errorf("file not moved to main: %v", err)
-	}
-	if _, err := os.Stat(filepath.Join(e.outputDir, "google-cloud-my-library", "src", "test", "java", "com", "foo_test.java")); err != nil {
-		t.Errorf("file not moved to test: %v", err)
+	// 3. Verify: Check that all files were moved to their expected destinations.
+	expectedFiles := []string{
+		"google-cloud-my-library/src/main/java/com/google/foo.java",
+		"google-cloud-my-library/src/test/java/com/google/foo_test.java",
+		"proto-google-cloud-my-library-v1/src/main/java/com/google/bar.proto",
+		"grpc-google-cloud-my-library-v1/src/main/java/com/google/bar_grpc.java",
+		"samples/snippets/com/google/baz.java",
+		"proto-google-cloud-my-library-v1/src/main/java/com/google/resname.java",
 	}
-	if _, err := os.Stat(filepath.Join(e.outputDir, "proto-google-cloud-my-library-v1", "src", "main", "java", "bar.proto")); err != nil {
-		t.Errorf("file not moved to proto: %v", err)
-	}
-	if _, err := os.Stat(filepath.Join(e.outputDir, "samples", "snippets", "com", "baz.java")); err != nil {
-		t.Errorf("file not moved to samples: %v", err)
+	for _, path := range expectedFiles {
+		fullPath := filepath.Join(e.outputDir, path)
+		if _, err := os.Stat(fullPath); err != nil {
+			t.Errorf("expected file not found at %s: %v", fullPath, err)
+		}
 	}
 }
 
@@ -502,7 +483,7 @@ func TestCleanupIntermediateFiles(t *testing.T) {
 	defer e.cleanup(t)
 
 	// Create a file that cannot be deleted.
-	protectedDir := filepath.Join(e.outputDir, "com")
+	protectedDir := filepath.Join(e.outputDir, "proto")
 	if err := os.Mkdir(protectedDir, 0755); err != nil {
 		t.Fatalf("failed to create protected dir: %v", err)
 	}
@@ -515,7 +496,12 @@ func TestCleanupIntermediateFiles(t *testing.T) {
 	}
 	defer os.Chmod(protectedDir, 0755) // Restore permissions for cleanup.
 
-	cleanupIntermediateFiles(e.outputDir)
+	outputConfig := &protoc.OutputConfig{
+		GAPICDir: filepath.Join(e.outputDir, "gapic"),
+		GRPCDir:  filepath.Join(e.outputDir, "grpc"),
+		ProtoDir: protectedDir,
+	}
+	cleanupIntermediateFiles(outputConfig)
 
 	if !strings.Contains(buf.String(), "failed to clean up intermediate file") {
 		t.Errorf("cleanupIntermediateFiles() should log an error on failure, but did not. Log: %s", buf.String())

internal/librariangen/protoc/protoc.go

@@ -19,7 +19,6 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
-
 )
 
 // ConfigProvider is an interface that describes the configuration needed
@@ -34,8 +33,15 @@ type ConfigProvider interface {
 	HasGAPIC() bool
 }
 
+// OutputConfig provides paths to directories to be used for protoc output.
+type OutputConfig struct {
+	GAPICDir string
+	GRPCDir  string
+	ProtoDir string
+}
+
 // Build constructs the full protoc command arguments for a given API.
-func Build(apiServiceDir string, config ConfigProvider, sourceDir, outputDir string) ([]string, error) {
+func Build(apiServiceDir string, config ConfigProvider, sourceDir string, outputConfig *OutputConfig) ([]string, error) {
 	// Gather all .proto files in the API's source directory.
 	entries, err := os.ReadDir(apiServiceDir)
 	if err != nil {
@@ -78,9 +84,12 @@ func Build(apiServiceDir string, config ConfigProvider, sourceDir, outputDir str
 		"--experimental_allow_proto3_optional",
 	}
 
-	args = append(args, fmt.Sprintf("--java_out=%s", outputDir))
+	args = append(args, fmt.Sprintf("--java_out=%s", outputConfig.ProtoDir))
+	if config.Transport() != "" && config.Transport() != "rest" {
+		args = append(args, fmt.Sprintf("--java_grpc_out=%s", outputConfig.GRPCDir))
+	}
 	if config.HasGAPIC() {
-		args = append(args, fmt.Sprintf("--java_gapic_out=metadata:%s", filepath.Join(outputDir, "java_gapic.zip")))
+		args = append(args, fmt.Sprintf("--java_gapic_out=metadata:%s", outputConfig.GAPICDir))
 
 		if len(gapicOpts) > 0 {
 			args = append(args, "--java_gapic_opt="+strings.Join(gapicOpts, ","))