feat(testing): Refactor and add validation to release init E2E test (#2298)

The primary motivations for this change were:
1. To add a new test case simulating a real-world Python library
release.
2. To add a new test case simulating a real-world Go library release.
3. To introduce a permanent contract test that validates the `librarian`
tool's output, specifically to prevent regressions of the bug described
in issue #2254, where commits were associated with the wrong package.

Part of #2254
Fixes #2289

Author: jinseopkim0

e2e_test.go

@@ -429,35 +429,64 @@ func TestRunGenerate_MultipleLibraries(t *testing.T) {
 func TestReleaseInit(t *testing.T) {
 	t.Parallel()
 	for _, test := range []struct {
-		name                string
-		initialRepoStateDir string
-		updatedState        string
-		wantChangelog       string
-		libraryID           string
-		push                bool
-		wantErr             bool
+		name        string
+		testDataDir string
+		libraryID   string
+		changePath  string
+		tagID       string
+		tagVersion  string
+		tagFormat   string
+		push        bool
+		wantErr     bool
 	}{
 		{
-			name:                "runs successfully without push",
-			initialRepoStateDir: "testdata/e2e/release/init/repo_init",
-			updatedState:        "testdata/e2e/release/init/updated-state.yaml",
-			wantChangelog:       "testdata/e2e/release/init/CHANGELOG.md",
-			libraryID:           "go-google-cloud-pubsub-v1",
+			name:        "release with multiple commits",
+			testDataDir: "testdata/e2e/release/init/multiple_commits",
+			libraryID:   "go-google-cloud-pubsub-v1",
+			changePath:  "google-cloud-pubsub/v1",
+			tagID:       "go-google-cloud-pubsub-v1",
+			tagVersion:  "1.0.0",
+			tagFormat:   "%s/v%s",
+		},
+		{
+			name:        "release with multiple commits with push",
+			testDataDir: "testdata/e2e/release/init/multiple_commits",
+			libraryID:   "go-google-cloud-pubsub-v1",
+			changePath:  "google-cloud-pubsub/v1",
+			tagID:       "go-google-cloud-pubsub-v1",
+			tagVersion:  "1.0.0",
+			tagFormat:   "%s/v%s",
+			push:        true,
+		},
+		{
+			name:        "release with multiple nested commits",
+			testDataDir: "testdata/e2e/release/init/multiple_nested_commits",
+			libraryID:   "python-google-cloud-video-live-stream-v1",
+			changePath:  "packages/google-cloud-video-live-stream",
+			tagID:       "python-google-cloud-video-live-stream-v1",
+			tagVersion:  "v1.12.0",
+			tagFormat:   "%s-%s", // Format for {id}-{version}
 		},
 		{
-			name:                "runs successfully with push",
-			initialRepoStateDir: "testdata/e2e/release/init/repo_init",
-			updatedState:        "testdata/e2e/release/init/updated-state.yaml",
-			wantChangelog:       "testdata/e2e/release/init/CHANGELOG.md",
-			libraryID:           "go-google-cloud-pubsub-v1",
-			push:                true, // Enable --push for this case
+			name:        "release with single commit",
+			testDataDir: "testdata/e2e/release/init/single_commit",
+			libraryID:   "dlp",
+			changePath:  "dlp",
+			tagID:       "dlp",
+			tagVersion:  "1.24.0",
+			tagFormat:   "%s/v%s",
 		},
 	} {
 		t.Run(test.name, func(t *testing.T) {
 			workRoot := t.TempDir()
 			repo := t.TempDir()
 
-			if err := initRepo(t, repo, test.initialRepoStateDir); err != nil {
+			initialRepoStateDir := filepath.Join(test.testDataDir, "repo_init")
+			updatedState := filepath.Join(test.testDataDir, "state.yaml")
+			wantChangelog := filepath.Join(test.testDataDir, "CHANGELOG.md")
+			commitMsgPath := filepath.Join(test.testDataDir, "commit_msg.txt")
+
+			if err := initRepo(t, repo, initialRepoStateDir); err != nil {
 				t.Fatalf("prepare test error = %v", err)
 			}
 
@@ -471,51 +500,26 @@ func TestReleaseInit(t *testing.T) {
 				runGit(t, repo, "remote", "set-url", "origin", bareRepoDir)
 			}
 
-			runGit(t, repo, "tag", "go-google-cloud-pubsub-v1-1.0.0")
+			// Dynamically create the tag based on the format string
+			tagName := fmt.Sprintf(test.tagFormat, test.tagID, test.tagVersion)
+			runGit(t, repo, "tag", tagName)
+
 			// Add a new commit to simulate a change.
-			newFilePath := filepath.Join(repo, "google-cloud-pubsub/v1", "new-file.txt")
+			newFilePath := filepath.Join(repo, test.changePath, "new-file.txt")
+			if err := os.MkdirAll(filepath.Dir(newFilePath), 0755); err != nil {
+				t.Fatalf("Failed to create directory for new file: %v", err)
+			}
 			if err := os.WriteFile(newFilePath, []byte("new file"), 0644); err != nil {
 				t.Fatal(err)
 			}
 			runGit(t, repo, "add", newFilePath)
-			commitMsg := `
-chore: Update generation configuration at Tue Aug 26 02:31:23 UTC 2025 (#11734)
-
-This pull request is generated with proto changes between
-[googleapis/googleapis@525c95a](https://github.com/googleapis/googleapis/commit/525c95a7a122ec2869ae06cd02fa5013819463f6)
-(exclusive) and
-[googleapis/googleapis@b738e78](https://github.com/googleapis/googleapis/commit/b738e78ed63effb7d199ed2d61c9e03291b6077f)
-(inclusive).
-
-BEGIN_COMMIT_OVERRIDE
-BEGIN_NESTED_COMMIT
-feat: [go-google-cloud-pubsub-v1] Support promptable voices by specifying a model name and a prompt
-feat: [go-google-cloud-pubsub-v1] Add enum value M4A to enum AudioEncoding
-docs: [go-google-cloud-pubsub-v1] A comment for method 'StreamingSynthesize' in service 'TextToSpeech' is changed
-docs: [go-google-cloud-pubsub-v1] A comment for enum value 'AUDIO_ENCODING_UNSPECIFIED' in enum 'AudioEncoding' is changed
-docs: [go-google-cloud-pubsub-v1] A comment for enum value 'OGG_OPUS' in enum 'AudioEncoding' is changed
-docs: [go-google-cloud-pubsub-v1] A comment for enum value 'PCM' in enum 'AudioEncoding' is changed
-docs: [go-google-cloud-pubsub-v1] A comment for field 'low_latency_journey_synthesis' in message '.google.cloud.texttospeech.v1beta1.AdvancedVoiceOptions' is changed
-docs: [go-google-cloud-pubsub-v1] A comment for enum value 'PHONETIC_ENCODING_IPA' in enum 'PhoneticEncoding' is changed
-docs: [go-google-cloud-pubsub-v1] A comment for enum value 'PHONETIC_ENCODING_X_SAMPA' in enum 'PhoneticEncoding' is changed
-docs: [go-google-cloud-pubsub-v1] A comment for field 'phrase' in message '.google.cloud.texttospeech.v1beta1.CustomPronunciationParams' is changed
-docs: [go-google-cloud-pubsub-v1] A comment for field 'pronunciations' in message '.google.cloud.texttospeech.v1beta1.CustomPronunciations' is changed
-docs: [go-google-cloud-pubsub-v1] A comment for message 'MultiSpeakerMarkup' is changed
-docs: [go-google-cloud-pubsub-v1] A comment for field 'custom_pronunciations' in message '.google.cloud.texttospeech.v1beta1.SynthesisInput' is changed
-docs: [go-google-cloud-pubsub-v1] A comment for field 'voice_clone' in message '.google.cloud.texttospeech.v1beta1.VoiceSelectionParams' is changed
-docs: [go-google-cloud-pubsub-v1] A comment for field 'speaking_rate' in message '.google.cloud.texttospeech.v1beta1.AudioConfig' is changed
-docs: [go-google-cloud-pubsub-v1] A comment for field 'audio_encoding' in message '.google.cloud.texttospeech.v1beta1.StreamingAudioConfig' is changed
-docs: [go-google-cloud-pubsub-v1] A comment for field 'text' in message '.google.cloud.texttospeech.v1beta1.StreamingSynthesisInput' is changed
-
-PiperOrigin-RevId: 799242210
-
-Source Link:
-[googleapis/googleapis@b738e78](https://github.com/googleapis/googleapis/commit/b738e78ed63effb7d199ed2d61c9e03291b6077f)
-END_NESTED_COMMIT
-END_COMMIT_OVERRIDE
-`
+			commitMsgBytes, err := os.ReadFile(commitMsgPath)
+			if err != nil {
+				t.Fatalf("Failed to read commit message file: %v", err)
+			}
+			commitMsg := string(commitMsgBytes)
 			runGit(t, repo, "commit", "-m", commitMsg)
-			runGit(t, repo, "log", "--oneline", "go-google-cloud-pubsub-v1-1.0.0..HEAD", "--", "google-cloud-pubsub/v1")
+			runGit(t, repo, "log", "--oneline", fmt.Sprintf("%s..HEAD", tagName), "--", test.changePath)
 
 			server := newMockGitHubServer(t, "release")
 			defer server.Close()
@@ -540,8 +544,7 @@ END_COMMIT_OVERRIDE
 			cmd.Env = append(cmd.Env, "LIBRARIAN_GITHUB_BASE_URL="+server.URL)
 			cmd.Stderr = os.Stderr
 			cmd.Stdout = os.Stdout
-			err := cmd.Run()
-			if err != nil {
+			if err := cmd.Run(); err != nil {
 				t.Fatalf("Failed to run release init: %v", err)
 			}
 
@@ -552,7 +555,7 @@ END_COMMIT_OVERRIDE
 			if err != nil {
 				t.Fatalf("Failed to read updated state.yaml from output directory: %v", err)
 			}
-			wantBytes, readErr := os.ReadFile(test.updatedState)
+			wantBytes, readErr := os.ReadFile(updatedState)
 			if readErr != nil {
 				t.Fatalf("Failed to read expected state for comparison: %v", readErr)
 			}
@@ -565,16 +568,17 @@ END_COMMIT_OVERRIDE
 				t.Fatalf("Failed to unmarshal expected state: %v", err)
 			}
 
-			if diff := cmp.Diff(wantState, gotState); diff != "" {
+			// Use cmpopts.IgnoreFields to ignore the dynamic commit hash.
+			if diff := cmp.Diff(wantState, gotState, cmpopts.IgnoreFields(config.LibraryState{}, "LastGeneratedCommit")); diff != "" {
 				t.Fatalf("Generated yaml mismatch (-want +got): %s", diff)
 			}
 
 			// Verify the CHANGELOG.md file content
-			gotChangelog, err := os.ReadFile(filepath.Join(outputDir, "google-cloud-pubsub/v1", "CHANGELOG.md"))
+			gotChangelog, err := os.ReadFile(filepath.Join(outputDir, test.changePath, "CHANGELOG.md"))
 			if err != nil {
 				t.Fatalf("Failed to read CHANGELOG.md from output directory: %v", err)
 			}
-			wantChangelogBytes, err := os.ReadFile(test.wantChangelog)
+			wantChangelogBytes, err := os.ReadFile(wantChangelog)
 			if err != nil {
 				t.Fatalf("Failed to read expected changelog for comparison: %v", err)
 			}

testdata/e2e/release/init/CHANGELOG.md renamed to testdata/e2e/release/init/multiple_commits/CHANGELOG.md

@@ -6,14 +6,14 @@
 - docs: [go-google-cloud-pubsub-v1] A comment for enum value 'AUDIO_ENCODING_UNSPECIFIED' in enum 'AudioEncoding' is changed
 - docs: [go-google-cloud-pubsub-v1] A comment for enum value 'OGG_OPUS' in enum 'AudioEncoding' is changed
 - docs: [go-google-cloud-pubsub-v1] A comment for enum value 'PCM' in enum 'AudioEncoding' is changed
-- docs: [go-google-cloud-pubsub-v1] A comment for field 'low_latency_journey_synthesis' in message '.google.cloud.texttospeech.v1beta1.AdvancedVoiceOptions' is changed
+- docs: [go-google-cloud-pubsub-v1] A comment for field 'low_latency_journey_synthesis' in message '.google.cloud.go-google-cloud-pubsub-v1.v1beta1.AdvancedVoiceOptions' is changed
 - docs: [go-google-cloud-pubsub-v1] A comment for enum value 'PHONETIC_ENCODING_IPA' in enum 'PhoneticEncoding' is changed
 - docs: [go-google-cloud-pubsub-v1] A comment for enum value 'PHONETIC_ENCODING_X_SAMPA' in enum 'PhoneticEncoding' is changed
-- docs: [go-google-cloud-pubsub-v1] A comment for field 'phrase' in message '.google.cloud.texttospeech.v1beta1.CustomPronunciationParams' is changed
-- docs: [go-google-cloud-pubsub-v1] A comment for field 'pronunciations' in message '.google.cloud.texttospeech.v1beta1.CustomPronunciations' is changed
+- docs: [go-google-cloud-pubsub-v1] A comment for field 'phrase' in message '.google.cloud.go-google-cloud-pubsub-v1.v1beta1.CustomPronunciationParams' is changed
+- docs: [go-google-cloud-pubsub-v1] A comment for field 'pronunciations' in message '.google.cloud.go-google-cloud-pubsub-v1.v1beta1.CustomPronunciations' is changed
 - docs: [go-google-cloud-pubsub-v1] A comment for message 'MultiSpeakerMarkup' is changed
-- docs: [go-google-cloud-pubsub-v1] A comment for field 'custom_pronunciations' in message '.google.cloud.texttospeech.v1beta1.SynthesisInput' is changed
-- docs: [go-google-cloud-pubsub-v1] A comment for field 'voice_clone' in message '.google.cloud.texttospeech.v1beta1.VoiceSelectionParams' is changed
-- docs: [go-google-cloud-pubsub-v1] A comment for field 'speaking_rate' in message '.google.cloud.texttospeech.v1beta1.AudioConfig' is changed
-- docs: [go-google-cloud-pubsub-v1] A comment for field 'audio_encoding' in message '.google.cloud.texttospeech.v1beta1.StreamingAudioConfig' is changed
-- docs: [go-google-cloud-pubsub-v1] A comment for field 'text' in message '.google.cloud.texttospeech.v1beta1.StreamingSynthesisInput' is changed
+- docs: [go-google-cloud-pubsub-v1] A comment for field 'custom_pronunciations' in message '.google.cloud.go-google-cloud-pubsub-v1.v1beta1.SynthesisInput' is changed
+- docs: [go-google-cloud-pubsub-v1] A comment for field 'voice_clone' in message '.google.cloud.go-google-cloud-pubsub-v1.v1beta1.VoiceSelectionParams' is changed
+- docs: [go-google-cloud-pubsub-v1] A comment for field 'speaking_rate' in message '.google.cloud.go-google-cloud-pubsub-v1.v1beta1.AudioConfig' is changed
+- docs: [go-google-cloud-pubsub-v1] A comment for field 'audio_encoding' in message '.google.cloud.go-google-cloud-pubsub-v1.v1beta1.StreamingAudioConfig' is changed
+- docs: [go-google-cloud-pubsub-v1] A comment for field 'text' in message '.google.cloud.go-google-cloud-pubsub-v1.v1beta1.StreamingSynthesisInput' is changed

testdata/e2e/release/init/multiple_commits/commit_msg.txt

@@ -0,0 +1,38 @@
+chore: Update generation configuration at Tue Aug 26 02:31:23 UTC 2025 (#11734)
+
+This pull request is generated with proto changes between
+[googleapis/googleapis@525c95a](https://github.com/googleapis/googleapis/commit/525c95a7a122ec2869ae06cd02fa5013819463f6)
+(exclusive) and
+[googleapis/googleapis@b738e78](https://github.com/googleapis/googleapis/commit/b738e78ed63effb7d199ed2d61c9e03291b6077f)
+(inclusive).
+
+Librarian Version: v0.0.0-20250918210716-c3c71af9310a
+Language Image:
+us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-prod/librarian-go:latest
+
+BEGIN_COMMIT_OVERRIDE
+BEGIN_NESTED_COMMIT
+feat: [go-google-cloud-pubsub-v1] Support promptable voices by specifying a model name and a prompt
+feat: [go-google-cloud-pubsub-v1] Add enum value M4A to enum AudioEncoding
+docs: [go-google-cloud-pubsub-v1] A comment for method 'StreamingSynthesize' in service 'TextToSpeech' is changed
+docs: [go-google-cloud-pubsub-v1] A comment for enum value 'AUDIO_ENCODING_UNSPECIFIED' in enum 'AudioEncoding' is changed
+docs: [go-google-cloud-pubsub-v1] A comment for enum value 'OGG_OPUS' in enum 'AudioEncoding' is changed
+docs: [go-google-cloud-pubsub-v1] A comment for enum value 'PCM' in enum 'AudioEncoding' is changed
+docs: [go-google-cloud-pubsub-v1] A comment for field 'low_latency_journey_synthesis' in message '.google.cloud.go-google-cloud-pubsub-v1.v1beta1.AdvancedVoiceOptions' is changed
+docs: [go-google-cloud-pubsub-v1] A comment for enum value 'PHONETIC_ENCODING_IPA' in enum 'PhoneticEncoding' is changed
+docs: [go-google-cloud-pubsub-v1] A comment for enum value 'PHONETIC_ENCODING_X_SAMPA' in enum 'PhoneticEncoding' is changed
+docs: [go-google-cloud-pubsub-v1] A comment for field 'phrase' in message '.google.cloud.go-google-cloud-pubsub-v1.v1beta1.CustomPronunciationParams' is changed
+docs: [go-google-cloud-pubsub-v1] A comment for field 'pronunciations' in message '.google.cloud.go-google-cloud-pubsub-v1.v1beta1.CustomPronunciations' is changed
+docs: [go-google-cloud-pubsub-v1] A comment for message 'MultiSpeakerMarkup' is changed
+docs: [go-google-cloud-pubsub-v1] A comment for field 'custom_pronunciations' in message '.google.cloud.go-google-cloud-pubsub-v1.v1beta1.SynthesisInput' is changed
+docs: [go-google-cloud-pubsub-v1] A comment for field 'voice_clone' in message '.google.cloud.go-google-cloud-pubsub-v1.v1beta1.VoiceSelectionParams' is changed
+docs: [go-google-cloud-pubsub-v1] A comment for field 'speaking_rate' in message '.google.cloud.go-google-cloud-pubsub-v1.v1beta1.AudioConfig' is changed
+docs: [go-google-cloud-pubsub-v1] A comment for field 'audio_encoding' in message '.google.cloud.go-google-cloud-pubsub-v1.v1beta1.StreamingAudioConfig' is changed
+docs: [go-google-cloud-pubsub-v1] A comment for field 'text' in message '.google.cloud.go-google-cloud-pubsub-v1.v1beta1.StreamingSynthesisInput' is changed
+
+PiperOrigin-RevId: 799242210
+
+Source Link:
+[googleapis/googleapis@b738e78](https://github.com/googleapis/googleapis/commit/b738e78ed63effb7d199ed2d61c9e03291b6077f)
+END_NESTED_COMMIT
+END_COMMIT_OVERRIDE

testdata/e2e/release/init/repo_init/.librarian/state.yaml renamed to testdata/e2e/release/init/multiple_commits/repo_init/.librarian/state.yaml

@@ -6,3 +6,4 @@ libraries:
       - path: google/cloud/pubsub/v1
     source_roots:
       - google-cloud-pubsub/v1
+    tag_format: "{id}/v{version}"

testdata/e2e/release/init/repo_init/google-cloud-pubsub/v1/dummy.go renamed to testdata/e2e/release/init/multiple_commits/repo_init/google-cloud-pubsub/v1/dummy.go

@@ -6,3 +6,4 @@ libraries:
       - path: google/cloud/pubsub/v1
     source_roots:
       - google-cloud-pubsub/v1
+    tag_format: "{id}/v{version}"

testdata/e2e/release/init/updated-state.yaml renamed to testdata/e2e/release/init/multiple_commits/state.yaml

@@ -0,0 +1,9 @@
+## 1.13.0
+
+- feat: [python-google-cloud-video-live-stream-v1] Added H.265 (HEVC) codec support
+- feat: [python-google-cloud-video-live-stream-v1] Added UHD (4k) resolution support
+- feat: [python-google-cloud-video-live-stream-v1] Added Auto Transcription support
+- feat: [python-google-cloud-video-live-stream-v1] Added StartDistribution/StopDistribution methods and Distribution/DistributionStream messages used for distributing live streams to external RTMP/SRT endpoints
+- feat: [python-google-cloud-video-live-stream-v1] Added PreviewInput method used for the low latency input monitoring
+- feat: [python-google-cloud-video-live-stream-v1] Added UpdateEncryptions event to perform key rotation without restarting a channel
+- docs: [python-google-cloud-video-live-stream-v1] Update requirements of resource ID fields to be more clear