feat: bring legacy-librarian into main (#3564)

This is a one-by-one manual cherry pick of all of the commits that are
ahead of main on legacy-librarian. Because of file renames I could not
use a simple git cherry-pick for the operations so I decided to do it
all by hand, even if there where no conflicts.

The one set of files I did not update with these commits were the
`infra/prod/*.yaml` files. These files should point to latest once we
cut a new release and not some old commit from another branch.

This means before we cut over the jobs to run off of main we will want
to do a tagged release.

Updates: #3472

Author: codyoss

infra/imagebuilders/cli/Dockerfile

@@ -12,7 +12,28 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM golang:1.25.3 AS build
+# This stage builds the librariangen binary using the MOSS-compliant base image.
+FROM marketplace.gcr.io/google/debian12@sha256:326ccf35aa72f7cc8cbd25b69e819a81c5fd9ed675c6c9ffb51f3214a64f23cf AS builder
+
+# Set environment variables for tool versions for easy updates.
+ENV GO_VERSION=1.25.3
+
+# Install build dependencies.
+RUN apt-get update && \
+    apt-get install -y \
+    build-essential \
+    ca-certificates \
+    curl \
+    git \
+    wget && \
+    rm -rf /var/lib/apt/lists/*
+
+# Install the specific Go version required for compatibility.
+RUN wget https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz -O go.tar.gz && \
+    tar -C /usr/local -xzf go.tar.gz && \
+    rm go.tar.gz
+ENV PATH=/usr/local/go/bin:$PATH
+RUN go version
 
 WORKDIR /src
 
@@ -27,7 +48,7 @@ RUN CGO_ENABLED=0 GOOS=linux go build ./cmd/librarian
 # while in Docker. Note that for this to work, *this*
 # docker image should be run with
 #  -v /var/run/docker.sock:/var/run/docker.sock
-FROM golang:1.25.3
+FROM marketplace.gcr.io/google/debian12@sha256:326ccf35aa72f7cc8cbd25b69e819a81c5fd9ed675c6c9ffb51f3214a64f23cf
 WORKDIR /app
 
 # From https://docs.docker.com/engine/install/debian/
@@ -58,9 +79,9 @@ RUN echo \
 # https://docs.docker.com/engine/release-notes/29/#2900
 RUN apt update && \
     apt-get -y install \
-    docker-ce=5:28.5.2-1~debian.13~trixie \
-    docker-ce-cli=5:28.5.2-1~debian.13~trixie \
-    docker-ce-rootless-extras=5:28.5.2-1~debian.13~trixie && \
+    docker-ce=5:27.1.1-1~debian.12~bookworm \
+    docker-ce-cli=5:27.1.1-1~debian.12~bookworm \
+    docker-ce-rootless-extras=5:27.1.1-1~debian.12~bookworm && \
     rm -rf /var/lib/apt/lists/*
 
 # Add the Google Cloud SDK distribution URI as a package source
@@ -72,5 +93,5 @@ RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.
 # Install the gcloud CLI
 RUN apt-get update && apt-get install -y google-cloud-sdk
 
-COPY --from=build /src/librarian .
+COPY --from=builder /src/librarian .
 ENTRYPOINT ["/app/librarian"]

infra/imagebuilders/cloudbuild-exitgate-deprecated.yaml

@@ -12,7 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM marketplace.gcr.io/google/ubuntu2404 AS builder
+# This stage builds the librariangen binary using the MOSS-compliant base image.
+FROM marketplace.gcr.io/google/debian12@sha256:326ccf35aa72f7cc8cbd25b69e819a81c5fd9ed675c6c9ffb51f3214a64f23cf AS builder
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \

infra/imagebuilders/container/Dockerfile

@@ -14,18 +14,18 @@
 # limitations under the License.
 
 echo "in release container"
-cat /librarian/release-init-request.json
+cat /librarian/release-stage-request.json
 
 ls -al /librarian
 ls -al /repo
 ls -al /output
 
-new_version=$(jq -r '.libraries[0].version' /librarian/release-init-request.json)
+new_version=$(jq -r '.libraries[0].version' /librarian/release-stage-request.json)
 echo "release version: ${new_version}"
-mkdir /output/internal/
+mkdir -p /output/internal/cli/
 echo "${new_version}" > /output/internal/cli/version.txt
 
 ls -al /output
 
 echo "writing empty response"
-echo "{}" > /librarian/release-init-response.json
+echo "{}" > /librarian/release-stage-response.json

infra/imagebuilders/container/run.sh

@@ -11,7 +11,28 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-FROM golang:1.25.3 AS build
+
+# This stage builds the librariangen binary using the MOSS-compliant base image.
+FROM marketplace.gcr.io/google/debian12@sha256:326ccf35aa72f7cc8cbd25b69e819a81c5fd9ed675c6c9ffb51f3214a64f23cf AS builder
+
+# Set environment variables for tool versions for easy updates.
+ENV GO_VERSION=1.25.3
+
+# Install build dependencies.
+RUN apt-get update && \
+    apt-get install -y \
+    build-essential \
+    ca-certificates \
+    curl \
+    wget && \
+    rm -rf /var/lib/apt/lists/*
+
+# Install the specific Go version required for compatibility.
+RUN wget https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz -O go.tar.gz && \
+    tar -C /usr/local -xzf go.tar.gz && \
+    rm go.tar.gz
+ENV PATH=/usr/local/go/bin:$PATH
+RUN go version
 
 WORKDIR /src
 
@@ -22,9 +43,8 @@ RUN go mod download
 COPY . .
 RUN CGO_ENABLED=0 GOOS=linux go build ./cmd/automation
 
-FROM golang:1.25.3
-
+FROM marketplace.gcr.io/google/debian12@sha256:326ccf35aa72f7cc8cbd25b69e819a81c5fd9ed675c6c9ffb51f3214a64f23cf
 WORKDIR /app
 
-COPY --from=build /src/automation .
+COPY --from=builder /src/automation .
 ENTRYPOINT [ "/app/automation" ]

infra/imagebuilders/dispatcher/Dockerfile

@@ -72,4 +72,5 @@ availableSecrets:
       env: 'LIBRARIAN_GITHUB_TOKEN'
 options:
   logging: CLOUD_LOGGING_ONLY
+  machineType: 'E2_HIGHCPU_8'
 timeout: 10h

infra/prod/generate.yaml

@@ -0,0 +1,58 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+steps:
+  - id: get-librarian-image-sha
+    name: bash
+    args:
+      - '-c'
+      - |
+        wget https://github.com/mikefarah/yq/releases/download/v4.48.2/yq_linux_amd64 -O /usr/local/bin/yq
+        chmod +x /usr/local/bin/yq
+        yq -r '.librarian-image-sha' internal/automation/prod/repositories.yaml > /workspace/librarian_image_sha.txt
+        cat /workspace/librarian_image_sha.txt
+  - id: test-generate
+    name: gcr.io/cloud-builders/docker
+    entrypoint: bash
+    waitFor: ['get-librarian-image-sha']
+    args:
+      - -c
+      - |
+        docker run \
+          --rm \
+          us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-prod/librarian@$(cat /workspace/librarian_image_sha.txt) \
+          generate --help
+  - id: test-release-stage
+    name: gcr.io/cloud-builders/docker
+    entrypoint: bash
+    waitFor: ['get-librarian-image-sha']
+    args:
+      - -c
+      - |
+        docker run \
+          --rm \
+          us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-prod/librarian@$(cat /workspace/librarian_image_sha.txt) \
+          release stage --help
+  - id: test-release-tag
+    name: gcr.io/cloud-builders/docker
+    entrypoint: bash
+    waitFor: ['get-librarian-image-sha']
+    args:
+      - -c
+      - |
+        docker run \
+          --rm \
+          us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-prod/librarian@$(cat /workspace/librarian_image_sha.txt) \
+          release tag --help
+options:
+  logging: CLOUD_LOGGING_ONLY

infra/test/librarian-image-test.yaml

@@ -1,4 +1,4 @@
-librarian-image-sha: sha256:8ae52838e466022dd701cab8c3f002cb727b3e0e5688864b3d0d68f645541035
+librarian-image-sha: sha256:41df234d423cfa86db99d1610f4a463d3150879b5fbb5fe3faeaf31a2a731d3e
 repositories:
   - name: "gapic-generator-go"
     full-name: https://github.com/googleapis/gapic-generator-go
@@ -49,9 +49,9 @@ repositories:
     github-token-secret-name: "google-resumable-media-python-github-token"
     supported-commands:
       - publish-release
-  - name: "python-aiplatform"
-    full-name: https://github.com/googleapis/python-aiplatform
-    github-token-secret-name: "python-aiplatform-github-token"
+  - name: "proto-plus-python"
+    full-name: https://github.com/googleapis/proto-plus-python
+    github-token-secret-name: "proto-plus-python-github-token"
     supported-commands:
       - publish-release
   - name: "python-api-core"
@@ -84,11 +84,21 @@ repositories:
     github-token-secret-name: "python-bigquery-sqlalchemy-github-token"
     supported-commands:
       - publish-release
+  - name: "python-bigtable"
+    full-name: https://github.com/googleapis/python-bigtable
+    github-token-secret-name: "python-bigtable-github-token"
+    supported-commands:
+      - publish-release
   - name: "python-cloud-core"
     full-name: https://github.com/googleapis/python-cloud-core
     github-token-secret-name: "python-cloud-core-github-token"
     supported-commands:
       - publish-release
+  - name: "python-crc32c"
+    full-name: https://github.com/googleapis/python-crc32c
+    github-token-secret-name: "python-crc32c-github-token"
+    supported-commands:
+      - publish-release
   - name: "python-datastore"
     full-name: https://github.com/googleapis/python-datastore
     github-token-secret-name: "python-datastore-github-token"
@@ -114,6 +124,11 @@ repositories:
     github-token-secret-name: "python-firestore-github-token"
     supported-commands:
       - publish-release
+  - name: "python-logging"
+    full-name: https://github.com/googleapis/python-logging
+    github-token-secret-name: "python-logging-github-token"
+    supported-commands:
+      - publish-release
   - name: "python-ndb"
     full-name: https://github.com/googleapis/python-ndb
     github-token-secret-name: "python-ndb-github-token"
@@ -124,11 +139,21 @@ repositories:
     github-token-secret-name: "python-pubsub-github-token"
     supported-commands:
       - publish-release
+  - name: "python-pubsublite"
+    full-name: https://github.com/googleapis/python-pubsublite
+    github-token-secret-name: "python-pubsublite-github-token"
+    supported-commands:
+      - publish-release
   - name: "python-runtimeconfig"
     full-name: https://github.com/googleapis/python-runtimeconfig
     github-token-secret-name: "python-runtimeconfig-github-token"
     supported-commands:
       - publish-release
+  - name: "python-spanner"
+    full-name: https://github.com/googleapis/python-spanner
+    github-token-secret-name: "python-spanner-github-token"
+    supported-commands:
+      - publish-release
   - name: "python-spanner-django"
     full-name: https://github.com/googleapis/python-spanner-django
     github-token-secret-name: "python-spanner-django-github-token"

internal/legacylibrarian/legacyautomation/prod/repositories.yaml

@@ -168,7 +168,7 @@ func (c *Commit) IsBulkCommit() bool {
 
 var (
 	libraryIDRegex = regexp.MustCompile(`^[a-zA-Z0-9/._-]+$`)
-	semverRegex    = regexp.MustCompile(`^v?\d+\.\d+\.\d+$`)
+	semverRegex    = regexp.MustCompile(`^v?\d+\.\d+\.\d+(?:-([a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)*))?$`)
 	hexRegex       = regexp.MustCompile("^[a-fA-F0-9]+$")
 	tagFormatRegex = regexp.MustCompile(`{[^{}]*}`)
 )

internal/legacylibrarian/legacyconfig/state.go

@@ -374,15 +374,20 @@ func getDirectoryFilenames(dir string) ([]string, error) {
 // It uses the GitHub client to create a PR with the specified branch, title, and
 // description to the repository.
 func commitAndPush(ctx context.Context, info *commitInfo) error {
+	repo := info.languageRepo
 	if !info.push && !info.commit {
 		slog.Info("push flag and commit flag are not specified, skipping committing")
+		isClean, err := repo.IsClean()
+		if err != nil {
+			return fmt.Errorf("failed to check if repo is clean: %w", err)
+		}
+		if isClean {
+			slog.Info("no changes to commit, skipping pull request body creation.")
+			return nil
+		}
 		return writePRBody(info)
 	}
 
-	repo := info.languageRepo
-	if err := repo.AddAll(); err != nil {
-		return fmt.Errorf("failed to add all files to git: %w", err)
-	}
 	isClean, err := repo.IsClean()
 	if err != nil {
 		return fmt.Errorf("failed to check if repo is clean: %w", err)
@@ -393,6 +398,10 @@ func commitAndPush(ctx context.Context, info *commitInfo) error {
 		return nil
 	}
 
+	if err := repo.AddAll(); err != nil {
+		return fmt.Errorf("failed to add all files to git: %w", err)
+	}
+
 	datetimeNow := formatTimestamp(time.Now())
 	branch := fmt.Sprintf("librarian-%s", datetimeNow)
 	if err := repo.CreateBranchAndCheckout(branch); err != nil {