Merge branch 'main' into feat/analyze-pom

Author: JoeWang1127

.cloudbuild/library_generation/library_generation.Dockerfile

@@ -50,11 +50,9 @@ RUN chmod a+x compile-x86_64-alpine-linux.sh
 RUN sh compile-x86_64-alpine-linux.sh
 
 # python:3.12.7-alpine3.20
-FROM docker.io/library/python@sha256:38e179a0f0436c97ecc76bcd378d7293ab3ee79e4b8c440fdc7113670cb6e204 as final
+FROM docker.io/library/python@sha256:5049c050bdc68575a10bcb1885baa0689b6c15152d8a56a7e399fb49f783bf98 as final
 
-
-
-ARG OWLBOT_CLI_COMMITTISH=38fe6f89a2339ee75c77739b31b371f601b01bb3
+ARG OWLBOT_CLI_COMMITTISH=ab222d9a20bb27586433caedc70f049b7853db7e
 ARG PROTOC_VERSION=25.5
 ARG GRPC_VERSION=1.68.1
 ARG JAVA_FORMAT_VERSION=1.7
@@ -90,24 +88,23 @@ WORKDIR /protoc
 RUN source /src/library_generation/utils/utilities.sh \
 	&& download_protoc "${PROTOC_VERSION}" "${OS_ARCHITECTURE}"
 # we indicate protoc is available in the container via env vars
-ENV DOCKER_PROTOC_LOCATION=/protoc
+ENV DOCKER_PROTOC_LOCATION=/protoc/bin
 ENV DOCKER_PROTOC_VERSION="${PROTOC_VERSION}"
 
 # install grpc
 WORKDIR /grpc
 RUN source /src/library_generation/utils/utilities.sh \
 	&& download_grpc_plugin "${GRPC_VERSION}" "${OS_ARCHITECTURE}"
 # similar to protoc, we indicate grpc is available in the container via env vars
-ENV DOCKER_GRPC_LOCATION="/grpc/protoc-gen-grpc-java-${GRPC_VERSION}-${OS_ARCHITECTURE}.exe"
-ENV DOCKER_GRPC_VERSION="${GRPC_VERSION}"
-
+ENV DOCKER_GRPC_LOCATION="/grpc/protoc-gen-grpc-java.exe"
 
 # Here we transfer gapic-generator-java from the previous stage.
 # Note that the destination is a well-known location that will be assumed at runtime
 # We hard-code the location string to avoid making it configurable (via ARG) as
 # well as to avoid it making it overridable at runtime (via ENV).
 COPY --from=ggj-build "/sdk-platform-java/gapic-generator-java.jar" "${HOME}/.library_generation/gapic-generator-java.jar"
 RUN chmod 755 "${HOME}/.library_generation/gapic-generator-java.jar"
+ENV GAPIC_GENERATOR_LOCATION="${HOME}/.library_generation/gapic-generator-java.jar"
 
 RUN python -m pip install --upgrade pip
 
@@ -126,11 +123,13 @@ RUN git checkout "${OWLBOT_CLI_COMMITTISH}"
 RUN npm i && npm run compile && npm link
 RUN owl-bot copy-code --version
 RUN chmod o+rx $(which owl-bot)
+RUN apk del -r npm && apk cache clean
 
 # download the Java formatter
 ADD https://maven-central.storage-download.googleapis.com/maven2/com/google/googlejavaformat/google-java-format/${JAVA_FORMAT_VERSION}/google-java-format-${JAVA_FORMAT_VERSION}-all-deps.jar \
   "${HOME}"/.library_generation/google-java-format.jar
 RUN chmod 755 "${HOME}"/.library_generation/google-java-format.jar
+ENV JAVA_FORMATTER_LOCATION="${HOME}/.library_generation/google-java-format.jar"
 
 # allow users to access the script folders
 RUN chmod -R o+rx /src

.github/workflows/hermetic_library_generation.yaml

@@ -51,7 +51,7 @@ jobs:
       run: |
         GENERATOR_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout -pl gapic-generator-java)
         echo "GENERATOR_VERSION=${GENERATOR_VERSION}" >> "$GITHUB_ENV"
-        docker build \
+        DOCKER_BUILDKIT=1 docker build \
           -f .cloudbuild/library_generation/library_generation.Dockerfile \
           -t gcr.io/cloud-devrel-public-resources/java-library-generation:"${GENERATOR_VERSION}" \
           .

.github/workflows/verify_library_generation.yaml

@@ -27,7 +27,7 @@ jobs:
           git checkout -b "${head_ref}" fork/${head_ref}
           changed_directories="$(git diff --name-only "fork/${head_ref}" "origin/${base_ref}")"
         fi
-        if [[ ${changed_directories} =~ "hermetic_build/" ]]; then
+        if [[ ${changed_directories} =~ "hermetic_build/" ]] || [[ ${changed_directories} =~ ".cloudbuild/library_generation/" ]]; then
           echo "should_run=true" >> $GITHUB_OUTPUT
         else
           echo "should_run=false" >> $GITHUB_OUTPUT

gax-java/dependencies.properties

@@ -35,8 +35,10 @@ version.io_grpc=1.68.1
 # It should be constructed the following way:
 #   1) Take full artifact id (including the group and classifier (if any) portions) and remove version portion.
 #   2) Replace all characters which are neither alphabetic nor digits with the underscore ('_') character
+# {x-version-update-start:proto-google-common-protos:current}
 maven.com_google_api_grpc_proto_google_common_protos=com.google.api.grpc:proto-google-common-protos:2.46.0
 maven.com_google_api_grpc_grpc_google_common_protos=com.google.api.grpc:grpc-google-common-protos:2.46.0
+# {x-version-update-end}
 maven.com_google_auth_google_auth_library_oauth2_http=com.google.auth:google-auth-library-oauth2-http:1.30.0
 maven.com_google_auth_google_auth_library_credentials=com.google.auth:google-auth-library-credentials:1.30.0
 maven.io_opentelemetry_opentelemetry_api=io.opentelemetry:opentelemetry-api:1.42.1
@@ -67,10 +69,14 @@ maven.com_google_errorprone_error_prone_annotations=com.google.errorprone:error_
 maven.com_google_j2objc_j2objc_annotations=com.google.j2objc:j2objc-annotations:2.8
 maven.com_google_auto_value_auto_value=com.google.auto.value:auto-value:1.11.0
 maven.com_google_auto_value_auto_value_annotations=com.google.auto.value:auto-value-annotations:1.11.0
+# {x-version-update-start:api-common:current}
 maven.com_google_api_api_common=com.google.api:api-common:2.38.0
+# {x-version-update-end}
 maven.org_threeten_threetenbp=org.threeten:threetenbp:1.7.0
+# {x-version-update-start:proto-google-iam-v1:current}
 maven.com_google_api_grpc_grpc_google_iam_v1=com.google.api.grpc:grpc-google-iam-v1:1.41.0
 maven.com_google_api_grpc_proto_google_iam_v1=com.google.api.grpc:proto-google-iam-v1:1.41.0
+# {x-version-update-end}
 maven.com_google_http_client_google_http_client=com.google.http-client:google-http-client:1.45.0
 maven.com_google_http_client_google_http_client_gson=com.google.http-client:google-http-client-gson:1.45.0
 maven.org_codehaus_mojo_animal_sniffer_annotations=org.codehaus.mojo:animal-sniffer-annotations:1.24

hermetic_build/DEVELOPMENT.md

@@ -82,6 +82,21 @@ as per [POSIX env var definition](https://pubs.opengroup.org/onlinepubs/96999197
    mv /path/to/jar "${HOME}/.library_generation/gapic-generator-java.jar"
    ```
 
+#### Put the protobuf compiler in its well-known location
+1. Download protobuf compiler from [GitHub releases](https://github.com/protocolbuffers/protobuf/releases).
+2. Move the folder into its well-know location.
+
+   ```shell
+   unzip /path/to/zipfile -d "${HOME}/.library_generation/"
+   ```
+#### Put the GRPC plugin in its well-known location
+1. Download GRPC plugin from [Maven Central](https://central.sonatype.com/artifact/io.grpc/protoc-gen-grpc-java/versions).
+2. Move the folder into its well-know location.
+
+   ```shell
+   mv /path/to/protoc-gen-grpc-java.exe "${HOME}/.library_generation/protoc-gen-grpc-java.exe"
+   ```
+
 #### Put the java formatter jar in its well-known location
 
 1. Download google-java-format-{version}-all-deps.jar from [Maven Central](https://central.sonatype.com/artifact/com.google.googlejavaformat/google-java-format)
@@ -197,11 +212,6 @@ python hermetic_build/library_generation/cli/entry_point.py generate \
         --repository-path=/workspace \
         --api-definitions-path=/workspace/apis
    ```
-   Note that if you specify the generator version using environment variable,
-   `-e GENERATOR_VERSION="${LOCAL_GENERATOR_VERSION}"` in the above example,
-   you should not set `gapic_generator_version` and `protoc_version` in the
-   generation configuration because values in the generation configuration will
-   take precedence.
 
 # Debug the library generation container
 If you are working on changing the way the containers are created, you may want

hermetic_build/README.md

@@ -7,10 +7,6 @@ Running the docker image built from `hermetic_build/library_generation`
 directory, you can generate a repository containing GAPIC client libraries (a
 monorepo, for example, google-cloud-java) from a configuration file.
 
-Instead of running the docker image, if you prefer running the underlying python
-scripts directly, please refer to the [development guide](DEVELOPMENT.md#run-the-script)
-for additional instructions.
-
 ## Environment
 
 - OS: Linux
@@ -124,13 +120,11 @@ The repository level parameters define the version of API definition (proto)
 and tools.
 They are shared by library level parameters.
 
-| Name                    | Required | Notes                                        |
-|:------------------------|:--------:|:---------------------------------------------|
-| gapic_generator_version |    No    | set through env variable if not specified    |
-| protoc_version          |    No    | inferred from the generator if not specified |
-| grpc_version            |    No    | inferred from the generator if not specified |
-| googleapis_commitish    |   Yes    |                                              |
-| libraries_bom_version   |    No    | empty string if not specified                |
+| Name                    | Required | Notes                                     |
+|:------------------------|:--------:|:------------------------------------------|
+| gapic_generator_version |    No    | set through env variable if not specified |
+| googleapis_commitish    |   Yes    |                                           |
+| libraries_bom_version   |    No    | empty string if not specified             |
 
 ### Library level parameters
 
@@ -178,7 +172,6 @@ The GAPIC level parameters define how to generate a GAPIC library.
 
 ```yaml
 gapic_generator_version: 2.34.0
-protoc_version: 25.2
 googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026
 libraries_bom_version: 26.37.0
 libraries:

hermetic_build/common/model/generation_config.py

@@ -39,8 +39,6 @@ def __init__(
         libraries: list[LibraryConfig],
         gapic_generator_version: Optional[str] = None,
         libraries_bom_version: Optional[str] = None,
-        grpc_version: Optional[str] = None,
-        protoc_version: Optional[str] = None,
     ):
         self.googleapis_commitish = googleapis_commitish
         self.libraries_bom_version = (
@@ -50,8 +48,6 @@ def __init__(
             gapic_generator_version
         )
         self.libraries = libraries
-        self.grpc_version = grpc_version
-        self.protoc_version = protoc_version
         # explicit set to None so that we can compute the
         # value in getter.
         self.__contains_common_protos = None
@@ -172,8 +168,6 @@ def from_yaml(path_to_yaml: str) -> GenerationConfig:
             config, "googleapis_commitish", REPO_LEVEL_PARAMETER
         ),
         gapic_generator_version=__optional(config, GAPIC_GENERATOR_VERSION, None),
-        grpc_version=__optional(config, "grpc_version", None),
-        protoc_version=__optional(config, "protoc_version", None),
         libraries_bom_version=__optional(config, LIBRARIES_BOM_VERSION, None),
         libraries=parsed_libraries,
     )

hermetic_build/common/tests/model/config_change_unit_tests.py

@@ -309,8 +309,6 @@ def __get_a_gen_config(
         return GenerationConfig(
             gapic_generator_version="",
             googleapis_commitish=googleapis_commitish,
-            grpc_version="",
-            protoc_version="",
             libraries=libraries,
         )
 

hermetic_build/common/tests/model/generation_config_unit_tests.py

@@ -74,7 +74,6 @@ def test_generation_config_set_generator_version_from_env(self):
     def test_from_yaml_succeeds(self):
         config = from_yaml(f"{test_config_dir}/generation_config.yaml")
         self.assertEqual("2.34.0", config.gapic_generator_version)
-        self.assertEqual(25.2, config.protoc_version)
         self.assertEqual(
             "1a45bf7393b52407188c82e63101db7dc9c72026", config.googleapis_commitish
         )

hermetic_build/common/tests/resources/test-config/generation_config.yaml

@@ -1,5 +1,4 @@
 gapic_generator_version: 2.34.0
-protoc_version: 25.2
 googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026
 libraries_bom_version: 26.37.0
 libraries: