Merge branch 'main' into use-graalvm-ci-project

Author: diegomarquezp

.cloudbuild/library_generation/cloudbuild-library-generation-push-prod.yaml

@@ -30,6 +30,8 @@ steps:
       "--file", ".cloudbuild/library_generation/library_generation.Dockerfile", "."]
     id: library-generation-build
     waitFor: ["-"]
+    env: 
+      - 'DOCKER_BUILDKIT=1'
 
 options:
   logging: CLOUD_LOGGING_ONLY

.cloudbuild/library_generation/cloudbuild-library-generation-push.yaml

@@ -30,6 +30,8 @@ steps:
       "--file", ".cloudbuild/library_generation/library_generation.Dockerfile", "."]
     id: library-generation-build
     waitFor: ["-"]
+    env: 
+      - 'DOCKER_BUILDKIT=1'
 
 options:
   logging: CLOUD_LOGGING_ONLY

.github/scripts/hermetic_library_generation.sh

@@ -81,9 +81,6 @@ git checkout "${current_branch}"
 # copy generation configuration from target branch to current branch.
 git show "${target_branch}":"${generation_config}" > "${baseline_generation_config}"
 
-# get .m2 folder so it's mapped into the docker container
-m2_folder=$(dirname "$(mvn help:evaluate -Dexpression=settings.localRepository -q -DforceStdout)")
-
 # download api definitions from googleapis repository
 googleapis_commitish=$(grep googleapis_commitish "${generation_config}" | cut -d ":" -f 2 | xargs)
 api_def_dir=$(mktemp -d)

gax-java/gax/src/main/java/com/google/api/gax/core/GaxProperties.java

@@ -49,7 +49,7 @@ public class GaxProperties {
   private static final String GAX_VERSION = getLibraryVersion(GaxProperties.class, "version.gax");
   private static final String JAVA_VERSION = getRuntimeVersion();
   private static final String PROTOBUF_VERSION =
-      getBundleVersion(Any.class).orElse(DEFAULT_VERSION);
+      getProtobufVersion(Any.class, "com.google.protobuf.RuntimeVersion");;
 
   private GaxProperties() {}
 
@@ -148,4 +148,29 @@ static Optional<String> getBundleVersion(Class<?> clazz) {
       return Optional.empty();
     }
   }
+
+  /**
+   * Returns the Protobuf runtime version as reported by com.google.protobuf.RuntimeVersion, if
+   * class is available, otherwise by reading from MANIFEST file. If niether option is available
+   * defaults to protobuf version 3 as RuntimeVersion class is available in protobuf version 4+
+   */
+  @VisibleForTesting
+  static String getProtobufVersion(Class clazz, String protobufRuntimeVersionClassName) {
+    try {
+      Class<?> protobufRuntimeVersionClass = Class.forName(protobufRuntimeVersionClassName);
+      return protobufRuntimeVersionClass.getField("MAJOR").get(null)
+          + "."
+          + protobufRuntimeVersionClass.getField("MINOR").get(null)
+          + "."
+          + protobufRuntimeVersionClass.getField("PATCH").get(null);
+    } catch (ClassNotFoundException
+        | NoSuchFieldException
+        | IllegalAccessException
+        | SecurityException
+        | NullPointerException e) {
+      // If manifest file is not available default to protobuf generic version 3 as we know
+      // RuntimeVersion class is available in protobuf jar 4+.
+      return getBundleVersion(clazz).orElse("3");
+    }
+  }
 }

gax-java/gax/src/main/java/com/google/api/gax/rpc/ApiClientHeaderProvider.java

@@ -88,7 +88,7 @@ private static String checkAndAppendProtobufVersionIfNecessary(
     // TODO(b/366417603): appending protobuf version to existing client library token until resolved
     Pattern pattern = Pattern.compile("(gccl|gapic)\\S*");
     Matcher matcher = pattern.matcher(apiClientHeaderValue);
-    if (matcher.find()) {
+    if (matcher.find() && GaxProperties.getProtobufVersion() != null) {
       return apiClientHeaderValue.substring(0, matcher.end())
           + "--"
           + PROTOBUF_HEADER_VERSION_KEY

gax-java/gax/src/main/resources/META-INF/native-image/com.google.api/gax/reflect-config.json

@@ -0,0 +1,10 @@
+[
+  {
+    "name": "com.google.protobuf.RuntimeVersion",
+    "fields" : [
+      { "name" : "MAJOR" },
+      { "name" : "MINOR" },
+      { "name" : "PATCH" }
+    ]
+  }
+]

gax-java/gax/src/test/java/com/google/api/gax/core/GaxPropertiesTest.java

@@ -35,6 +35,7 @@
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import com.google.common.base.Strings;
+import com.google.protobuf.Any;
 import java.io.IOException;
 import java.util.Optional;
 import java.util.regex.Pattern;
@@ -160,12 +161,8 @@ void testGetJavaRuntimeInfo_nullJavaVersion() {
 
   @Test
   public void testGetProtobufVersion() throws IOException {
-    Version version = readVersion(GaxProperties.getProtobufVersion());
-
-    assertTrue(version.major >= 3);
-    if (version.major == 3) {
-      assertTrue(version.minor >= 25);
-    }
+    assertTrue(
+        Pattern.compile("^\\d+\\.\\d+\\.\\d+").matcher(GaxProperties.getProtobufVersion()).find());
   }
 
   @Test
@@ -175,6 +172,36 @@ public void testGetBundleVersion_noManifestFile() throws IOException {
     assertFalse(version.isPresent());
   }
 
+  @Test
+  void testGetProtobufVersion_success() {
+    String version =
+        GaxProperties.getProtobufVersion(
+            Any.class, "com.google.api.gax.core.GaxPropertiesTest$RuntimeVersion");
+
+    assertEquals("3.13.6", version);
+  }
+
+  @Test
+  void testGetProtobufVersion_classNotFoundException() throws Exception {
+    String version = GaxProperties.getProtobufVersion(Any.class, "foo.NonExistantClass");
+
+    assertTrue(Pattern.compile("^\\d+\\.\\d+\\.\\d+").matcher(version).find());
+  }
+
+  @Test
+  void testgetProtobufVersion_noSuchFieldException() throws Exception {
+    String version = GaxProperties.getProtobufVersion(Any.class, "java.lang.Class");
+
+    assertTrue(Pattern.compile("^\\d+\\.\\d+\\.\\d+").matcher(version).find());
+  }
+
+  @Test
+  void testGetProtobufVersion_noManifest() throws Exception {
+    String version = GaxProperties.getProtobufVersion(GaxProperties.class, "foo.NonExistantClass");
+
+    assertEquals("3", version);
+  }
+
   private Version readVersion(String version) {
     assertTrue(Pattern.compile("^\\d+\\.\\d+\\.\\d+").matcher(version).find());
     String[] versionComponents = version.split("\\.");
@@ -194,4 +221,11 @@ public Version(int major, int minor) {
       this.minor = minor;
     }
   }
+
+  // Test class that emulates com.google.protobuf.RuntimeVersion for reflection lookup of fields
+  class RuntimeVersion {
+    public static final int MAJOR = 3;
+    public static final int MINOR = 13;
+    public static final int PATCH = 6;
+  }
 }

hermetic_build/DEVELOPMENT.md

@@ -165,13 +165,10 @@ python hermetic_build/library_generation/cli/entry_point.py generate \
      -t local:image-tag \
      .
    ```
-   Please note that the build only works when using the new
-   [Docker BuildKit](https://docs.docker.com/build/buildkit/) (enabled through the `DOCKER_BUILDKIT` variable).
-   This is meant for local development only (in CloudTops) - GH Actions' Ubuntu-22.04 \
-   [comes with the latest Docker version](https://github.com/actions/runner-images/blob/e74605cd6d5407469cf224802f25057bafc23d70/images/ubuntu/Ubuntu2204-Readme.md?plain=1#L81-L83)
-   and is able to handle the build properly using the (updated) legacy builder.
+   Please note that the build only works when using the new [Docker BuildKit](https://docs.docker.com/build/buildkit/)
+   (enabled through the `DOCKER_BUILDKIT` variable).
 
-3. Set the version of gapic-generator-java
+2. Set the version of gapic-generator-java
 
    ```shell
    LOCAL_GENERATOR_VERSION=$(mvn \
@@ -182,7 +179,7 @@ python hermetic_build/library_generation/cli/entry_point.py generate \
      -q)
    ```
 
-4. Run the image
+3. Run the image
 
    ```shell
       # Assume you want to generate the library in the current working directory
@@ -199,7 +196,7 @@ python hermetic_build/library_generation/cli/entry_point.py generate \
         --library-names=apigee-connect,asset \
         --repository-path=/workspace \
         --api-definitions-path=/workspace/apis
-      ```
+   ```
    Note that if you specify the generator version using environment variable,
    `-e GENERATOR_VERSION="${LOCAL_GENERATOR_VERSION}"` in the above example,
    you should not set `gapic_generator_version` and `protoc_version` in the

hermetic_build/library_generation/generate_library.sh

@@ -115,7 +115,7 @@ if [ -z "${os_architecture}" ]; then
   os_architecture=$(detect_os_architecture)
 fi
 
-temp_destination_path="${output_folder}/temp_preprocessed"
+temp_destination_path="${output_folder}/temp_preprocessed-$RANDOM"
 mkdir -p "${output_folder}/${destination_path}"
 if [ -d "${temp_destination_path}" ]; then
   # we don't want the preprocessed sources of a previous run

hermetic_build/library_generation/tests/integration_tests.py

@@ -202,6 +202,7 @@ def __build_image(cls, docker_file: str, cwd: str):
         subprocess.check_call(
             ["docker", "build", "-f", docker_file, "-t", image_tag, "."],
             cwd=cwd,
+            env=dict(os.environ, DOCKER_BUILDKIT="1"),
         )
 
     @classmethod