chore(hermetic-build): include .github template updates as part of generation

hermetic_build/DEVELOPMENT.md

@@ -130,6 +130,9 @@ owl-bot copy-code --version
 The key step is `npm link`, which will make the command available in you current
 shell session.
 
+If you get a permission denied error when running the command `owl-bot`, try
+relinking owl-bot by running `npm unlink -g` and re-running the steps above.
+
 ## Run the script
 The entrypoint script (`hermetic_build/library_generation/cli/entry_point.py`)
 allows you to generate a GAPIC repository with a given api definition (proto,
@@ -153,12 +156,14 @@ export path_to_repo="$(pwd)/google-cloud-java"
 ### Install the scripts
 
 You can skip this step if you've installed the packages in [Install package dependencies](#install-package-dependencies).
+Use the `--editable` flag for your changes to take effect as soon as you modify any file inside
+the package.
 
 ```shell
 python -m pip install --require-hashes -r hermetic_build/common/requirements.txt
-python -m pip install hermetic_build/common
+python -m pip install --editable hermetic_build/common
 python -m pip install --require-hashes -r hermetic_build/library_generation/requirements.txt
-python -m pip install hermetic_build/library_generation
+python -m pip install --editable hermetic_build/library_generation
 ```
 
 ### Run the script
@@ -212,10 +217,19 @@ python hermetic_build/library_generation/cli/entry_point.py generate \
         --api-definitions-path=/workspace/apis
    ```
 
-# Debug the library generation container
+# Debugging tips
+## Debug the scripts
+Especially on local setups, consider adding the `xtrace` (`set -x`) flag to
+ - `hermetic_build/library_generation/generate_library.sh`
+ - `hermetic_build/library_generation/utils/utilities.sh`
+
+This will allow you to observe how the tools you prepared in `~/.library_generation` are being used.
+
+
+## Debug the library generation container
 If you are working on changing the way the containers are created, you may want
 to inspect the containers to check the setup.
-It would be convenient in such case to have a text editor/viewer available. 
+It would be convenient in such case to have a text editor/viewer available.
 You can achieve this by modifying the Dockerfile as follows:
 
 ```dockerfile

hermetic_build/library_generation/owlbot/templates/java_library/.github/generated-files-bot.yml

@@ -0,0 +1,12 @@
+externalManifests:
+- type: json
+  file: 'synth.metadata'
+  jsonpath: '$.generatedFiles[*]'
+- type: json
+  file: '.github/readme/synth.metadata/synth.metadata'
+  jsonpath: '$.generatedFiles[*]'
+ignoreAuthors:
+- 'renovate-bot'
+- 'yoshi-automation'
+- 'release-please[bot]'
+- 'gcf-owl-bot[bot]'

hermetic_build/library_generation/owlbot/templates/java_library/.kokoro/build.bat

@@ -0,0 +1,18 @@
+:: Copyright 2022 Google LLC
+::
+:: Licensed under the Apache License, Version 2.0 (the "License");
+:: you may not use this file except in compliance with the License.
+:: You may obtain a copy of the License at
+::
+::      http://www.apache.org/licenses/LICENSE-2.0
+::
+:: Unless required by applicable law or agreed to in writing, software
+:: distributed under the License is distributed on an "AS IS" BASIS,
+:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+:: See the License for the specific language governing permissions and
+:: limitations under the License.
+:: Github action job to test core java library features on
+:: downstream client libraries before they are released.
+:: See documentation in type-shell-output.bat
+
+"C:\Program Files\Git\bin\bash.exe" %~dp0build.sh

hermetic_build/library_generation/owlbot/templates/java_library/.kokoro/build.sh

@@ -1,12 +1,11 @@
 #!/bin/bash
-
-# Copyright 2018 Google LLC
+# Copyright 2019 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
-#     https://www.apache.org/licenses/LICENSE-2.0
+#      http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
@@ -16,13 +15,121 @@
 
 set -eo pipefail
 
-cd github/synthtool
+## Get the directory of the build script
+scriptDir=$(realpath $(dirname "${BASH_SOURCE[0]}"))
+## cd to the parent directory, i.e. the root of the git repo
+cd ${scriptDir}/..
+
+# include common functions
+source ${scriptDir}/common.sh
+
+# Print out Maven & Java version
+mvn -version
+echo ${JOB_TYPE}
+
+# attempt to install 3 times with exponential backoff (starting with 10 seconds)
+retry_with_backoff 3 10 \
+  mvn install -B -V -ntp \
+    -DskipTests=true \
+    -Dclirr.skip=true \
+    -Denforcer.skip=true \
+    -Dmaven.javadoc.skip=true \
+    -Dgcloud.download.skip=true \
+    -T 1C
+
+# if GOOGLE_APPLICATION_CREDENTIALS is specified as a relative path, prepend Kokoro root directory onto it
+if [[ ! -z "${GOOGLE_APPLICATION_CREDENTIALS}" && "${GOOGLE_APPLICATION_CREDENTIALS}" != /* ]]; then
+    export GOOGLE_APPLICATION_CREDENTIALS=$(realpath ${KOKORO_GFILE_DIR}/${GOOGLE_APPLICATION_CREDENTIALS})
+fi
+
+RETURN_CODE=0
+set +e
+
+case ${JOB_TYPE} in
+test)
+    echo "SUREFIRE_JVM_OPT: ${SUREFIRE_JVM_OPT}"
+    mvn test -B -ntp -Dclirr.skip=true -Denforcer.skip=true ${SUREFIRE_JVM_OPT}
+    RETURN_CODE=$?
+    ;;
+lint)
+    mvn com.coveo:fmt-maven-plugin:check -B -ntp
+    RETURN_CODE=$?
+    ;;
+javadoc)
+    mvn javadoc:javadoc javadoc:test-javadoc -B -ntp
+    RETURN_CODE=$?
+    ;;
+integration)
+    mvn -B ${INTEGRATION_TEST_ARGS} \
+      -ntp \
+      -Penable-integration-tests \
+      -DtrimStackTrace=false \
+      -Dclirr.skip=true \
+      -Denforcer.skip=true \
+      -fae \
+      verify
+    RETURN_CODE=$?
+    ;;
+graalvm)
+    # Run Unit and Integration Tests with Native Image
+    mvn -B ${INTEGRATION_TEST_ARGS} -ntp -Pnative test
+    RETURN_CODE=$?
+    ;;
+graalvm17)
+    # Run Unit and Integration Tests with Native Image
+    mvn -B ${INTEGRATION_TEST_ARGS} -ntp -Pnative test
+    RETURN_CODE=$?
+    ;;
+samples)
+    SAMPLES_DIR=samples
+    # only run ITs in snapshot/ on presubmit PRs. run ITs in all 3 samples/ subdirectories otherwise.
+    if [[ ! -z ${KOKORO_GITHUB_PULL_REQUEST_NUMBER} ]]
+    then
+      SAMPLES_DIR=samples/snapshot
+    fi
+
+    if [[ -f ${SAMPLES_DIR}/pom.xml ]]
+    then
+        for FILE in ${KOKORO_GFILE_DIR}/secret_manager/*-samples-secrets; do
+          [[ -f "$FILE" ]] || continue
+          source "$FILE"
+        done
+
+        pushd ${SAMPLES_DIR}
+        mvn -B \
+          -ntp \
+          -DtrimStackTrace=false \
+          -Dclirr.skip=true \
+          -Denforcer.skip=true \
+          -fae \
+          verify
+        RETURN_CODE=$?
+        popd
+    else
+        echo "no sample pom.xml found - skipping sample tests"
+    fi
+    ;;
+clirr)
+    mvn -B -ntp -Denforcer.skip=true clirr:check
+    RETURN_CODE=$?
+    ;;
+*)
+    ;;
+esac
+
+if [ "${REPORT_COVERAGE}" == "true" ]
+then
+  bash ${KOKORO_GFILE_DIR}/codecov.sh
+fi
 
-# Disable buffering, so that the logs stream through.
-export PYTHONUNBUFFERED=1
+# fix output location of logs
+bash .kokoro/coerce_logs.sh
 
-# Run tests
-nox -s lint test
+if [[ "${ENABLE_FLAKYBOT}" == "true" ]]
+then
+    chmod +x ${KOKORO_GFILE_DIR}/linux_amd64/flakybot
+    ${KOKORO_GFILE_DIR}/linux_amd64/flakybot -repo={{metadata['repo']['repo']}}
+fi
 
-# remove all files, preventing kokoro from trying to sync them.
-rm -rf *
+echo "exiting with ${RETURN_CODE}"
+exit ${RETURN_CODE}

hermetic_build/library_generation/owlbot/templates/java_library/.kokoro/coerce_logs.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script finds and moves sponge logs so that they can be found by placer
+# and are not flagged as flaky by sponge.
+
+set -eo pipefail
+
+## Get the directory of the build script
+scriptDir=$(realpath $(dirname "${BASH_SOURCE[0]}"))
+## cd to the parent directory, i.e. the root of the git repo
+cd ${scriptDir}/..
+
+job=$(basename ${KOKORO_JOB_NAME})
+
+echo "coercing sponge logs..."
+for xml in `find . -name *-sponge_log.xml`
+do
+  class=$(basename ${xml} | cut -d- -f2)
+  dir=$(dirname ${xml})/${job}/${class}
+  text=$(dirname ${xml})/${class}-sponge_log.txt
+  mkdir -p ${dir}
+  mv ${xml} ${dir}/sponge_log.xml
+  mv ${text} ${dir}/sponge_log.txt
+done

hermetic_build/library_generation/owlbot/templates/java_library/.kokoro/common.cfg

@@ -1,19 +1,13 @@
 # Format: //devtools/kokoro/config/proto/build.proto
 
-# Download trampoline resources.
+# Download trampoline resources. These will be in ${KOKORO_GFILE_DIR}
 gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline"
 
-# Use the trampoline script to run in docker.
-build_file: "synthtool/.kokoro/trampoline.sh"
-
-# Configure the docker image for kokoro-trampoline.
-env_vars: {
-    key: "TRAMPOLINE_IMAGE"
-    value: "gcr.io/cloud-devrel-kokoro-resources/python"
-}
+# All builds use the trampoline script to run in docker.
+build_file: "{{ metadata['repo']['repo_short'] }}/.kokoro/trampoline.sh"
 
 # Tell the trampoline which build file to use.
 env_vars: {
-    key: "TRAMPOLINE_BUILD_FILE"
-    value: "github/synthtool/.kokoro/build.sh"
+  key: "TRAMPOLINE_BUILD_FILE"
+  value: "github/{{ metadata['repo']['repo_short'] }}/.kokoro/build.sh"
 }

hermetic_build/library_generation/owlbot/templates/java_library/.kokoro/common.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+function retry_with_backoff {
+    attempts_left=$1
+    sleep_seconds=$2
+    shift 2
+    command=$@
+
+
+    # store current flag state
+    flags=$-
+
+    # allow a failures to continue
+    set +e
+    ${command}
+    exit_code=$?
+
+    # restore "e" flag
+    if [[ ${flags} =~ e ]]
+    then set -e
+    else set +e
+    fi
+
+    if [[ $exit_code == 0 ]]
+    then
+        return 0
+    fi
+
+    # failure
+    if [[ ${attempts_left} -gt 0 ]]
+    then
+        echo "failure (${exit_code}), sleeping ${sleep_seconds}..."
+        sleep ${sleep_seconds}
+        new_attempts=$((${attempts_left} - 1))
+        new_sleep=$((${sleep_seconds} * 2))
+        retry_with_backoff ${new_attempts} ${new_sleep} ${command}
+    fi
+
+    return $exit_code
+}
+
+## Helper functionss
+function now() { date +"%Y-%m-%d %H:%M:%S" | tr -d '\n'; }
+function msg() { println "$*" >&2; }
+function println() { printf '%s\n' "$(now) $*"; }
+
+## Helper comment to trigger updated repo dependency release

hermetic_build/library_generation/owlbot/templates/java_library/.kokoro/continuous/common.cfg

@@ -0,0 +1,25 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Build logs will be here
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.xml"
+    regex: "**/*sponge_log.txt"
+  }
+}
+
+# Download trampoline resources.
+gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline"
+
+# Use the trampoline script to run in docker.
+build_file: "{{ metadata['repo']['repo_short'] }}/.kokoro/trampoline.sh"
+
+env_vars: {
+  key: "TRAMPOLINE_BUILD_FILE"
+  value: "github/{{ metadata['repo']['repo_short'] }}/.kokoro/build.sh"
+}
+
+env_vars: {
+  key: "JOB_TYPE"
+  value: "test"
+}

hermetic_build/library_generation/owlbot/templates/java_library/.kokoro/continuous/java8.cfg

@@ -0,0 +1,12 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Configure the docker image for kokoro-trampoline.
+env_vars: {
+  key: "TRAMPOLINE_IMAGE"
+  value: "gcr.io/cloud-devrel-kokoro-resources/java8"
+}
+
+env_vars: {
+  key: "REPORT_COVERAGE"
+  value: "true"
+}