From 79b6ece44df9270d5bf5a81345567bd58e7fd916 Mon Sep 17 00:00:00 2001 From: Tomo Suzuki Date: Wed, 26 Nov 2025 21:14:28 +0000 Subject: [PATCH 1/8] fix: hermetic build to create one line per new library b/373612851 --- .../library_generation/owlbot/src/fix_poms.py | 31 +++++++++++++------ .../owlbot/templates/poms/bom_pom.xml.j2 | 4 +-- .../owlbot/templates/poms/cloud_pom.xml.j2 | 4 +-- .../owlbot/templates/poms/grpc_pom.xml.j2 | 4 +-- .../owlbot/templates/poms/parent_pom.xml.j2 | 4 +-- .../owlbot/templates/poms/proto_pom.xml.j2 | 4 +-- 6 files changed, 31 insertions(+), 20 deletions(-) diff --git a/hermetic_build/library_generation/owlbot/src/fix_poms.py b/hermetic_build/library_generation/owlbot/src/fix_poms.py index 9ee7514a4d..6fad5b0388 100644 --- a/hermetic_build/library_generation/owlbot/src/fix_poms.py +++ b/hermetic_build/library_generation/owlbot/src/fix_poms.py @@ -358,6 +358,9 @@ def main(versions_file, monorepo): with open(".repo-metadata.json", "r") as fp: repo_metadata = json.load(fp) group_id, artifact_id = repo_metadata["distribution_name"].split(":") + # Derive the Release Please key in java-xxx format + base_name = re.sub(r'^(google-cloud-|grpc-google-cloud-|proto-google-cloud-)', '', artifact_id) + release_please_key = f"java-{base_name}" name = repo_metadata["name_pretty"] existing_modules = load_versions(versions_file, group_id) print(f"monorepo? {monorepo}") @@ -456,6 +459,7 @@ def main(versions_file, monorepo): parent_module=parent_module, main_module=main_module, monorepo=monorepo, + release_please_key=release_please_key, ) if ( path not in excluded_dependencies_list @@ -498,6 +502,7 @@ def main(versions_file, monorepo): main_module=main_module, proto_module=existing_modules[proto_artifact_id], monorepo=monorepo, + release_please_key=release_please_key, ) if ( path not in excluded_dependencies_list @@ -549,6 +554,7 @@ def main(versions_file, monorepo): proto_modules=proto_modules, grpc_modules=grpc_modules, monorepo=monorepo, + release_please_key=release_please_key, ) if os.path.isfile(f"{artifact_id}-bom/pom.xml"): @@ -567,6 +573,7 @@ def main(versions_file, monorepo): main_module=main_module, monorepo=monorepo, monorepo_version=monorepo_version, + release_please_key=release_please_key, ) if os.path.isfile("pom.xml"): @@ -584,24 +591,28 @@ def main(versions_file, monorepo): name=name, monorepo=monorepo, monorepo_version=monorepo_version, + release_please_key=release_please_key, ) print(f"updating modules in {versions_file}") existing_modules.pop(parent_artifact_id) - # add extra modules to versions.txt - for dependency_module in extra_managed_modules: - if dependency_module not in existing_modules: - existing_modules[dependency_module] = module.Module( - group_id=__proto_group_id(group_id), - artifact_id=dependency_module, - version=main_module.version, - release_version=main_module.release_version, - ) + + # Consolidate all modules into a single `java-` entry. + # It's safe to use the first module's version and group ID, + # as they will be consistent for a new library. + templates.render( template_name="versions.txt.j2", output_name=versions_file, - modules=existing_modules.values(), + # Only write the main artifact module to versions.txt + modules=[ + module.Module( + group_id=main_module.group_id, + artifact_id=release_please_key, + version=main_module.version, + release_version=main_module.release_version,) + ], ) diff --git a/hermetic_build/library_generation/owlbot/templates/poms/bom_pom.xml.j2 b/hermetic_build/library_generation/owlbot/templates/poms/bom_pom.xml.j2 index ddcef5226f..dcd930c1ae 100644 --- a/hermetic_build/library_generation/owlbot/templates/poms/bom_pom.xml.j2 +++ b/hermetic_build/library_generation/owlbot/templates/poms/bom_pom.xml.j2 @@ -3,7 +3,7 @@ 4.0.0 {{main_module.group_id}} {{main_module.artifact_id}}-bom - {{main_module.version}} + {{main_module.version}} pom {% if monorepo -%} @@ -34,7 +34,7 @@ {{module.group_id}} {{module.artifact_id}} - {{module.version}} + {{module.version}} {% endfor %} diff --git a/hermetic_build/library_generation/owlbot/templates/poms/cloud_pom.xml.j2 b/hermetic_build/library_generation/owlbot/templates/poms/cloud_pom.xml.j2 index d0ae8cd7c5..3087201693 100644 --- a/hermetic_build/library_generation/owlbot/templates/poms/cloud_pom.xml.j2 +++ b/hermetic_build/library_generation/owlbot/templates/poms/cloud_pom.xml.j2 @@ -3,7 +3,7 @@ 4.0.0 {{module.group_id}} {{module.artifact_id}} - {{module.version}} + {{module.version}} jar Google {{name}} {%- if not monorepo %} @@ -13,7 +13,7 @@ {{parent_module.group_id}} {{parent_module.artifact_id}} - {{parent_module.version}} + {{parent_module.version}} {{module.artifact_id}} diff --git a/hermetic_build/library_generation/owlbot/templates/poms/grpc_pom.xml.j2 b/hermetic_build/library_generation/owlbot/templates/poms/grpc_pom.xml.j2 index 514861e7a7..4390cf92c0 100644 --- a/hermetic_build/library_generation/owlbot/templates/poms/grpc_pom.xml.j2 +++ b/hermetic_build/library_generation/owlbot/templates/poms/grpc_pom.xml.j2 @@ -4,13 +4,13 @@ 4.0.0 {{module.group_id}} {{module.artifact_id}} - {{module.version}} + {{module.version}} {{module.artifact_id}} GRPC library for {{main_module.artifact_id}} {{parent_module.group_id}} {{parent_module.artifact_id}} - {{parent_module.version}} + {{parent_module.version}} diff --git a/hermetic_build/library_generation/owlbot/templates/poms/parent_pom.xml.j2 b/hermetic_build/library_generation/owlbot/templates/poms/parent_pom.xml.j2 index dcf922340e..2b87c0f601 100644 --- a/hermetic_build/library_generation/owlbot/templates/poms/parent_pom.xml.j2 +++ b/hermetic_build/library_generation/owlbot/templates/poms/parent_pom.xml.j2 @@ -4,7 +4,7 @@ {{main_module.group_id}} {{main_module.artifact_id}}-parent pom - {{main_module.version}} + {{main_module.version}} Google {{name}} Parent Java idiomatic client for Google Cloud Platform services. @@ -37,7 +37,7 @@ {% for module in modules %} {{module.group_id}} {{module.artifact_id}} - {{module.version}} + {{module.version}} {% endfor %} diff --git a/hermetic_build/library_generation/owlbot/templates/poms/proto_pom.xml.j2 b/hermetic_build/library_generation/owlbot/templates/poms/proto_pom.xml.j2 index 886cd02663..11869120ec 100644 --- a/hermetic_build/library_generation/owlbot/templates/poms/proto_pom.xml.j2 +++ b/hermetic_build/library_generation/owlbot/templates/poms/proto_pom.xml.j2 @@ -4,13 +4,13 @@ 4.0.0 {{module.group_id}} {{module.artifact_id}} - {{module.version}} + {{module.version}} {{module.artifact_id}} Proto library for {{main_module.artifact_id}} {{parent_module.group_id}} {{parent_module.artifact_id}} - {{parent_module.version}} + {{parent_module.version}} From 2c94a9ebb14640529b4f7057d14b2787e0daad27 Mon Sep 17 00:00:00 2001 From: Tomo Suzuki Date: Wed, 26 Nov 2025 21:17:48 +0000 Subject: [PATCH 2/8] format --- .../library_generation/owlbot/src/fix_poms.py | 14 +++++---- .../tests/resources/goldens/owlbot-golden.py | 29 ++++++++++--------- 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/hermetic_build/library_generation/owlbot/src/fix_poms.py b/hermetic_build/library_generation/owlbot/src/fix_poms.py index 6fad5b0388..3d7564c7e2 100644 --- a/hermetic_build/library_generation/owlbot/src/fix_poms.py +++ b/hermetic_build/library_generation/owlbot/src/fix_poms.py @@ -359,7 +359,9 @@ def main(versions_file, monorepo): repo_metadata = json.load(fp) group_id, artifact_id = repo_metadata["distribution_name"].split(":") # Derive the Release Please key in java-xxx format - base_name = re.sub(r'^(google-cloud-|grpc-google-cloud-|proto-google-cloud-)', '', artifact_id) + base_name = re.sub( + r"^(google-cloud-|grpc-google-cloud-|proto-google-cloud-)", "", artifact_id + ) release_please_key = f"java-{base_name}" name = repo_metadata["name_pretty"] existing_modules = load_versions(versions_file, group_id) @@ -597,7 +599,6 @@ def main(versions_file, monorepo): print(f"updating modules in {versions_file}") existing_modules.pop(parent_artifact_id) - # Consolidate all modules into a single `java-` entry. # It's safe to use the first module's version and group ID, # as they will be consistent for a new library. @@ -608,10 +609,11 @@ def main(versions_file, monorepo): # Only write the main artifact module to versions.txt modules=[ module.Module( - group_id=main_module.group_id, - artifact_id=release_please_key, - version=main_module.version, - release_version=main_module.release_version,) + group_id=main_module.group_id, + artifact_id=release_please_key, + version=main_module.version, + release_version=main_module.release_version, + ) ], ) diff --git a/hermetic_build/library_generation/tests/resources/goldens/owlbot-golden.py b/hermetic_build/library_generation/tests/resources/goldens/owlbot-golden.py index 2ba11e6bba..7559eaf034 100644 --- a/hermetic_build/library_generation/tests/resources/goldens/owlbot-golden.py +++ b/hermetic_build/library_generation/tests/resources/goldens/owlbot-golden.py @@ -21,16 +21,19 @@ s.move(library) s.remove_staging_dirs() -java.common_templates(monorepo=True, excludes=[ - ".github/*", - ".kokoro/*", - "samples/*", - "CODE_OF_CONDUCT.md", - "CONTRIBUTING.md", - "LICENSE", - "SECURITY.md", - "java.header", - "license-checks.xml", - "renovate.json", - ".gitignore" -]) \ No newline at end of file +java.common_templates( + monorepo=True, + excludes=[ + ".github/*", + ".kokoro/*", + "samples/*", + "CODE_OF_CONDUCT.md", + "CONTRIBUTING.md", + "LICENSE", + "SECURITY.md", + "java.header", + "license-checks.xml", + "renovate.json", + ".gitignore", + ], +) From c27510fd42f9bb3ec73404521c96f1e77c0d5498 Mon Sep 17 00:00:00 2001 From: Tomo Suzuki Date: Thu, 27 Nov 2025 01:39:24 +0000 Subject: [PATCH 3/8] fix: fix_poms.py to add one line in versions.txt for new library --- .../library_generation/owlbot/src/fix_poms.py | 41 ++++++--- .../tests/owlbot/fix_poms_unit_tests.py | 89 +++++++++++++++++++ 2 files changed, 117 insertions(+), 13 deletions(-) diff --git a/hermetic_build/library_generation/owlbot/src/fix_poms.py b/hermetic_build/library_generation/owlbot/src/fix_poms.py index 3d7564c7e2..b9c2e63de8 100644 --- a/hermetic_build/library_generation/owlbot/src/fix_poms.py +++ b/hermetic_build/library_generation/owlbot/src/fix_poms.py @@ -362,9 +362,10 @@ def main(versions_file, monorepo): base_name = re.sub( r"^(google-cloud-|grpc-google-cloud-|proto-google-cloud-)", "", artifact_id ) - release_please_key = f"java-{base_name}" + release_please_key = base_name name = repo_metadata["name_pretty"] existing_modules = load_versions(versions_file, group_id) + original_modules = set(existing_modules.keys()) print(f"monorepo? {monorepo}") # extra modules that need to be manages in versions.txt @@ -597,24 +598,38 @@ def main(versions_file, monorepo): ) print(f"updating modules in {versions_file}") + # The parent pom is not a standalone artifact, so we don't add it to + # versions.txt. existing_modules.pop(parent_artifact_id) - # Consolidate all modules into a single `java-` entry. - # It's safe to use the first module's version and group ID, - # as they will be consistent for a new library. + # Determine if new modules were added. + added_modules = set(existing_modules.keys()) - original_modules + if added_modules: + print(f"New modules detected: {added_modules}") + # Remove all the individual components that were just added. + for key in added_modules: + existing_modules.pop(key) + # Add the single consolidated entry instead. + existing_modules[release_please_key] = module.Module( + group_id=group_id, + artifact_id=release_please_key, + version=main_module.version, + release_version=main_module.release_version, + ) - templates.render( - template_name="versions.txt.j2", - output_name=versions_file, - # Only write the main artifact module to versions.txt - modules=[ - module.Module( - group_id=main_module.group_id, - artifact_id=release_please_key, + # add extra modules to versions.txt + for dependency_module in extra_managed_modules: + if dependency_module not in existing_modules: + existing_modules[dependency_module] = module.Module( + group_id=__proto_group_id(group_id), + artifact_id=dependency_module, version=main_module.version, release_version=main_module.release_version, ) - ], + templates.render( + template_name="versions.txt.j2", + output_name=versions_file, + modules=existing_modules.values(), ) diff --git a/hermetic_build/library_generation/tests/owlbot/fix_poms_unit_tests.py b/hermetic_build/library_generation/tests/owlbot/fix_poms_unit_tests.py index 2e66454827..ea5ef25cc1 100644 --- a/hermetic_build/library_generation/tests/owlbot/fix_poms_unit_tests.py +++ b/hermetic_build/library_generation/tests/owlbot/fix_poms_unit_tests.py @@ -14,6 +14,8 @@ import os import shutil import unittest +import tempfile +from pathlib import Path from library_generation.owlbot.src.fix_poms import main from library_generation.tests.compare_poms import compare_pom_in_subdir @@ -22,6 +24,15 @@ class FixPomsTest(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.mkdtemp() + self.start_dir = os.getcwd() + os.chdir(self.temp_dir) + + def tearDown(self): + os.chdir(self.start_dir) + + def test_update_poms_group_id_does_not_start_with_google_correctly(self): ad_manager_resource = os.path.join(resources_dir, "java-admanager") versions_file = os.path.join(ad_manager_resource, "versions.txt") @@ -35,6 +46,84 @@ def test_update_poms_group_id_does_not_start_with_google_correctly(self): for sub_dir in sub_dirs: self.__remove_file_in_subdir(ad_manager_resource, sub_dir) + + def test_add_new_library(self): + # Setup test environment in a structure similar to a real repo + repo_dir = os.path.join(self.temp_dir, "google-cloud-java") + os.makedirs(repo_dir) + source_resource_dir = os.path.join(resources_dir, "google-cloud-java") + new_library_dir_name = "java-newapi" + new_library_path = os.path.join(repo_dir, new_library_dir_name) + + # Copy test resources to the temporary directory + shutil.copytree(os.path.join(source_resource_dir, "java-newapi"), new_library_path) + + # Modify the .repo-metadata.json in the temporary new library folder + repo_metadata_path = os.path.join(new_library_path, ".repo-metadata.json") + with open(repo_metadata_path, "r") as f: + repo_metadata_content = f.read() + repo_metadata_content = repo_metadata_content.replace("google-cloud-newfolder", "google-cloud-newapi") + with open(repo_metadata_path, "w") as f: + f.write(repo_metadata_content) + + source_versions_file = os.path.join(source_resource_dir, "versions.txt") + dest_versions_file = os.path.join(repo_dir, "versions.txt") + shutil.copyfile(source_versions_file, dest_versions_file) + + # Create dummy directories for proto and grpc modules. + # In the full generation process, `generate_library.sh` would create these. + os.makedirs(os.path.join(new_library_path, "proto-google-cloud-newapi-v1")) + os.makedirs(os.path.join(new_library_path, "grpc-google-cloud-newapi-v1")) + os.chdir(new_library_path) + + # Get initial state of versions.txt (excluding comments and empty lines) + with open(dest_versions_file, "r") as f: + initial_active_lines = [line.strip() for line in f if line.strip() and not line.strip().startswith("#")] + + # Execute the main function from fix_poms.py + main(dest_versions_file, "true") + + # --- Verify versions.txt --- + with open(dest_versions_file, "r") as f: + updated_versions_content = f.readlines() + + updated_active_lines = [line.strip() for line in updated_versions_content if line.strip() and not line.strip().startswith("#")] + + self.assertEqual(len(initial_active_lines) + 1, len(updated_active_lines), + "A single line should have been added to versions.txt") + + expected_new_line = "newapi:0.0.0:0.0.1-SNAPSHOT" + self.assertTrue(any(expected_new_line in line for line in updated_active_lines), + f"The new line '{expected_new_line}' was not found in versions.txt") + + filtered_updated_lines = [line for line in updated_active_lines if expected_new_line not in line] + self.assertCountEqual(initial_active_lines, filtered_updated_lines, + "Existing lines in versions.txt should be unaffected.") + self.assertFalse(any("google-cloud-newapi" in line for line in updated_versions_content), + "The artifactId 'google-cloud-newapi' should not be in versions.txt") + + # --- Verify generated pom.xml files --- + expected_poms = [ + "pom.xml", + "google-cloud-newapi/pom.xml", + "google-cloud-newapi-bom/pom.xml", + "proto-google-cloud-newapi-v1/pom.xml", + "grpc-google-cloud-newapi-v1/pom.xml", + ] + for pom_path in expected_poms: + full_pom_path = os.path.join(new_library_path, pom_path) + self.assertTrue(os.path.exists(full_pom_path), f"Expected POM file not found: {full_pom_path}") + with open(full_pom_path, "r") as f: + pom_content = f.read() + self.assertIn("", pom_content, + f"x-version-update tag in {pom_path} does not reference 'newapi'") + # Ensure the artifactId is not changed to the release_please_key + if pom_path == "google-cloud-newapi/pom.xml": + self.assertIn("google-cloud-newapi", pom_content) + if pom_path == "proto-google-cloud-newapi-v1/pom.xml": + self.assertIn("proto-google-cloud-newapi-v1", pom_content) + + @classmethod def __copy__golden(cls, base_dir: str, subdir: str): golden = os.path.join(base_dir, subdir, "pom-golden.xml") From b3aff43cf738b8a03fe606af2c83e18bca7259ed Mon Sep 17 00:00:00 2001 From: Tomo Suzuki Date: Thu, 27 Nov 2025 01:56:34 +0000 Subject: [PATCH 4/8] simplified the logic --- .../library_generation/owlbot/src/fix_poms.py | 58 ++++++++++--------- .../test-owlbot/java-admanager/versions.txt | 3 +- 2 files changed, 33 insertions(+), 28 deletions(-) diff --git a/hermetic_build/library_generation/owlbot/src/fix_poms.py b/hermetic_build/library_generation/owlbot/src/fix_poms.py index b9c2e63de8..855b5a4b8e 100644 --- a/hermetic_build/library_generation/owlbot/src/fix_poms.py +++ b/hermetic_build/library_generation/owlbot/src/fix_poms.py @@ -358,11 +358,8 @@ def main(versions_file, monorepo): with open(".repo-metadata.json", "r") as fp: repo_metadata = json.load(fp) group_id, artifact_id = repo_metadata["distribution_name"].split(":") - # Derive the Release Please key in java-xxx format - base_name = re.sub( - r"^(google-cloud-|grpc-google-cloud-|proto-google-cloud-)", "", artifact_id - ) - release_please_key = base_name + # Use api_shortname from repo_metadata as the key for versions.txt + release_please_key = repo_metadata["api_shortname"] name = repo_metadata["name_pretty"] existing_modules = load_versions(versions_file, group_id) original_modules = set(existing_modules.keys()) @@ -598,38 +595,45 @@ def main(versions_file, monorepo): ) print(f"updating modules in {versions_file}") - # The parent pom is not a standalone artifact, so we don't add it to - # versions.txt. - existing_modules.pop(parent_artifact_id) - - # Determine if new modules were added. - added_modules = set(existing_modules.keys()) - original_modules - if added_modules: - print(f"New modules detected: {added_modules}") - # Remove all the individual components that were just added. - for key in added_modules: - existing_modules.pop(key) - # Add the single consolidated entry instead. - existing_modules[release_please_key] = module.Module( + + # existing_modules contains all components, needed for pom.xml rendering. + + versions_txt_modules = {} + # Start with all modules that were in the original versions.txt + for key in original_modules: + if key in existing_modules: + versions_txt_modules[key] = existing_modules[key] + + # If this is a new library (release_please_key was not in original_modules), + # add the single consolidated entry to our versions.txt dictionary. + if release_please_key not in original_modules: + versions_txt_modules[release_please_key] = module.Module( group_id=group_id, artifact_id=release_please_key, version=main_module.version, release_version=main_module.release_version, ) - # add extra modules to versions.txt + # Add extra managed modules if they aren't already included for dependency_module in extra_managed_modules: - if dependency_module not in existing_modules: - existing_modules[dependency_module] = module.Module( - group_id=__proto_group_id(group_id), - artifact_id=dependency_module, - version=main_module.version, - release_version=main_module.release_version, - ) + if dependency_module not in versions_txt_modules: + # These should already be in existing_modules if loaded from versions.txt + if dependency_module in existing_modules: + versions_txt_modules[dependency_module] = existing_modules[dependency_module] + # This else block should ideally not be reached in the monorepo case. + else: + versions_txt_modules[dependency_module] = module.Module( + group_id=__proto_group_id(group_id), + artifact_id=dependency_module, + version=main_module.version, + release_version=main_module.release_version, + ) + + # Render versions.txt using the specially prepared versions_txt_modules templates.render( template_name="versions.txt.j2", output_name=versions_file, - modules=existing_modules.values(), + modules=versions_txt_modules.values(), ) diff --git a/hermetic_build/library_generation/tests/resources/test-owlbot/java-admanager/versions.txt b/hermetic_build/library_generation/tests/resources/test-owlbot/java-admanager/versions.txt index 3f073728f3..930a79a07e 100644 --- a/hermetic_build/library_generation/tests/resources/test-owlbot/java-admanager/versions.txt +++ b/hermetic_build/library_generation/tests/resources/test-owlbot/java-admanager/versions.txt @@ -1,6 +1,7 @@ # Format: # module:released-version:current-version +admanager:0.0.0:0.0.1-SNAPSHOT google-cloud-java:1.36.0:1.37.0-SNAPSHOT -ad-manager:0.1.0:0.2.0-SNAPSHOT proto-ad-manager-v1:0.1.0:0.2.0-SNAPSHOT +ad-manager:0.1.0:0.2.0-SNAPSHOT From 3476024cbe4af41cb3a77ebd5ac4ebe92d64f443 Mon Sep 17 00:00:00 2001 From: Tomo Suzuki Date: Thu, 27 Nov 2025 01:58:32 +0000 Subject: [PATCH 5/8] add test file --- .../java-newapi/.repo-metadata.json | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 hermetic_build/library_generation/tests/resources/test-owlbot/google-cloud-java/java-newapi/.repo-metadata.json diff --git a/hermetic_build/library_generation/tests/resources/test-owlbot/google-cloud-java/java-newapi/.repo-metadata.json b/hermetic_build/library_generation/tests/resources/test-owlbot/google-cloud-java/java-newapi/.repo-metadata.json new file mode 100644 index 0000000000..5bc5879b8b --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test-owlbot/google-cloud-java/java-newapi/.repo-metadata.json @@ -0,0 +1,16 @@ +{ + "api_shortname": "newapi", + "name_pretty": "Google New API for Test", + "product_documentation": "https://developers.google.com/", + "api_description": "The Ad Manager API enables an app to integrate with Google Ad Manager. You can read Ad Manager data and run reports using the API.", + "client_documentation": "https://cloud.google.com/java/docs/reference/ad-manager/latest/overview", + "release_level": "preview", + "transport": "http", + "language": "java", + "repo": "googleapis/google-cloud-java", + "repo_short": "java-newapi", + "distribution_name": "com.google.cloud:google-cloud-newapi", + "api_id": "newapi.googleapis.com", + "library_type": "GAPIC_AUTO", + "requires_billing": true +} From 29fdc2e0c36fb0e445f1e694bcfc959b31f6b419 Mon Sep 17 00:00:00 2001 From: Tomo Suzuki Date: Thu, 27 Nov 2025 01:58:57 +0000 Subject: [PATCH 6/8] Test versions.txt --- .../resources/test-owlbot/google-cloud-java/versions.txt | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 hermetic_build/library_generation/tests/resources/test-owlbot/google-cloud-java/versions.txt diff --git a/hermetic_build/library_generation/tests/resources/test-owlbot/google-cloud-java/versions.txt b/hermetic_build/library_generation/tests/resources/test-owlbot/google-cloud-java/versions.txt new file mode 100644 index 0000000000..4424aed41e --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test-owlbot/google-cloud-java/versions.txt @@ -0,0 +1,7 @@ +# Format: +# module:released-version:current-version + +google-cloud-java:1.36.0:1.37.0-SNAPSHOT +ad-manager:0.1.0:0.2.0-SNAPSHOT +proto-ad-manager-v1:0.1.0:0.2.0-SNAPSHOT +java-ad-manager:0.1.0:0.2.0-SNAPSHOT From 3ab2087388ac7c74ec25b80e8665397280c2223e Mon Sep 17 00:00:00 2001 From: Tomo Suzuki Date: Tue, 2 Dec 2025 11:17:42 +0000 Subject: [PATCH 7/8] formatted --- .../library_generation/owlbot/src/fix_poms.py | 6 +- .../tests/owlbot/fix_poms_unit_tests.py | 73 +++++++++++++------ 2 files changed, 56 insertions(+), 23 deletions(-) diff --git a/hermetic_build/library_generation/owlbot/src/fix_poms.py b/hermetic_build/library_generation/owlbot/src/fix_poms.py index 855b5a4b8e..1a2fc8e1ac 100644 --- a/hermetic_build/library_generation/owlbot/src/fix_poms.py +++ b/hermetic_build/library_generation/owlbot/src/fix_poms.py @@ -619,10 +619,12 @@ def main(versions_file, monorepo): if dependency_module not in versions_txt_modules: # These should already be in existing_modules if loaded from versions.txt if dependency_module in existing_modules: - versions_txt_modules[dependency_module] = existing_modules[dependency_module] + versions_txt_modules[dependency_module] = existing_modules[ + dependency_module + ] # This else block should ideally not be reached in the monorepo case. else: - versions_txt_modules[dependency_module] = module.Module( + versions_txt_modules[dependency_module] = module.Module( group_id=__proto_group_id(group_id), artifact_id=dependency_module, version=main_module.version, diff --git a/hermetic_build/library_generation/tests/owlbot/fix_poms_unit_tests.py b/hermetic_build/library_generation/tests/owlbot/fix_poms_unit_tests.py index ea5ef25cc1..c71577d91e 100644 --- a/hermetic_build/library_generation/tests/owlbot/fix_poms_unit_tests.py +++ b/hermetic_build/library_generation/tests/owlbot/fix_poms_unit_tests.py @@ -32,7 +32,6 @@ def setUp(self): def tearDown(self): os.chdir(self.start_dir) - def test_update_poms_group_id_does_not_start_with_google_correctly(self): ad_manager_resource = os.path.join(resources_dir, "java-admanager") versions_file = os.path.join(ad_manager_resource, "versions.txt") @@ -46,7 +45,6 @@ def test_update_poms_group_id_does_not_start_with_google_correctly(self): for sub_dir in sub_dirs: self.__remove_file_in_subdir(ad_manager_resource, sub_dir) - def test_add_new_library(self): # Setup test environment in a structure similar to a real repo repo_dir = os.path.join(self.temp_dir, "google-cloud-java") @@ -56,13 +54,17 @@ def test_add_new_library(self): new_library_path = os.path.join(repo_dir, new_library_dir_name) # Copy test resources to the temporary directory - shutil.copytree(os.path.join(source_resource_dir, "java-newapi"), new_library_path) + shutil.copytree( + os.path.join(source_resource_dir, "java-newapi"), new_library_path + ) # Modify the .repo-metadata.json in the temporary new library folder repo_metadata_path = os.path.join(new_library_path, ".repo-metadata.json") with open(repo_metadata_path, "r") as f: repo_metadata_content = f.read() - repo_metadata_content = repo_metadata_content.replace("google-cloud-newfolder", "google-cloud-newapi") + repo_metadata_content = repo_metadata_content.replace( + "google-cloud-newfolder", "google-cloud-newapi" + ) with open(repo_metadata_path, "w") as f: f.write(repo_metadata_content) @@ -78,7 +80,11 @@ def test_add_new_library(self): # Get initial state of versions.txt (excluding comments and empty lines) with open(dest_versions_file, "r") as f: - initial_active_lines = [line.strip() for line in f if line.strip() and not line.strip().startswith("#")] + initial_active_lines = [ + line.strip() + for line in f + if line.strip() and not line.strip().startswith("#") + ] # Execute the main function from fix_poms.py main(dest_versions_file, "true") @@ -87,20 +93,36 @@ def test_add_new_library(self): with open(dest_versions_file, "r") as f: updated_versions_content = f.readlines() - updated_active_lines = [line.strip() for line in updated_versions_content if line.strip() and not line.strip().startswith("#")] + updated_active_lines = [ + line.strip() + for line in updated_versions_content + if line.strip() and not line.strip().startswith("#") + ] - self.assertEqual(len(initial_active_lines) + 1, len(updated_active_lines), - "A single line should have been added to versions.txt") + self.assertEqual( + len(initial_active_lines) + 1, + len(updated_active_lines), + "A single line should have been added to versions.txt", + ) expected_new_line = "newapi:0.0.0:0.0.1-SNAPSHOT" - self.assertTrue(any(expected_new_line in line for line in updated_active_lines), - f"The new line '{expected_new_line}' was not found in versions.txt") + self.assertTrue( + any(expected_new_line in line for line in updated_active_lines), + f"The new line '{expected_new_line}' was not found in versions.txt", + ) - filtered_updated_lines = [line for line in updated_active_lines if expected_new_line not in line] - self.assertCountEqual(initial_active_lines, filtered_updated_lines, - "Existing lines in versions.txt should be unaffected.") - self.assertFalse(any("google-cloud-newapi" in line for line in updated_versions_content), - "The artifactId 'google-cloud-newapi' should not be in versions.txt") + filtered_updated_lines = [ + line for line in updated_active_lines if expected_new_line not in line + ] + self.assertCountEqual( + initial_active_lines, + filtered_updated_lines, + "Existing lines in versions.txt should be unaffected.", + ) + self.assertFalse( + any("google-cloud-newapi" in line for line in updated_versions_content), + "The artifactId 'google-cloud-newapi' should not be in versions.txt", + ) # --- Verify generated pom.xml files --- expected_poms = [ @@ -112,17 +134,26 @@ def test_add_new_library(self): ] for pom_path in expected_poms: full_pom_path = os.path.join(new_library_path, pom_path) - self.assertTrue(os.path.exists(full_pom_path), f"Expected POM file not found: {full_pom_path}") + self.assertTrue( + os.path.exists(full_pom_path), + f"Expected POM file not found: {full_pom_path}", + ) with open(full_pom_path, "r") as f: pom_content = f.read() - self.assertIn("", pom_content, - f"x-version-update tag in {pom_path} does not reference 'newapi'") + self.assertIn( + "", + pom_content, + f"x-version-update tag in {pom_path} does not reference 'newapi'", + ) # Ensure the artifactId is not changed to the release_please_key if pom_path == "google-cloud-newapi/pom.xml": - self.assertIn("google-cloud-newapi", pom_content) + self.assertIn( + "google-cloud-newapi", pom_content + ) if pom_path == "proto-google-cloud-newapi-v1/pom.xml": - self.assertIn("proto-google-cloud-newapi-v1", pom_content) - + self.assertIn( + "proto-google-cloud-newapi-v1", pom_content + ) @classmethod def __copy__golden(cls, base_dir: str, subdir: str): From 3a5e6bc58b7f8ad93caf32b025624f9b75f5427f Mon Sep 17 00:00:00 2001 From: Tomo Suzuki Date: Wed, 3 Dec 2025 03:44:06 +0000 Subject: [PATCH 8/8] Not touching the owlbot.py golden file --- .../tests/resources/goldens/owlbot-golden.py | 29 +++++++++---------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/hermetic_build/library_generation/tests/resources/goldens/owlbot-golden.py b/hermetic_build/library_generation/tests/resources/goldens/owlbot-golden.py index 7559eaf034..2ba11e6bba 100644 --- a/hermetic_build/library_generation/tests/resources/goldens/owlbot-golden.py +++ b/hermetic_build/library_generation/tests/resources/goldens/owlbot-golden.py @@ -21,19 +21,16 @@ s.move(library) s.remove_staging_dirs() -java.common_templates( - monorepo=True, - excludes=[ - ".github/*", - ".kokoro/*", - "samples/*", - "CODE_OF_CONDUCT.md", - "CONTRIBUTING.md", - "LICENSE", - "SECURITY.md", - "java.header", - "license-checks.xml", - "renovate.json", - ".gitignore", - ], -) +java.common_templates(monorepo=True, excludes=[ + ".github/*", + ".kokoro/*", + "samples/*", + "CODE_OF_CONDUCT.md", + "CONTRIBUTING.md", + "LICENSE", + "SECURITY.md", + "java.header", + "license-checks.xml", + "renovate.json", + ".gitignore" +]) \ No newline at end of file