diff --git a/hermetic_build/library_generation/owlbot/src/fix_poms.py b/hermetic_build/library_generation/owlbot/src/fix_poms.py index 9ee7514a4d..1a2fc8e1ac 100644 --- a/hermetic_build/library_generation/owlbot/src/fix_poms.py +++ b/hermetic_build/library_generation/owlbot/src/fix_poms.py @@ -358,8 +358,11 @@ def main(versions_file, monorepo): with open(".repo-metadata.json", "r") as fp: repo_metadata = json.load(fp) group_id, artifact_id = repo_metadata["distribution_name"].split(":") + # Use api_shortname from repo_metadata as the key for versions.txt + release_please_key = repo_metadata["api_shortname"] name = repo_metadata["name_pretty"] existing_modules = load_versions(versions_file, group_id) + original_modules = set(existing_modules.keys()) print(f"monorepo? {monorepo}") # extra modules that need to be manages in versions.txt @@ -456,6 +459,7 @@ def main(versions_file, monorepo): parent_module=parent_module, main_module=main_module, monorepo=monorepo, + release_please_key=release_please_key, ) if ( path not in excluded_dependencies_list @@ -498,6 +502,7 @@ def main(versions_file, monorepo): main_module=main_module, proto_module=existing_modules[proto_artifact_id], monorepo=monorepo, + release_please_key=release_please_key, ) if ( path not in excluded_dependencies_list @@ -549,6 +554,7 @@ def main(versions_file, monorepo): proto_modules=proto_modules, grpc_modules=grpc_modules, monorepo=monorepo, + release_please_key=release_please_key, ) if os.path.isfile(f"{artifact_id}-bom/pom.xml"): @@ -567,6 +573,7 @@ def main(versions_file, monorepo): main_module=main_module, monorepo=monorepo, monorepo_version=monorepo_version, + release_please_key=release_please_key, ) if os.path.isfile("pom.xml"): @@ -584,24 +591,51 @@ def main(versions_file, monorepo): name=name, monorepo=monorepo, monorepo_version=monorepo_version, + release_please_key=release_please_key, ) print(f"updating modules in {versions_file}") - existing_modules.pop(parent_artifact_id) - # add extra modules to versions.txt + # existing_modules contains all components, needed for pom.xml rendering. + + versions_txt_modules = {} + # Start with all modules that were in the original versions.txt + for key in original_modules: + if key in existing_modules: + versions_txt_modules[key] = existing_modules[key] + + # If this is a new library (release_please_key was not in original_modules), + # add the single consolidated entry to our versions.txt dictionary. + if release_please_key not in original_modules: + versions_txt_modules[release_please_key] = module.Module( + group_id=group_id, + artifact_id=release_please_key, + version=main_module.version, + release_version=main_module.release_version, + ) + + # Add extra managed modules if they aren't already included for dependency_module in extra_managed_modules: - if dependency_module not in existing_modules: - existing_modules[dependency_module] = module.Module( - group_id=__proto_group_id(group_id), - artifact_id=dependency_module, - version=main_module.version, - release_version=main_module.release_version, - ) + if dependency_module not in versions_txt_modules: + # These should already be in existing_modules if loaded from versions.txt + if dependency_module in existing_modules: + versions_txt_modules[dependency_module] = existing_modules[ + dependency_module + ] + # This else block should ideally not be reached in the monorepo case. + else: + versions_txt_modules[dependency_module] = module.Module( + group_id=__proto_group_id(group_id), + artifact_id=dependency_module, + version=main_module.version, + release_version=main_module.release_version, + ) + + # Render versions.txt using the specially prepared versions_txt_modules templates.render( template_name="versions.txt.j2", output_name=versions_file, - modules=existing_modules.values(), + modules=versions_txt_modules.values(), ) diff --git a/hermetic_build/library_generation/owlbot/templates/poms/bom_pom.xml.j2 b/hermetic_build/library_generation/owlbot/templates/poms/bom_pom.xml.j2 index ddcef5226f..dcd930c1ae 100644 --- a/hermetic_build/library_generation/owlbot/templates/poms/bom_pom.xml.j2 +++ b/hermetic_build/library_generation/owlbot/templates/poms/bom_pom.xml.j2 @@ -3,7 +3,7 @@ 4.0.0 {{main_module.group_id}} {{main_module.artifact_id}}-bom - {{main_module.version}} + {{main_module.version}} pom {% if monorepo -%} @@ -34,7 +34,7 @@ {{module.group_id}} {{module.artifact_id}} - {{module.version}} + {{module.version}} {% endfor %} diff --git a/hermetic_build/library_generation/owlbot/templates/poms/cloud_pom.xml.j2 b/hermetic_build/library_generation/owlbot/templates/poms/cloud_pom.xml.j2 index d0ae8cd7c5..3087201693 100644 --- a/hermetic_build/library_generation/owlbot/templates/poms/cloud_pom.xml.j2 +++ b/hermetic_build/library_generation/owlbot/templates/poms/cloud_pom.xml.j2 @@ -3,7 +3,7 @@ 4.0.0 {{module.group_id}} {{module.artifact_id}} - {{module.version}} + {{module.version}} jar Google {{name}} {%- if not monorepo %} @@ -13,7 +13,7 @@ {{parent_module.group_id}} {{parent_module.artifact_id}} - {{parent_module.version}} + {{parent_module.version}} {{module.artifact_id}} diff --git a/hermetic_build/library_generation/owlbot/templates/poms/grpc_pom.xml.j2 b/hermetic_build/library_generation/owlbot/templates/poms/grpc_pom.xml.j2 index 514861e7a7..4390cf92c0 100644 --- a/hermetic_build/library_generation/owlbot/templates/poms/grpc_pom.xml.j2 +++ b/hermetic_build/library_generation/owlbot/templates/poms/grpc_pom.xml.j2 @@ -4,13 +4,13 @@ 4.0.0 {{module.group_id}} {{module.artifact_id}} - {{module.version}} + {{module.version}} {{module.artifact_id}} GRPC library for {{main_module.artifact_id}} {{parent_module.group_id}} {{parent_module.artifact_id}} - {{parent_module.version}} + {{parent_module.version}} diff --git a/hermetic_build/library_generation/owlbot/templates/poms/parent_pom.xml.j2 b/hermetic_build/library_generation/owlbot/templates/poms/parent_pom.xml.j2 index dcf922340e..2b87c0f601 100644 --- a/hermetic_build/library_generation/owlbot/templates/poms/parent_pom.xml.j2 +++ b/hermetic_build/library_generation/owlbot/templates/poms/parent_pom.xml.j2 @@ -4,7 +4,7 @@ {{main_module.group_id}} {{main_module.artifact_id}}-parent pom - {{main_module.version}} + {{main_module.version}} Google {{name}} Parent Java idiomatic client for Google Cloud Platform services. @@ -37,7 +37,7 @@ {% for module in modules %} {{module.group_id}} {{module.artifact_id}} - {{module.version}} + {{module.version}} {% endfor %} diff --git a/hermetic_build/library_generation/owlbot/templates/poms/proto_pom.xml.j2 b/hermetic_build/library_generation/owlbot/templates/poms/proto_pom.xml.j2 index 886cd02663..11869120ec 100644 --- a/hermetic_build/library_generation/owlbot/templates/poms/proto_pom.xml.j2 +++ b/hermetic_build/library_generation/owlbot/templates/poms/proto_pom.xml.j2 @@ -4,13 +4,13 @@ 4.0.0 {{module.group_id}} {{module.artifact_id}} - {{module.version}} + {{module.version}} {{module.artifact_id}} Proto library for {{main_module.artifact_id}} {{parent_module.group_id}} {{parent_module.artifact_id}} - {{parent_module.version}} + {{parent_module.version}} diff --git a/hermetic_build/library_generation/tests/owlbot/fix_poms_unit_tests.py b/hermetic_build/library_generation/tests/owlbot/fix_poms_unit_tests.py index 2e66454827..c71577d91e 100644 --- a/hermetic_build/library_generation/tests/owlbot/fix_poms_unit_tests.py +++ b/hermetic_build/library_generation/tests/owlbot/fix_poms_unit_tests.py @@ -14,6 +14,8 @@ import os import shutil import unittest +import tempfile +from pathlib import Path from library_generation.owlbot.src.fix_poms import main from library_generation.tests.compare_poms import compare_pom_in_subdir @@ -22,6 +24,14 @@ class FixPomsTest(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.mkdtemp() + self.start_dir = os.getcwd() + os.chdir(self.temp_dir) + + def tearDown(self): + os.chdir(self.start_dir) + def test_update_poms_group_id_does_not_start_with_google_correctly(self): ad_manager_resource = os.path.join(resources_dir, "java-admanager") versions_file = os.path.join(ad_manager_resource, "versions.txt") @@ -35,6 +45,116 @@ def test_update_poms_group_id_does_not_start_with_google_correctly(self): for sub_dir in sub_dirs: self.__remove_file_in_subdir(ad_manager_resource, sub_dir) + def test_add_new_library(self): + # Setup test environment in a structure similar to a real repo + repo_dir = os.path.join(self.temp_dir, "google-cloud-java") + os.makedirs(repo_dir) + source_resource_dir = os.path.join(resources_dir, "google-cloud-java") + new_library_dir_name = "java-newapi" + new_library_path = os.path.join(repo_dir, new_library_dir_name) + + # Copy test resources to the temporary directory + shutil.copytree( + os.path.join(source_resource_dir, "java-newapi"), new_library_path + ) + + # Modify the .repo-metadata.json in the temporary new library folder + repo_metadata_path = os.path.join(new_library_path, ".repo-metadata.json") + with open(repo_metadata_path, "r") as f: + repo_metadata_content = f.read() + repo_metadata_content = repo_metadata_content.replace( + "google-cloud-newfolder", "google-cloud-newapi" + ) + with open(repo_metadata_path, "w") as f: + f.write(repo_metadata_content) + + source_versions_file = os.path.join(source_resource_dir, "versions.txt") + dest_versions_file = os.path.join(repo_dir, "versions.txt") + shutil.copyfile(source_versions_file, dest_versions_file) + + # Create dummy directories for proto and grpc modules. + # In the full generation process, `generate_library.sh` would create these. + os.makedirs(os.path.join(new_library_path, "proto-google-cloud-newapi-v1")) + os.makedirs(os.path.join(new_library_path, "grpc-google-cloud-newapi-v1")) + os.chdir(new_library_path) + + # Get initial state of versions.txt (excluding comments and empty lines) + with open(dest_versions_file, "r") as f: + initial_active_lines = [ + line.strip() + for line in f + if line.strip() and not line.strip().startswith("#") + ] + + # Execute the main function from fix_poms.py + main(dest_versions_file, "true") + + # --- Verify versions.txt --- + with open(dest_versions_file, "r") as f: + updated_versions_content = f.readlines() + + updated_active_lines = [ + line.strip() + for line in updated_versions_content + if line.strip() and not line.strip().startswith("#") + ] + + self.assertEqual( + len(initial_active_lines) + 1, + len(updated_active_lines), + "A single line should have been added to versions.txt", + ) + + expected_new_line = "newapi:0.0.0:0.0.1-SNAPSHOT" + self.assertTrue( + any(expected_new_line in line for line in updated_active_lines), + f"The new line '{expected_new_line}' was not found in versions.txt", + ) + + filtered_updated_lines = [ + line for line in updated_active_lines if expected_new_line not in line + ] + self.assertCountEqual( + initial_active_lines, + filtered_updated_lines, + "Existing lines in versions.txt should be unaffected.", + ) + self.assertFalse( + any("google-cloud-newapi" in line for line in updated_versions_content), + "The artifactId 'google-cloud-newapi' should not be in versions.txt", + ) + + # --- Verify generated pom.xml files --- + expected_poms = [ + "pom.xml", + "google-cloud-newapi/pom.xml", + "google-cloud-newapi-bom/pom.xml", + "proto-google-cloud-newapi-v1/pom.xml", + "grpc-google-cloud-newapi-v1/pom.xml", + ] + for pom_path in expected_poms: + full_pom_path = os.path.join(new_library_path, pom_path) + self.assertTrue( + os.path.exists(full_pom_path), + f"Expected POM file not found: {full_pom_path}", + ) + with open(full_pom_path, "r") as f: + pom_content = f.read() + self.assertIn( + "", + pom_content, + f"x-version-update tag in {pom_path} does not reference 'newapi'", + ) + # Ensure the artifactId is not changed to the release_please_key + if pom_path == "google-cloud-newapi/pom.xml": + self.assertIn( + "google-cloud-newapi", pom_content + ) + if pom_path == "proto-google-cloud-newapi-v1/pom.xml": + self.assertIn( + "proto-google-cloud-newapi-v1", pom_content + ) + @classmethod def __copy__golden(cls, base_dir: str, subdir: str): golden = os.path.join(base_dir, subdir, "pom-golden.xml") diff --git a/hermetic_build/library_generation/tests/resources/test-owlbot/google-cloud-java/java-newapi/.repo-metadata.json b/hermetic_build/library_generation/tests/resources/test-owlbot/google-cloud-java/java-newapi/.repo-metadata.json new file mode 100644 index 0000000000..5bc5879b8b --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test-owlbot/google-cloud-java/java-newapi/.repo-metadata.json @@ -0,0 +1,16 @@ +{ + "api_shortname": "newapi", + "name_pretty": "Google New API for Test", + "product_documentation": "https://developers.google.com/", + "api_description": "The Ad Manager API enables an app to integrate with Google Ad Manager. You can read Ad Manager data and run reports using the API.", + "client_documentation": "https://cloud.google.com/java/docs/reference/ad-manager/latest/overview", + "release_level": "preview", + "transport": "http", + "language": "java", + "repo": "googleapis/google-cloud-java", + "repo_short": "java-newapi", + "distribution_name": "com.google.cloud:google-cloud-newapi", + "api_id": "newapi.googleapis.com", + "library_type": "GAPIC_AUTO", + "requires_billing": true +} diff --git a/hermetic_build/library_generation/tests/resources/test-owlbot/google-cloud-java/versions.txt b/hermetic_build/library_generation/tests/resources/test-owlbot/google-cloud-java/versions.txt new file mode 100644 index 0000000000..4424aed41e --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test-owlbot/google-cloud-java/versions.txt @@ -0,0 +1,7 @@ +# Format: +# module:released-version:current-version + +google-cloud-java:1.36.0:1.37.0-SNAPSHOT +ad-manager:0.1.0:0.2.0-SNAPSHOT +proto-ad-manager-v1:0.1.0:0.2.0-SNAPSHOT +java-ad-manager:0.1.0:0.2.0-SNAPSHOT diff --git a/hermetic_build/library_generation/tests/resources/test-owlbot/java-admanager/versions.txt b/hermetic_build/library_generation/tests/resources/test-owlbot/java-admanager/versions.txt index 3f073728f3..930a79a07e 100644 --- a/hermetic_build/library_generation/tests/resources/test-owlbot/java-admanager/versions.txt +++ b/hermetic_build/library_generation/tests/resources/test-owlbot/java-admanager/versions.txt @@ -1,6 +1,7 @@ # Format: # module:released-version:current-version +admanager:0.0.0:0.0.1-SNAPSHOT google-cloud-java:1.36.0:1.37.0-SNAPSHOT -ad-manager:0.1.0:0.2.0-SNAPSHOT proto-ad-manager-v1:0.1.0:0.2.0-SNAPSHOT +ad-manager:0.1.0:0.2.0-SNAPSHOT