chore: refactor utility functions (#2557)

In this PR:
- Code Refactor 
  - Refactor pom generation functions to a separate file.
  - Refactor monorepo post processing functions to a separate file.
  - Refactor unit tests to separate files.
- Expose CLI tool used in google-cloud-java
  - Generate root pom.xml
  - Generate gapic-libraries-bom/pom.xml

To use these tools in google-cloud-java (through docker container):
```
python library_generation/cli/generate_monorepo_root_pom.py generate --repository-path=.

python library_generation/cli/generate_monorepo_gapic_bom.py generate --repository-path=. --versions-file=./versions.txt
```

Author: JoeWang1127

library_generation/cli/generate_monorepo_gapic_bom.py

@@ -0,0 +1,50 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import click as click
+
+from library_generation.utils.pom_generator import generate_gapic_bom
+from library_generation.utils.pom_generator import generate_root_pom
+
+
+@click.group(invoke_without_command=False)
+@click.pass_context
+@click.version_option(message="%(version)s")
+def main(ctx):
+    pass
+
+
+@main.command()
+@click.option(
+    "--repository-path",
+    required=True,
+    type=str,
+    help="""
+    Path to which the generated pom.xml goes.
+    """,
+)
+@click.option(
+    "--versions-file",
+    required=True,
+    type=str,
+    help="""
+    The file containing version of libraries.
+    Throw FileNotFoundError if the file doesn't exist.
+    """,
+)
+def generate(repository_path: str, versions_file: str) -> None:
+    generate_gapic_bom(repository_path=repository_path, versions_file=versions_file)
+
+
+if __name__ == "__main__":
+    main()

library_generation/cli/generate_monorepo_root_pom.py

@@ -0,0 +1,39 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import click as click
+from library_generation.utils.pom_generator import generate_root_pom
+
+
+@click.group(invoke_without_command=False)
+@click.pass_context
+@click.version_option(message="%(version)s")
+def main(ctx):
+    pass
+
+
+@main.command()
+@click.option(
+    "--repository-path",
+    required=True,
+    type=str,
+    help="""
+    Path to which the generated pom.xml goes.
+    """,
+)
+def generate(repository_path: str) -> None:
+    generate_root_pom(repository_path=repository_path)
+
+
+if __name__ == "__main__":
+    main()

library_generation/generate_repo.py

@@ -18,6 +18,7 @@
 import os
 from library_generation.generate_composed_library import generate_composed_library
 from library_generation.model.generation_config import from_yaml
+from library_generation.utils.monorepo_postprocessor import monorepo_postprocessing
 
 
 @click.group(invoke_without_command=False)
@@ -112,7 +113,7 @@ def generate_from_yaml(
     if not config.is_monorepo:
         return
 
-    util.monorepo_postprocessing(
+    monorepo_postprocessing(
         repository_path=repository_path, versions_file=repo_config.versions_file
     )
 

library_generation/test/test_utils.py

@@ -0,0 +1,40 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import unittest
+from difflib import unified_diff
+from pathlib import Path
+
+from typing import List
+
+
+class FileComparator(unittest.TestCase):
+    def compare_files(self, expect: str, actual: str):
+        with open(expect, "r") as f:
+            expected_lines = f.readlines()
+        with open(actual, "r") as f:
+            actual_lines = f.readlines()
+
+        diff = list(unified_diff(expected_lines, actual_lines))
+        self.assertEqual(
+            first=[], second=diff, msg="Unexpected file contents:\n" + "".join(diff)
+        )
+
+
+def cleanup(files: List[str]):
+    for file in files:
+        path = Path(file).resolve()
+        if path.is_file():
+            path.unlink()
+        elif path.is_dir():
+            path.rmdir()

library_generation/test/unit_tests.py renamed to library_generation/test/utilities_unit_tests.py

@@ -21,23 +21,23 @@
 import io
 import contextlib
 from pathlib import Path
-from difflib import unified_diff
-
-from typing import List
 from parameterized import parameterized
 from library_generation import utilities as util
 from library_generation.model.gapic_config import GapicConfig
 from library_generation.model.generation_config import GenerationConfig
 from library_generation.model.gapic_inputs import parse as parse_build_file
 from library_generation.model.generation_config import from_yaml
 from library_generation.model.library_config import LibraryConfig
+from library_generation.test.test_utils import FileComparator
+from library_generation.test.test_utils import cleanup
 from library_generation.utilities import find_versioned_proto_path
 from library_generation.utilities import get_file_paths
 
 script_dir = os.path.dirname(os.path.realpath(__file__))
 resources_dir = os.path.join(script_dir, "resources")
 build_file = Path(os.path.join(resources_dir, "misc")).resolve()
 test_config_dir = Path(os.path.join(resources_dir, "test-config")).resolve()
+file_comparator = FileComparator()
 library_1 = LibraryConfig(
     api_shortname="baremetalsolution",
     name_pretty="Bare Metal Solution",
@@ -391,18 +391,6 @@ def test_remove_version_from_returns_self(self):
             "google/cloud/aiplatform", util.remove_version_from(proto_path)
         )
 
-    def test_get_version_from_returns_current(self):
-        versions_file = f"{resources_dir}/misc/versions.txt"
-        artifact = "gax-grpc"
-        self.assertEqual(
-            "2.33.1-SNAPSHOT", util.get_version_from(versions_file, artifact)
-        )
-
-    def test_get_version_from_returns_released(self):
-        versions_file = f"{resources_dir}/misc/versions.txt"
-        artifact = "gax-grpc"
-        self.assertEqual("2.34.0", util.get_version_from(versions_file, artifact, True))
-
     def test_get_library_returns_library_name(self):
         self.assertEqual("bare-metal-solution", util.get_library_name(library_1))
 
@@ -414,32 +402,32 @@ def test_generate_prerequisite_files_non_monorepo_success(self):
             num_libraries=1, library_type="GAPIC_COMBO"
         )
 
-        self.__compare_files(
+        file_comparator.compare_files(
             f"{library_path}/.repo-metadata.json",
             f"{library_path}/.repo-metadata-non-monorepo-golden.json",
         )
         # since this is a single library, we treat this as HW repository,
         # meaning that the owlbot yaml will be inside a .github folder
-        self.__compare_files(
+        file_comparator.compare_files(
             f"{library_path}/.github/.OwlBot.yaml",
             f"{library_path}/.OwlBot-golden.yaml",
         )
-        self.__compare_files(
+        file_comparator.compare_files(
             f"{library_path}/owlbot.py", f"{library_path}/owlbot-golden.py"
         )
 
     def test_generate_prerequisite_files_monorepo_success(self):
         library_path = self.__setup_prerequisite_files(num_libraries=2)
 
-        self.__compare_files(
+        file_comparator.compare_files(
             f"{library_path}/.repo-metadata.json",
             f"{library_path}/.repo-metadata-monorepo-golden.json",
         )
-        self.__compare_files(
+        file_comparator.compare_files(
             f"{library_path}/.OwlBot.yaml",
             f"{library_path}/.OwlBot-golden.yaml",
         )
-        self.__compare_files(
+        file_comparator.compare_files(
             f"{library_path}/owlbot.py", f"{library_path}/owlbot-golden.py"
         )
 
@@ -488,37 +476,6 @@ def test_prepare_repo_split_repo_success(self):
         library_path = sorted([Path(key).name for key in repo_config.libraries])
         self.assertEqual(["misc"], library_path)
 
-    def test_monorepo_postprocessing_valid_repository_success(self):
-        repository_path = f"{resources_dir}/test_monorepo_postprocessing"
-        versions_file = f"{repository_path}/versions.txt"
-        files = [
-            f"{repository_path}/pom.xml",
-            f"{repository_path}/gapic-libraries-bom/pom.xml",
-        ]
-        self.__cleanup(files)
-        util.monorepo_postprocessing(
-            repository_path=repository_path, versions_file=versions_file
-        )
-        self.__compare_files(
-            expect=f"{repository_path}/pom-golden.xml",
-            actual=f"{repository_path}/pom.xml",
-        )
-        self.__compare_files(
-            expect=f"{repository_path}/gapic-libraries-bom/pom-golden.xml",
-            actual=f"{repository_path}/gapic-libraries-bom/pom.xml",
-        )
-
-    def __compare_files(self, expect: str, actual: str):
-        with open(expect, "r") as f:
-            expected_lines = f.readlines()
-        with open(actual, "r") as f:
-            actual_lines = f.readlines()
-
-        diff = list(unified_diff(expected_lines, actual_lines))
-        self.assertEqual(
-            first=[], second=diff, msg="Unexpected file contents:\n" + "".join(diff)
-        )
-
     def __setup_prerequisite_files(
         self, num_libraries: int, library_type: str = "GAPIC_AUTO"
     ) -> str:
@@ -528,7 +485,7 @@ def __setup_prerequisite_files(
             f"{library_path}/.OwlBot.yaml",
             f"{library_path}/owlbot.py",
         ]
-        self.__cleanup(files)
+        cleanup(files)
         config = self.__get_a_gen_config(num_libraries, library_type=library_type)
         proto_path = "google/cloud/baremetalsolution/v2"
         transport = "grpc"
@@ -593,15 +550,6 @@ def __get_a_gen_config(
             libraries=libraries,
         )
 
-    @staticmethod
-    def __cleanup(files: List[str]):
-        for file in files:
-            path = Path(file).resolve()
-            if path.is_file():
-                path.unlink()
-            elif path.is_dir():
-                path.rmdir()
-
 
 if __name__ == "__main__":
     unittest.main()

library_generation/test/utils/monorepo_postprocessor_unit_tests.py

@@ -0,0 +1,45 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+import unittest
+
+from library_generation.test.test_utils import FileComparator
+from library_generation.test.test_utils import cleanup
+from library_generation.utils.monorepo_postprocessor import monorepo_postprocessing
+
+script_dir = os.path.dirname(os.path.realpath(__file__))
+resources_dir = os.path.join(script_dir, "..", "resources")
+file_comparator = FileComparator()
+
+
+class MonorepoPostprocessorTest(unittest.TestCase):
+    def test_monorepo_postprocessing_valid_repository_success(self):
+        repository_path = f"{resources_dir}/test_monorepo_postprocessing"
+        versions_file = f"{repository_path}/versions.txt"
+        files = [
+            f"{repository_path}/pom.xml",
+            f"{repository_path}/gapic-libraries-bom/pom.xml",
+        ]
+        cleanup(files)
+        monorepo_postprocessing(
+            repository_path=repository_path, versions_file=versions_file
+        )
+        file_comparator.compare_files(
+            expect=f"{repository_path}/pom-golden.xml",
+            actual=f"{repository_path}/pom.xml",
+        )
+        file_comparator.compare_files(
+            expect=f"{repository_path}/gapic-libraries-bom/pom-golden.xml",
+            actual=f"{repository_path}/gapic-libraries-bom/pom.xml",
+        )

library_generation/test/utils/pom_generator_unit_tests.py

@@ -0,0 +1,32 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+import unittest
+
+from library_generation.utils.pom_generator import get_version_from
+
+script_dir = os.path.dirname(os.path.realpath(__file__))
+resources_dir = os.path.join(script_dir, "..", "resources")
+
+
+class PomGeneratorTest(unittest.TestCase):
+    def test_get_version_from_returns_current(self):
+        versions_file = f"{resources_dir}/misc/versions.txt"
+        artifact = "gax-grpc"
+        self.assertEqual("2.33.1-SNAPSHOT", get_version_from(versions_file, artifact))
+
+    def test_get_version_from_returns_released(self):
+        versions_file = f"{resources_dir}/misc/versions.txt"
+        artifact = "gax-grpc"
+        self.assertEqual("2.34.0", get_version_from(versions_file, artifact, True))