diff --git a/hermetic_build/common/cli/get_changed_libraries.py b/hermetic_build/common/cli/get_changed_libraries.py index f9b3d40013..a1eeecba99 100644 --- a/hermetic_build/common/cli/get_changed_libraries.py +++ b/hermetic_build/common/cli/get_changed_libraries.py @@ -15,7 +15,7 @@ import click as click -from common.model.generation_config import from_yaml +from common.model.generation_config import GenerationConfig from common.utils.generation_config_comparator import compare_config @@ -69,8 +69,8 @@ def create( "current-generation-config-path." ) config_change = compare_config( - baseline_config=from_yaml(baseline_generation_config_path), - current_config=from_yaml(current_generation_config_path), + baseline_config=GenerationConfig.from_yaml(baseline_generation_config_path), + current_config=GenerationConfig.from_yaml(current_generation_config_path), ) click.echo(",".join(config_change.get_changed_libraries())) diff --git a/hermetic_build/common/model/generation_config.py b/hermetic_build/common/model/generation_config.py index c8aaf6b2ad..fabbde41cc 100644 --- a/hermetic_build/common/model/generation_config.py +++ b/hermetic_build/common/model/generation_config.py @@ -105,77 +105,77 @@ def __validate(self) -> None: ) seen_library_names[library_name] = library.name_pretty + @staticmethod + def from_yaml(path_to_yaml: str) -> "GenerationConfig": + """ + Parses a yaml located in path_to_yaml. + :param path_to_yaml: the path to the configuration file + :return the parsed configuration represented by the "model" classes + """ + with open(path_to_yaml, "r") as file_stream: + config = yaml.safe_load(file_stream) + + libraries = _required(config, "libraries", REPO_LEVEL_PARAMETER) + if not libraries: + raise ValueError(f"Library is None in {path_to_yaml}.") + + parsed_libraries = list() + for library in libraries: + gapics = _required(library, "GAPICs") + + parsed_gapics = list() + if not gapics: + raise ValueError(f"GAPICs is None in {library}.") + for gapic in gapics: + proto_path = _required(gapic, "proto_path", GAPIC_LEVEL_PARAMETER) + new_gapic = GapicConfig(proto_path) + parsed_gapics.append(new_gapic) + + new_library = LibraryConfig( + api_shortname=_required(library, "api_shortname"), + api_description=_required(library, "api_description"), + name_pretty=_required(library, "name_pretty"), + product_documentation=_required(library, "product_documentation"), + gapic_configs=parsed_gapics, + library_type=_optional(library, "library_type", "GAPIC_AUTO"), + release_level=_optional(library, "release_level", "preview"), + api_id=_optional(library, "api_id", None), + api_reference=_optional(library, "api_reference", None), + codeowner_team=_optional(library, "codeowner_team", None), + excluded_poms=_optional(library, "excluded_poms", None), + excluded_dependencies=_optional(library, "excluded_dependencies", None), + client_documentation=_optional(library, "client_documentation", None), + distribution_name=_optional(library, "distribution_name", None), + googleapis_commitish=_optional(library, "googleapis_commitish", None), + group_id=_optional(library, "group_id", "com.google.cloud"), + issue_tracker=_optional(library, "issue_tracker", None), + library_name=_optional(library, "library_name", None), + rest_documentation=_optional(library, "rest_documentation", None), + rpc_documentation=_optional(library, "rpc_documentation", None), + cloud_api=_optional(library, "cloud_api", True), + requires_billing=_optional(library, "requires_billing", True), + extra_versioned_modules=_optional( + library, "extra_versioned_modules", None + ), + recommended_package=_optional(library, "recommended_package", None), + min_java_version=_optional(library, "min_java_version", None), + transport=_optional(library, "transport", None), + ) + parsed_libraries.append(new_library) -def from_yaml(path_to_yaml: str) -> GenerationConfig: - """ - Parses a yaml located in path_to_yaml. - :param path_to_yaml: the path to the configuration file - :return the parsed configuration represented by the "model" classes - """ - with open(path_to_yaml, "r") as file_stream: - config = yaml.safe_load(file_stream) - - libraries = __required(config, "libraries", REPO_LEVEL_PARAMETER) - if not libraries: - raise ValueError(f"Library is None in {path_to_yaml}.") - - parsed_libraries = list() - for library in libraries: - gapics = __required(library, "GAPICs") - - parsed_gapics = list() - if not gapics: - raise ValueError(f"GAPICs is None in {library}.") - for gapic in gapics: - proto_path = __required(gapic, "proto_path", GAPIC_LEVEL_PARAMETER) - new_gapic = GapicConfig(proto_path) - parsed_gapics.append(new_gapic) - - new_library = LibraryConfig( - api_shortname=__required(library, "api_shortname"), - api_description=__required(library, "api_description"), - name_pretty=__required(library, "name_pretty"), - product_documentation=__required(library, "product_documentation"), - gapic_configs=parsed_gapics, - library_type=__optional(library, "library_type", "GAPIC_AUTO"), - release_level=__optional(library, "release_level", "preview"), - api_id=__optional(library, "api_id", None), - api_reference=__optional(library, "api_reference", None), - codeowner_team=__optional(library, "codeowner_team", None), - excluded_poms=__optional(library, "excluded_poms", None), - excluded_dependencies=__optional(library, "excluded_dependencies", None), - client_documentation=__optional(library, "client_documentation", None), - distribution_name=__optional(library, "distribution_name", None), - googleapis_commitish=__optional(library, "googleapis_commitish", None), - group_id=__optional(library, "group_id", "com.google.cloud"), - issue_tracker=__optional(library, "issue_tracker", None), - library_name=__optional(library, "library_name", None), - rest_documentation=__optional(library, "rest_documentation", None), - rpc_documentation=__optional(library, "rpc_documentation", None), - cloud_api=__optional(library, "cloud_api", True), - requires_billing=__optional(library, "requires_billing", True), - extra_versioned_modules=__optional( - library, "extra_versioned_modules", None + parsed_config = GenerationConfig( + googleapis_commitish=_required( + config, "googleapis_commitish", REPO_LEVEL_PARAMETER ), - recommended_package=__optional(library, "recommended_package", None), - min_java_version=__optional(library, "min_java_version", None), - transport=__optional(library, "transport", None), + gapic_generator_version=_optional(config, GAPIC_GENERATOR_VERSION, None), + libraries_bom_version=_optional(config, LIBRARIES_BOM_VERSION, None), + libraries=parsed_libraries, ) - parsed_libraries.append(new_library) - - parsed_config = GenerationConfig( - googleapis_commitish=__required( - config, "googleapis_commitish", REPO_LEVEL_PARAMETER - ), - gapic_generator_version=__optional(config, GAPIC_GENERATOR_VERSION, None), - libraries_bom_version=__optional(config, LIBRARIES_BOM_VERSION, None), - libraries=parsed_libraries, - ) - return parsed_config + return parsed_config -def __required(config: dict, key: str, level: str = LIBRARY_LEVEL_PARAMETER): +def _required(config: dict, key: str, level: str = LIBRARY_LEVEL_PARAMETER): if key not in config: message = ( f"{level}, {key}, is not found in {config} in yaml." @@ -186,7 +186,7 @@ def __required(config: dict, key: str, level: str = LIBRARY_LEVEL_PARAMETER): return config[key] -def __optional(config: dict, key: str, default: any): +def _optional(config: dict, key: str, default: any): if key not in config: return default return config[key] diff --git a/hermetic_build/common/model/library_config.py b/hermetic_build/common/model/library_config.py index b4156a6408..9b9d31b810 100644 --- a/hermetic_build/common/model/library_config.py +++ b/hermetic_build/common/model/library_config.py @@ -82,6 +82,15 @@ def __init__( self.distribution_name = self.__get_distribution_name(distribution_name) self.transport = self.__validate_transport(transport) + def set_gapic_configs(self, gapic_configs: list[GapicConfig]) -> None: + """ + Sets the gapic_configs for the library. + + Args: + gapic_configs: The new list of GapicConfig objects. + """ + self.gapic_configs = gapic_configs + def get_library_name(self) -> str: """ Return the library name of a given LibraryConfig object diff --git a/hermetic_build/common/tests/model/generation_config_unit_tests.py b/hermetic_build/common/tests/model/generation_config_unit_tests.py index e769a9a723..2b89a1acc8 100644 --- a/hermetic_build/common/tests/model/generation_config_unit_tests.py +++ b/hermetic_build/common/tests/model/generation_config_unit_tests.py @@ -14,7 +14,7 @@ import os import unittest from pathlib import Path -from common.model.generation_config import from_yaml, GenerationConfig +from common.model.generation_config import GenerationConfig from common.model.library_config import LibraryConfig script_dir = os.path.dirname(os.path.realpath(__file__)) @@ -72,7 +72,7 @@ def test_generation_config_set_generator_version_from_env(self): os.environ.pop("GENERATOR_VERSION") def test_from_yaml_succeeds(self): - config = from_yaml(f"{test_config_dir}/generation_config.yaml") + config = GenerationConfig.from_yaml(f"{test_config_dir}/generation_config.yaml") self.assertEqual("2.34.0", config.gapic_generator_version) self.assertEqual( "1a45bf7393b52407188c82e63101db7dc9c72026", config.googleapis_commitish @@ -105,7 +105,7 @@ def test_from_yaml_succeeds(self): self.assertEqual("google/cloud/asset/v1p7beta1", gapics[4].proto_path) def test_get_proto_path_to_library_name_success(self): - paths = from_yaml( + paths = GenerationConfig.from_yaml( f"{test_config_dir}/generation_config.yaml" ).get_proto_path_to_library_name() self.assertEqual( @@ -181,7 +181,7 @@ def test_from_yaml_without_googleapis_commitish_raise_exception(self): self.assertRaisesRegex( ValueError, "Repo level parameter, googleapis_commitish", - from_yaml, + GenerationConfig.from_yaml, f"{test_config_dir}/config_without_googleapis.yaml", ) @@ -189,7 +189,7 @@ def test_from_yaml_without_libraries_raise_exception(self): self.assertRaisesRegex( ValueError, "Repo level parameter, libraries", - from_yaml, + GenerationConfig.from_yaml, f"{test_config_dir}/config_without_libraries.yaml", ) @@ -197,7 +197,7 @@ def test_from_yaml_without_api_shortname_raise_exception(self): self.assertRaisesRegex( ValueError, "Library level parameter, api_shortname", - from_yaml, + GenerationConfig.from_yaml, f"{test_config_dir}/config_without_api_shortname.yaml", ) @@ -205,7 +205,7 @@ def test_from_yaml_without_api_description_raise_exception(self): self.assertRaisesRegex( ValueError, r"Library level parameter, api_description.*'api_shortname': 'apigeeconnect'.*", - from_yaml, + GenerationConfig.from_yaml, f"{test_config_dir}/config_without_api_description.yaml", ) @@ -213,7 +213,7 @@ def test_from_yaml_without_name_pretty_raise_exception(self): self.assertRaisesRegex( ValueError, r"Library level parameter, name_pretty.*'api_shortname': 'apigeeconnect'.*", - from_yaml, + GenerationConfig.from_yaml, f"{test_config_dir}/config_without_name_pretty.yaml", ) @@ -221,7 +221,7 @@ def test_from_yaml_without_product_documentation_raise_exception(self): self.assertRaisesRegex( ValueError, r"Library level parameter, product_documentation.*'api_shortname': 'apigeeconnect'.*", - from_yaml, + GenerationConfig.from_yaml, f"{test_config_dir}/config_without_product_docs.yaml", ) @@ -229,7 +229,7 @@ def test_from_yaml_without_gapics_raise_exception(self): self.assertRaisesRegex( ValueError, "Library level parameter, GAPICs.*'api_shortname': 'apigeeconnect'.*", - from_yaml, + GenerationConfig.from_yaml, f"{test_config_dir}/config_without_gapics_key.yaml", ) @@ -237,7 +237,7 @@ def test_from_yaml_without_proto_path_raise_exception(self): self.assertRaisesRegex( ValueError, "GAPIC level parameter, proto_path", - from_yaml, + GenerationConfig.from_yaml, f"{test_config_dir}/config_without_proto_path.yaml", ) @@ -245,7 +245,7 @@ def test_from_yaml_with_zero_library_raise_exception(self): self.assertRaisesRegex( ValueError, "Library is None", - from_yaml, + GenerationConfig.from_yaml, f"{test_config_dir}/config_without_library_value.yaml", ) @@ -253,6 +253,6 @@ def test_from_yaml_with_zero_proto_path_raise_exception(self): self.assertRaisesRegex( ValueError, r"GAPICs is None in.*'api_shortname': 'apigeeconnect'.*", - from_yaml, + GenerationConfig.from_yaml, f"{test_config_dir}/config_without_gapics_value.yaml", ) diff --git a/hermetic_build/common/tests/utils/proto_path_utils_unit_tests.py b/hermetic_build/common/tests/utils/proto_path_utils_unit_tests.py index 90b3dd3f55..80d7797571 100644 --- a/hermetic_build/common/tests/utils/proto_path_utils_unit_tests.py +++ b/hermetic_build/common/tests/utils/proto_path_utils_unit_tests.py @@ -16,6 +16,7 @@ import unittest from pathlib import Path from common.utils.proto_path_utils import find_versioned_proto_path +from common.utils.proto_path_utils import ends_with_version script_dir = os.path.dirname(os.path.realpath(__file__)) resources_dir = os.path.join(script_dir, "..", "resources") @@ -37,3 +38,20 @@ def test_find_versioned_proto_without_version_return_itself(self): proto_path = "google/type/color.proto" expected = "google/type/color.proto" self.assertEqual(expected, find_versioned_proto_path(proto_path)) + + def test_ends_with_version_valid(self): + self.assertTrue(ends_with_version("google/cloud/gsuiteaddons/v1")) + self.assertTrue(ends_with_version("google/iam/v1beta")) + self.assertTrue(ends_with_version("google/iam/v2betav1")) + self.assertTrue(ends_with_version("google/cloud/alloydb/connectors/v1alpha")) + self.assertTrue(ends_with_version("v1")) + self.assertTrue(ends_with_version("anything/v123")) + + def test_ends_with_version_invalid(self): + self.assertFalse(ends_with_version("google/apps/script/type")) + self.assertFalse(ends_with_version("google/apps/script/type/docs")) + self.assertFalse( + ends_with_version("google/cloud/alloydb/connectors/v123/something") + ) + self.assertFalse(ends_with_version("")) + self.assertFalse(ends_with_version("noVersion")) diff --git a/hermetic_build/common/utils/proto_path_utils.py b/hermetic_build/common/utils/proto_path_utils.py index 49a86dcbd2..e28b4b0870 100644 --- a/hermetic_build/common/utils/proto_path_utils.py +++ b/hermetic_build/common/utils/proto_path_utils.py @@ -31,3 +31,21 @@ def find_versioned_proto_path(proto_path: str) -> str: idx = proto_path.find(version) return proto_path[:idx] + version return proto_path + + +def ends_with_version(proto_path: str) -> bool: + """ + Checks if a given proto_path string ends with a version identifier. + + :param proto_path: The proto_path string to check. + + :return: + True if the proto_path ends with a version, False otherwise. + """ + version_regex = re.compile(r"^v[1-9].*") + parts = proto_path.rsplit("/", 1) + if len(parts) > 1: + last_part = parts[1] + else: + last_part = parts[0] + return bool(version_regex.match(last_part)) diff --git a/hermetic_build/library_generation/cli/entry_point.py b/hermetic_build/library_generation/cli/entry_point.py index 6671cdd145..417944c218 100644 --- a/hermetic_build/library_generation/cli/entry_point.py +++ b/hermetic_build/library_generation/cli/entry_point.py @@ -18,7 +18,7 @@ import shutil from pathlib import Path from library_generation.generate_repo import generate_from_yaml -from common.model.generation_config import from_yaml, GenerationConfig +from common.model.generation_config import GenerationConfig @click.group(invoke_without_command=False) @@ -57,6 +57,7 @@ def main(ctx): ) @click.option( "--library-names", + required=False, type=str, default=None, show_default=True, @@ -64,6 +65,25 @@ def main(ctx): A list of library names that will be generated, separated by comma. The library name of a library is the value of library_name or api_shortname, if library_name is not specified, in the generation configuration. + + If neither --library-names or --api-path specified, all libraries in the + generation configuration will be generated. + """, +) +@click.option( + "--api-path", + required=False, + type=str, + default=None, + show_default=True, + help=""" + Path within the API root (e.g. googleapis) to the API to + generate/build/configure etc. This is expected to be a major-versioned + API directory, e.g. google/cloud/functions/v2. + + Takes precedence over --library-names when specidied. + If neither --library-names or --api-path specified, all libraries in the + generation configuration will be generated. """, ) @click.option( @@ -95,6 +115,7 @@ def generate( generation_config_path: Optional[str], generation_input: Optional[str], library_names: Optional[str], + api_path: Optional[str], repository_path: str, api_definitions_path: str, ): @@ -116,6 +137,7 @@ def generate( generation_config_path=generation_config_path, generation_input=generation_input, library_names=library_names, + api_path=api_path, repository_path=repository_path, api_definitions_path=api_definitions_path, ) @@ -123,10 +145,11 @@ def generate( def __generate_repo_impl( generation_config_path: Optional[str], + generation_input: Optional[str], library_names: Optional[str], + api_path: Optional[str], repository_path: str, api_definitions_path: str, - generation_input: Optional[str], ): """ Implementation method for generate(). @@ -156,7 +179,7 @@ def __generate_repo_impl( ) repository_path = os.path.abspath(repository_path) api_definitions_path = os.path.abspath(api_definitions_path) - generation_config = from_yaml(generation_config_path) + generation_config = GenerationConfig.from_yaml(generation_config_path) include_library_names = _parse_library_name_from( includes=library_names, generation_config=generation_config ) @@ -165,6 +188,7 @@ def __generate_repo_impl( repository_path=repository_path, api_definitions_path=api_definitions_path, target_library_names=include_library_names, + target_api_path=api_path, ) @@ -192,6 +216,7 @@ def _copy_versions_file(generation_input_path, repository_path): print(f"Copied '{source_file}' to '{destination_file}'") except Exception as e: print(f"An error occurred while copying the versions.txt: {e}") + raise # Re-raises the caught exception def _needs_full_repo_generation(generation_config: GenerationConfig) -> bool: @@ -230,7 +255,7 @@ def validate_generation_config(generation_config_path: str) -> None: if generation_config_path is None: generation_config_path = "generation_config.yaml" try: - from_yaml(os.path.abspath(generation_config_path)) + GenerationConfig.from_yaml(os.path.abspath(generation_config_path)) print(f"{generation_config_path} is validated without any errors.") except ValueError as err: print(err) diff --git a/hermetic_build/library_generation/generate_repo.py b/hermetic_build/library_generation/generate_repo.py index 4990c4632c..634ddffd38 100755 --- a/hermetic_build/library_generation/generate_repo.py +++ b/hermetic_build/library_generation/generate_repo.py @@ -12,21 +12,26 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import copy import shutil from pathlib import Path from typing import Optional import library_generation.utils.utilities as util from common.model.generation_config import GenerationConfig from common.model.library_config import LibraryConfig +from common.utils.proto_path_utils import ends_with_version from library_generation.generate_composed_library import generate_composed_library from library_generation.utils.monorepo_postprocessor import monorepo_postprocessing +from common.model.gapic_config import GapicConfig + def generate_from_yaml( config: GenerationConfig, repository_path: str, api_definitions_path: str, target_library_names: Optional[list[str]], + target_api_path: Optional[str] = None, ) -> None: """ Based on the generation config, generates libraries via @@ -43,7 +48,9 @@ def generate_from_yaml( If not specified, all libraries in the configuration yaml will be generated. """ target_libraries = get_target_libraries( - config=config, target_library_names=target_library_names + config=config, + target_library_names=target_library_names, + target_api_path=target_api_path, ) repo_config = util.prepare_repo( gen_config=config, library_config=target_libraries, repo_path=repository_path @@ -76,6 +83,24 @@ def generate_from_yaml( def get_target_libraries( + config: GenerationConfig, + target_library_names: list[str] = None, + target_api_path: Optional[str] = None, +) -> list[LibraryConfig]: + """ + Returns LibraryConfig objects whose library_name is in target_library_names. + + :param config: a GenerationConfig object. + :param target_library_names: library_name of target libraries. + If not specified, all libraries in the given config will be returned. + :return: LibraryConfig objects. + """ + if target_api_path is None: + return _get_target_libraries_from_name(config, target_library_names) + return _get_target_libraries_from_api_path(config, target_api_path) + + +def _get_target_libraries_from_name( config: GenerationConfig, target_library_names: list[str] = None ) -> list[LibraryConfig]: """ @@ -94,3 +119,36 @@ def get_target_libraries( for library in config.libraries if library.get_library_name() in target_libraries ] + + +def _get_target_libraries_from_api_path( + config: GenerationConfig, target_api_path: Optional[str] = None +) -> list[LibraryConfig]: + """ + Retrieves a copy of the LibraryConfig objects that contain the specified + target API path, without other proto_path versions that were not specified. + dependency proto_paths are kept. + + :param config: The GenerationConfig object. + :param target_api_path: The target proto path to search for. + + :return: + A list of LibraryConfig objects matching the target API path, or an + empty list if no matches are found. + """ + if not ends_with_version(target_api_path): + raise ValueError("api_path is not ending with a version is not supported") + target_libraries = [] + for library in config.libraries: + target_library = copy.deepcopy(library) + gapic_config_list = [] + for item in library.gapic_configs: + if item.proto_path == target_api_path or not ends_with_version( + item.proto_path + ): + gapic_config_list.append(GapicConfig(item.proto_path)) + if gapic_config_list: + target_library.set_gapic_configs(gapic_config_list) + target_libraries.append(target_library) + return target_libraries + return [] diff --git a/hermetic_build/library_generation/tests/cli/entry_point_unit_tests.py b/hermetic_build/library_generation/tests/cli/entry_point_unit_tests.py index 3c89e04ed6..83a4932805 100644 --- a/hermetic_build/library_generation/tests/cli/entry_point_unit_tests.py +++ b/hermetic_build/library_generation/tests/cli/entry_point_unit_tests.py @@ -22,7 +22,7 @@ validate_generation_config, __generate_repo_impl as generate_impl, ) -from common.model.generation_config import from_yaml +from common.model.generation_config import GenerationConfig script_dir = os.path.dirname(os.path.realpath(__file__)) test_resource_dir = os.path.join(script_dir, "..", "resources", "test-config") @@ -105,13 +105,14 @@ def test_generate_non_monorepo_without_library_names_full_generation( with target_library_names=None in order to trigger the full generation """ config_path = f"{test_resource_dir}/generation_config.yaml" - self.assertFalse(from_yaml(config_path).is_monorepo()) + self.assertFalse(GenerationConfig.from_yaml(config_path).is_monorepo()) # we call the implementation method directly since click # does special handling when a method is annotated with @main.command() generate_impl( generation_config_path=config_path, generation_input=None, library_names=None, + api_path=None, repository_path=".", api_definitions_path=".", ) @@ -120,6 +121,7 @@ def test_generate_non_monorepo_without_library_names_full_generation( repository_path=ANY, api_definitions_path=ANY, target_library_names=None, + target_api_path=None, ) @patch("library_generation.cli.entry_point.generate_from_yaml") @@ -134,13 +136,14 @@ def test_generate_non_monorepo_with_library_names_full_generation( target_library_names equals includes. """ config_path = f"{test_resource_dir}/generation_config.yaml" - self.assertFalse(from_yaml(config_path).is_monorepo()) + self.assertFalse(GenerationConfig.from_yaml(config_path).is_monorepo()) # we call the implementation method directly since click # does special handling when a method is annotated with @main.command() generate_impl( generation_config_path=config_path, generation_input=None, library_names="non-existent-library", + api_path=None, repository_path=".", api_definitions_path=".", ) @@ -149,6 +152,7 @@ def test_generate_non_monorepo_with_library_names_full_generation( repository_path=ANY, api_definitions_path=ANY, target_library_names=None, + target_api_path=None, ) @patch("library_generation.cli.entry_point.generate_from_yaml") @@ -163,13 +167,14 @@ def test_generate_monorepo_with_common_protos_without_library_names_triggers_ful target_library_names=None in order to trigger the full generation """ config_path = f"{test_resource_dir}/monorepo_with_common_protos.yaml" - self.assertTrue(from_yaml(config_path).is_monorepo()) + self.assertTrue(GenerationConfig.from_yaml(config_path).is_monorepo()) # we call the implementation method directly since click # does special handling when a method is annotated with @main.command() generate_impl( generation_config_path=config_path, generation_input=None, library_names=None, + api_path=None, repository_path=".", api_definitions_path=".", ) @@ -178,6 +183,7 @@ def test_generate_monorepo_with_common_protos_without_library_names_triggers_ful repository_path=ANY, api_definitions_path=ANY, target_library_names=None, + target_api_path=None, ) @patch("library_generation.cli.entry_point.generate_from_yaml") @@ -191,13 +197,14 @@ def test_generate_monorepo_with_common_protos_with_library_names_triggers_full_g target_library_names is the same as includes. """ config_path = f"{test_resource_dir}/monorepo_with_common_protos.yaml" - self.assertTrue(from_yaml(config_path).is_monorepo()) + self.assertTrue(GenerationConfig.from_yaml(config_path).is_monorepo()) # we call the implementation method directly since click # does special handling when a method is annotated with @main.command() generate_impl( generation_config_path=config_path, generation_input=None, library_names="iam,non-existent-library", + api_path=None, repository_path=".", api_definitions_path=".", ) @@ -206,6 +213,7 @@ def test_generate_monorepo_with_common_protos_with_library_names_triggers_full_g repository_path=ANY, api_definitions_path=ANY, target_library_names=None, + target_api_path=None, ) @patch("library_generation.cli.entry_point.generate_from_yaml") @@ -221,13 +229,14 @@ def test_generate_monorepo_without_library_names_trigger_full_generation( generation. """ config_path = f"{test_resource_dir}/monorepo_without_common_protos.yaml" - self.assertTrue(from_yaml(config_path).is_monorepo()) + self.assertTrue(GenerationConfig.from_yaml(config_path).is_monorepo()) # we call the implementation method directly since click # does special handling when a method is annotated with @main.command() generate_impl( generation_config_path=config_path, generation_input=None, library_names=None, + api_path=None, repository_path=".", api_definitions_path=".", ) @@ -236,6 +245,7 @@ def test_generate_monorepo_without_library_names_trigger_full_generation( repository_path=ANY, api_definitions_path=ANY, target_library_names=None, + target_api_path=None, ) @patch("library_generation.cli.entry_point.generate_from_yaml") @@ -251,13 +261,14 @@ def test_generate_monorepo_with_library_names_trigger_selective_generation( generation. """ config_path = f"{test_resource_dir}/monorepo_without_common_protos.yaml" - self.assertTrue(from_yaml(config_path).is_monorepo()) + self.assertTrue(GenerationConfig.from_yaml(config_path).is_monorepo()) # we call the implementation method directly since click # does special handling when a method is annotated with @main.command() generate_impl( generation_config_path=config_path, generation_input=None, library_names="asset", + api_path=None, repository_path=".", api_definitions_path=".", ) @@ -266,12 +277,15 @@ def test_generate_monorepo_with_library_names_trigger_selective_generation( repository_path=ANY, api_definitions_path=ANY, target_library_names=["asset"], + target_api_path=None, ) - @patch("library_generation.cli.entry_point.from_yaml") + @patch("library_generation.cli.entry_point.generate_from_yaml") + @patch("common.model.generation_config.GenerationConfig.from_yaml") def test_generate_provide_generation_input( self, - from_yaml, + mock_from_yaml, + mock_generate_from_yaml, ): """ This test confirms that when no generation_config_path and @@ -286,10 +300,11 @@ def test_generate_provide_generation_input( generation_config_path=None, generation_input=test_resource_dir, library_names="asset", + api_path=None, repository_path="./test-output", api_definitions_path=".", ) - from_yaml.assert_called_with(os.path.abspath(config_path)) + mock_from_yaml.assert_called_with(os.path.abspath(config_path)) self.assertTrue(os.path.exists(f"test-output/versions.txt")) def tearDown(self): diff --git a/hermetic_build/library_generation/tests/cli/versions.txt b/hermetic_build/library_generation/tests/cli/versions.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hermetic_build/library_generation/tests/generate_repo_unit_tests.py b/hermetic_build/library_generation/tests/generate_repo_unit_tests.py index 3d81e2a235..ca61eec2f4 100644 --- a/hermetic_build/library_generation/tests/generate_repo_unit_tests.py +++ b/hermetic_build/library_generation/tests/generate_repo_unit_tests.py @@ -18,6 +18,8 @@ from common.model.generation_config import GenerationConfig from common.model.library_config import LibraryConfig +from common.model.gapic_config import GapicConfig + class GenerateRepoTest(unittest.TestCase): def test_get_target_library_returns_selected_libraries(self): @@ -54,6 +56,79 @@ def test_get_target_library_given_an_non_existent_library_returns_only_existing_ ) self.assertEqual([one_library, another_library], target_libraries) + def test_get_target_library_returns_selected_api_path(self): + one_library = GenerateRepoTest.__get_an_empty_library_config() + one_library.api_shortname = "one_library" + one_library.gapic_configs = [GapicConfig("google/cloud/one/library/v1")] + another_library = GenerateRepoTest.__get_an_empty_library_config() + another_library.api_shortname = "another_library" + another_library_gapic_config = list() + another_library_gapic_config.append( + GapicConfig("google/cloud/another/library/v1") + ) + another_library_gapic_config.append( + GapicConfig("google/cloud/another/library/v2") + ) + another_library.gapic_configs = another_library_gapic_config + config = GenerateRepoTest.__get_an_empty_generation_config() + config.libraries.extend([one_library, another_library]) + target_libraries = get_target_libraries( + config, target_api_path="google/cloud/another/library/v2" + ) + another_library_v1 = GenerateRepoTest.__get_an_empty_library_config() + another_library_v1.api_shortname = "another_library" + another_library_v1.gapic_configs = [ + GapicConfig("google/cloud/another/library/v2") + ] + + self.assertEqual([another_library_v1], target_libraries) + + def test_get_target_library_returns_selected_api_path_plus_dep(self): + one_library = GenerateRepoTest.__get_an_empty_library_config() + one_library.api_shortname = "one_library" + one_library.gapic_configs = [GapicConfig("google/cloud/one/library/v1")] + another_library = GenerateRepoTest.__get_an_empty_library_config() + another_library.api_shortname = "another_library" + another_library_gapic_config = list() + another_library_gapic_config.append( + GapicConfig("google/cloud/another/library/v1") + ) + another_library_gapic_config.append( + GapicConfig("google/cloud/another/library/v2") + ) + another_library_gapic_config.append( + GapicConfig("google/cloud/another/library/type") + ) + another_library.gapic_configs = another_library_gapic_config + config = GenerateRepoTest.__get_an_empty_generation_config() + config.libraries.extend([one_library, another_library]) + target_libraries = get_target_libraries( + config, target_api_path="google/cloud/another/library/v2" + ) + another_library_v1 = GenerateRepoTest.__get_an_empty_library_config() + another_library_v1.api_shortname = "another_library" + another_library_v1.gapic_configs = [ + GapicConfig("google/cloud/another/library/v2"), + GapicConfig("google/cloud/another/library/type"), + ] + + self.assertEqual([another_library_v1], target_libraries) + + def test_get_target_library_invalid_target_api_path(self): + """ + Tests when the api_path is invalid and sys.exit is called. + """ + config = GenerateRepoTest.__get_an_empty_library_config() + with self.assertRaises(ValueError) as context: + _target_libraries = get_target_libraries( + config, target_api_path="google/cloud/another/library/type" + ) + + self.assertEqual( + str(context.exception), + "api_path is not ending with a version is not supported", + ) + @staticmethod def __get_an_empty_generation_config() -> GenerationConfig: return GenerationConfig( diff --git a/hermetic_build/release_note_generation/cli/generate_release_note.py b/hermetic_build/release_note_generation/cli/generate_release_note.py index a54e228f62..2dd35ebf82 100644 --- a/hermetic_build/release_note_generation/cli/generate_release_note.py +++ b/hermetic_build/release_note_generation/cli/generate_release_note.py @@ -15,7 +15,7 @@ from typing import Optional import click as click from release_note_generation.generate_pr_description import generate_pr_descriptions -from common.model.generation_config import from_yaml +from common.model.generation_config import GenerationConfig from common.utils.generation_config_comparator import compare_config @@ -86,8 +86,8 @@ def generate( ) return config_change = compare_config( - baseline_config=from_yaml(baseline_generation_config_path), - current_config=from_yaml(current_generation_config_path), + baseline_config=GenerationConfig.from_yaml(baseline_generation_config_path), + current_config=GenerationConfig.from_yaml(current_generation_config_path), ) generate_pr_descriptions( config_change=config_change,