chore: allow --api-path as option to generate cmd and generate per api/majorversion (#3712)

This change allow generator to generate for a specified api-path. When
"--api-path" is not specified, fallback to "--library-names", and
__behavior for existing command usages should not change__.

For now, api-path should only take in path to api/version, e.g.
google/cloud/functions/v2

Author: zhumin8

hermetic_build/common/cli/get_changed_libraries.py

@@ -15,7 +15,7 @@
 
 import click as click
 
-from common.model.generation_config import from_yaml
+from common.model.generation_config import GenerationConfig
 from common.utils.generation_config_comparator import compare_config
 
 
@@ -69,8 +69,8 @@ def create(
             "current-generation-config-path."
         )
     config_change = compare_config(
-        baseline_config=from_yaml(baseline_generation_config_path),
-        current_config=from_yaml(current_generation_config_path),
+        baseline_config=GenerationConfig.from_yaml(baseline_generation_config_path),
+        current_config=GenerationConfig.from_yaml(current_generation_config_path),
     )
     click.echo(",".join(config_change.get_changed_libraries()))
 

hermetic_build/common/model/generation_config.py

@@ -105,77 +105,77 @@ def __validate(self) -> None:
                 )
             seen_library_names[library_name] = library.name_pretty
 
+    @staticmethod
+    def from_yaml(path_to_yaml: str) -> "GenerationConfig":
+        """
+        Parses a yaml located in path_to_yaml.
+        :param path_to_yaml: the path to the configuration file
+        :return the parsed configuration represented by the "model" classes
+        """
+        with open(path_to_yaml, "r") as file_stream:
+            config = yaml.safe_load(file_stream)
+
+        libraries = _required(config, "libraries", REPO_LEVEL_PARAMETER)
+        if not libraries:
+            raise ValueError(f"Library is None in {path_to_yaml}.")
+
+        parsed_libraries = list()
+        for library in libraries:
+            gapics = _required(library, "GAPICs")
+
+            parsed_gapics = list()
+            if not gapics:
+                raise ValueError(f"GAPICs is None in {library}.")
+            for gapic in gapics:
+                proto_path = _required(gapic, "proto_path", GAPIC_LEVEL_PARAMETER)
+                new_gapic = GapicConfig(proto_path)
+                parsed_gapics.append(new_gapic)
+
+            new_library = LibraryConfig(
+                api_shortname=_required(library, "api_shortname"),
+                api_description=_required(library, "api_description"),
+                name_pretty=_required(library, "name_pretty"),
+                product_documentation=_required(library, "product_documentation"),
+                gapic_configs=parsed_gapics,
+                library_type=_optional(library, "library_type", "GAPIC_AUTO"),
+                release_level=_optional(library, "release_level", "preview"),
+                api_id=_optional(library, "api_id", None),
+                api_reference=_optional(library, "api_reference", None),
+                codeowner_team=_optional(library, "codeowner_team", None),
+                excluded_poms=_optional(library, "excluded_poms", None),
+                excluded_dependencies=_optional(library, "excluded_dependencies", None),
+                client_documentation=_optional(library, "client_documentation", None),
+                distribution_name=_optional(library, "distribution_name", None),
+                googleapis_commitish=_optional(library, "googleapis_commitish", None),
+                group_id=_optional(library, "group_id", "com.google.cloud"),
+                issue_tracker=_optional(library, "issue_tracker", None),
+                library_name=_optional(library, "library_name", None),
+                rest_documentation=_optional(library, "rest_documentation", None),
+                rpc_documentation=_optional(library, "rpc_documentation", None),
+                cloud_api=_optional(library, "cloud_api", True),
+                requires_billing=_optional(library, "requires_billing", True),
+                extra_versioned_modules=_optional(
+                    library, "extra_versioned_modules", None
+                ),
+                recommended_package=_optional(library, "recommended_package", None),
+                min_java_version=_optional(library, "min_java_version", None),
+                transport=_optional(library, "transport", None),
+            )
+            parsed_libraries.append(new_library)
 
-def from_yaml(path_to_yaml: str) -> GenerationConfig:
-    """
-    Parses a yaml located in path_to_yaml.
-    :param path_to_yaml: the path to the configuration file
-    :return the parsed configuration represented by the "model" classes
-    """
-    with open(path_to_yaml, "r") as file_stream:
-        config = yaml.safe_load(file_stream)
-
-    libraries = __required(config, "libraries", REPO_LEVEL_PARAMETER)
-    if not libraries:
-        raise ValueError(f"Library is None in {path_to_yaml}.")
-
-    parsed_libraries = list()
-    for library in libraries:
-        gapics = __required(library, "GAPICs")
-
-        parsed_gapics = list()
-        if not gapics:
-            raise ValueError(f"GAPICs is None in {library}.")
-        for gapic in gapics:
-            proto_path = __required(gapic, "proto_path", GAPIC_LEVEL_PARAMETER)
-            new_gapic = GapicConfig(proto_path)
-            parsed_gapics.append(new_gapic)
-
-        new_library = LibraryConfig(
-            api_shortname=__required(library, "api_shortname"),
-            api_description=__required(library, "api_description"),
-            name_pretty=__required(library, "name_pretty"),
-            product_documentation=__required(library, "product_documentation"),
-            gapic_configs=parsed_gapics,
-            library_type=__optional(library, "library_type", "GAPIC_AUTO"),
-            release_level=__optional(library, "release_level", "preview"),
-            api_id=__optional(library, "api_id", None),
-            api_reference=__optional(library, "api_reference", None),
-            codeowner_team=__optional(library, "codeowner_team", None),
-            excluded_poms=__optional(library, "excluded_poms", None),
-            excluded_dependencies=__optional(library, "excluded_dependencies", None),
-            client_documentation=__optional(library, "client_documentation", None),
-            distribution_name=__optional(library, "distribution_name", None),
-            googleapis_commitish=__optional(library, "googleapis_commitish", None),
-            group_id=__optional(library, "group_id", "com.google.cloud"),
-            issue_tracker=__optional(library, "issue_tracker", None),
-            library_name=__optional(library, "library_name", None),
-            rest_documentation=__optional(library, "rest_documentation", None),
-            rpc_documentation=__optional(library, "rpc_documentation", None),
-            cloud_api=__optional(library, "cloud_api", True),
-            requires_billing=__optional(library, "requires_billing", True),
-            extra_versioned_modules=__optional(
-                library, "extra_versioned_modules", None
+        parsed_config = GenerationConfig(
+            googleapis_commitish=_required(
+                config, "googleapis_commitish", REPO_LEVEL_PARAMETER
             ),
-            recommended_package=__optional(library, "recommended_package", None),
-            min_java_version=__optional(library, "min_java_version", None),
-            transport=__optional(library, "transport", None),
+            gapic_generator_version=_optional(config, GAPIC_GENERATOR_VERSION, None),
+            libraries_bom_version=_optional(config, LIBRARIES_BOM_VERSION, None),
+            libraries=parsed_libraries,
         )
-        parsed_libraries.append(new_library)
-
-    parsed_config = GenerationConfig(
-        googleapis_commitish=__required(
-            config, "googleapis_commitish", REPO_LEVEL_PARAMETER
-        ),
-        gapic_generator_version=__optional(config, GAPIC_GENERATOR_VERSION, None),
-        libraries_bom_version=__optional(config, LIBRARIES_BOM_VERSION, None),
-        libraries=parsed_libraries,
-    )
 
-    return parsed_config
+        return parsed_config
 
 
-def __required(config: dict, key: str, level: str = LIBRARY_LEVEL_PARAMETER):
+def _required(config: dict, key: str, level: str = LIBRARY_LEVEL_PARAMETER):
     if key not in config:
         message = (
             f"{level}, {key}, is not found in {config} in yaml."
@@ -186,7 +186,7 @@ def __required(config: dict, key: str, level: str = LIBRARY_LEVEL_PARAMETER):
     return config[key]
 
 
-def __optional(config: dict, key: str, default: any):
+def _optional(config: dict, key: str, default: any):
     if key not in config:
         return default
     return config[key]

hermetic_build/common/model/library_config.py

@@ -82,6 +82,15 @@ def __init__(
         self.distribution_name = self.__get_distribution_name(distribution_name)
         self.transport = self.__validate_transport(transport)
 
+    def set_gapic_configs(self, gapic_configs: list[GapicConfig]) -> None:
+        """
+        Sets the gapic_configs for the library.
+
+        Args:
+            gapic_configs: The new list of GapicConfig objects.
+        """
+        self.gapic_configs = gapic_configs
+
     def get_library_name(self) -> str:
         """
         Return the library name of a given LibraryConfig object

hermetic_build/common/tests/model/generation_config_unit_tests.py

@@ -14,7 +14,7 @@
 import os
 import unittest
 from pathlib import Path
-from common.model.generation_config import from_yaml, GenerationConfig
+from common.model.generation_config import GenerationConfig
 from common.model.library_config import LibraryConfig
 
 script_dir = os.path.dirname(os.path.realpath(__file__))
@@ -72,7 +72,7 @@ def test_generation_config_set_generator_version_from_env(self):
         os.environ.pop("GENERATOR_VERSION")
 
     def test_from_yaml_succeeds(self):
-        config = from_yaml(f"{test_config_dir}/generation_config.yaml")
+        config = GenerationConfig.from_yaml(f"{test_config_dir}/generation_config.yaml")
         self.assertEqual("2.34.0", config.gapic_generator_version)
         self.assertEqual(
             "1a45bf7393b52407188c82e63101db7dc9c72026", config.googleapis_commitish
@@ -105,7 +105,7 @@ def test_from_yaml_succeeds(self):
         self.assertEqual("google/cloud/asset/v1p7beta1", gapics[4].proto_path)
 
     def test_get_proto_path_to_library_name_success(self):
-        paths = from_yaml(
+        paths = GenerationConfig.from_yaml(
             f"{test_config_dir}/generation_config.yaml"
         ).get_proto_path_to_library_name()
         self.assertEqual(
@@ -181,78 +181,78 @@ def test_from_yaml_without_googleapis_commitish_raise_exception(self):
         self.assertRaisesRegex(
             ValueError,
             "Repo level parameter, googleapis_commitish",
-            from_yaml,
+            GenerationConfig.from_yaml,
             f"{test_config_dir}/config_without_googleapis.yaml",
         )
 
     def test_from_yaml_without_libraries_raise_exception(self):
         self.assertRaisesRegex(
             ValueError,
             "Repo level parameter, libraries",
-            from_yaml,
+            GenerationConfig.from_yaml,
             f"{test_config_dir}/config_without_libraries.yaml",
         )
 
     def test_from_yaml_without_api_shortname_raise_exception(self):
         self.assertRaisesRegex(
             ValueError,
             "Library level parameter, api_shortname",
-            from_yaml,
+            GenerationConfig.from_yaml,
             f"{test_config_dir}/config_without_api_shortname.yaml",
         )
 
     def test_from_yaml_without_api_description_raise_exception(self):
         self.assertRaisesRegex(
             ValueError,
             r"Library level parameter, api_description.*'api_shortname': 'apigeeconnect'.*",
-            from_yaml,
+            GenerationConfig.from_yaml,
             f"{test_config_dir}/config_without_api_description.yaml",
         )
 
     def test_from_yaml_without_name_pretty_raise_exception(self):
         self.assertRaisesRegex(
             ValueError,
             r"Library level parameter, name_pretty.*'api_shortname': 'apigeeconnect'.*",
-            from_yaml,
+            GenerationConfig.from_yaml,
             f"{test_config_dir}/config_without_name_pretty.yaml",
         )
 
     def test_from_yaml_without_product_documentation_raise_exception(self):
         self.assertRaisesRegex(
             ValueError,
             r"Library level parameter, product_documentation.*'api_shortname': 'apigeeconnect'.*",
-            from_yaml,
+            GenerationConfig.from_yaml,
             f"{test_config_dir}/config_without_product_docs.yaml",
         )
 
     def test_from_yaml_without_gapics_raise_exception(self):
         self.assertRaisesRegex(
             ValueError,
             "Library level parameter, GAPICs.*'api_shortname': 'apigeeconnect'.*",
-            from_yaml,
+            GenerationConfig.from_yaml,
             f"{test_config_dir}/config_without_gapics_key.yaml",
         )
 
     def test_from_yaml_without_proto_path_raise_exception(self):
         self.assertRaisesRegex(
             ValueError,
             "GAPIC level parameter, proto_path",
-            from_yaml,
+            GenerationConfig.from_yaml,
             f"{test_config_dir}/config_without_proto_path.yaml",
         )
 
     def test_from_yaml_with_zero_library_raise_exception(self):
         self.assertRaisesRegex(
             ValueError,
             "Library is None",
-            from_yaml,
+            GenerationConfig.from_yaml,
             f"{test_config_dir}/config_without_library_value.yaml",
         )
 
     def test_from_yaml_with_zero_proto_path_raise_exception(self):
         self.assertRaisesRegex(
             ValueError,
             r"GAPICs is None in.*'api_shortname': 'apigeeconnect'.*",
-            from_yaml,
+            GenerationConfig.from_yaml,
             f"{test_config_dir}/config_without_gapics_value.yaml",
         )

hermetic_build/common/tests/utils/proto_path_utils_unit_tests.py

@@ -16,6 +16,7 @@
 import unittest
 from pathlib import Path
 from common.utils.proto_path_utils import find_versioned_proto_path
+from common.utils.proto_path_utils import ends_with_version
 
 script_dir = os.path.dirname(os.path.realpath(__file__))
 resources_dir = os.path.join(script_dir, "..", "resources")
@@ -37,3 +38,20 @@ def test_find_versioned_proto_without_version_return_itself(self):
         proto_path = "google/type/color.proto"
         expected = "google/type/color.proto"
         self.assertEqual(expected, find_versioned_proto_path(proto_path))
+
+    def test_ends_with_version_valid(self):
+        self.assertTrue(ends_with_version("google/cloud/gsuiteaddons/v1"))
+        self.assertTrue(ends_with_version("google/iam/v1beta"))
+        self.assertTrue(ends_with_version("google/iam/v2betav1"))
+        self.assertTrue(ends_with_version("google/cloud/alloydb/connectors/v1alpha"))
+        self.assertTrue(ends_with_version("v1"))
+        self.assertTrue(ends_with_version("anything/v123"))
+
+    def test_ends_with_version_invalid(self):
+        self.assertFalse(ends_with_version("google/apps/script/type"))
+        self.assertFalse(ends_with_version("google/apps/script/type/docs"))
+        self.assertFalse(
+            ends_with_version("google/cloud/alloydb/connectors/v123/something")
+        )
+        self.assertFalse(ends_with_version(""))
+        self.assertFalse(ends_with_version("noVersion"))

hermetic_build/common/utils/proto_path_utils.py

@@ -31,3 +31,21 @@ def find_versioned_proto_path(proto_path: str) -> str:
             idx = proto_path.find(version)
             return proto_path[:idx] + version
     return proto_path
+
+
+def ends_with_version(proto_path: str) -> bool:
+    """
+    Checks if a given proto_path string ends with a version identifier.
+
+    :param proto_path: The proto_path string to check.
+
+    :return:
+        True if the proto_path ends with a version, False otherwise.
+    """
+    version_regex = re.compile(r"^v[1-9].*")
+    parts = proto_path.rsplit("/", 1)
+    if len(parts) > 1:
+        last_part = parts[1]
+    else:
+        last_part = parts[0]
+    return bool(version_regex.match(last_part))