chore: allow generation-input as folder that supplies config and vers… (#3700)

…ions file.

This change adds `--generation-input` option to generation command, only
intend to be used when `--generation-config-path` is not provided. If
none of these 2 provided, then default behavior does not change.

Also added a clean up to delete the temp output folder created in
generation process. This should only be an improvement to local dev
workflow.

Author: zhumin8

hermetic_build/library_generation/cli/entry_point.py

@@ -15,6 +15,8 @@
 import sys
 from typing import Optional
 import click as click
+import shutil
+from pathlib import Path
 from library_generation.generate_repo import generate_from_yaml
 from common.model.generation_config import from_yaml, GenerationConfig
 
@@ -35,6 +37,22 @@ def main(ctx):
     help="""
     Absolute or relative path to a generation_config.yaml that contains the
     metadata about library generation.
+    When not set, will use generation_config.yaml from --generation-input.
+    When neither this or --generation-input is set default to 
+    generation_config.yaml in the current working directory 
+    """,
+)
+@click.option(
+    "--generation-input",
+    required=False,
+    default=None,
+    type=str,
+    help="""
+    Absolute or relative path to a input folder that contains 
+    generation_config.yaml and versions.txt.
+    This is only used when --generation-config-path is not set.
+    When neither this or --generation-config-path is set default to 
+    generation_config.yaml in the current working directory 
     """,
 )
 @click.option(
@@ -75,6 +93,7 @@ def main(ctx):
 )
 def generate(
     generation_config_path: Optional[str],
+    generation_input: Optional[str],
     library_names: Optional[str],
     repository_path: str,
     api_definitions_path: str,
@@ -95,6 +114,7 @@ def generate(
     """
     __generate_repo_impl(
         generation_config_path=generation_config_path,
+        generation_input=generation_input,
         library_names=library_names,
         repository_path=repository_path,
         api_definitions_path=api_definitions_path,
@@ -106,15 +126,28 @@ def __generate_repo_impl(
     library_names: Optional[str],
     repository_path: str,
     api_definitions_path: str,
+    generation_input: Optional[str],
 ):
     """
     Implementation method for generate().
     The decoupling of generate and __generate_repo_impl is
     meant to allow testing of this implementation function.
     """
 
+    # only use generation_input when generation_config_path is not provided and
+    # generation_input provided. generation_config_path should be deprecated after
+    # migration to 1pp.
     default_generation_config_path = f"{os.getcwd()}/generation_config.yaml"
-    if generation_config_path is None:
+    if generation_config_path is None and generation_input is not None:
+        print(
+            "generation_config_path is not provided, using generation-input folder provided"
+        )
+        generation_config_path = f"{generation_input}/generation_config.yaml"
+        # copy versions.txt from generation_input to repository_path
+        # override if present.
+        _copy_versions_file(generation_input, repository_path)
+    if generation_config_path is None and generation_input is None:
+        print("Using default generation config path")
         generation_config_path = default_generation_config_path
     generation_config_path = os.path.abspath(generation_config_path)
     if not os.path.isfile(generation_config_path):
@@ -135,6 +168,32 @@ def __generate_repo_impl(
     )
 
 
+def _copy_versions_file(generation_input_path, repository_path):
+    """
+    Copies the versions.txt file from the generation_input folder to the repository_path.
+    Overrides the destination file if it already exists.
+
+    Args:
+        generation_input_path (str): The path to the generation_input folder.
+        repository_path (str): The path to the repository folder.
+    """
+    source_file = Path(generation_input_path) / "versions.txt"
+    destination_file = Path(repository_path) / "versions.txt"
+
+    if not source_file.exists():
+        destination_file.touch()
+        print(
+            f"generation-input does not contain versions.txt. "
+            f"Created empty versions file: {source_file}"
+        )
+        return
+    try:
+        shutil.copy2(source_file, destination_file)
+        print(f"Copied '{source_file}' to '{destination_file}'")
+    except Exception as e:
+        print(f"An error occurred while copying the versions.txt: {e}")
+
+
 def _needs_full_repo_generation(generation_config: GenerationConfig) -> bool:
     """
     Whether you should need a full repo generation, i.e., generate all

hermetic_build/library_generation/generate_repo.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import shutil
+from pathlib import Path
 from typing import Optional
 import library_generation.utils.utilities as util
 from common.model.generation_config import GenerationConfig
@@ -65,6 +66,14 @@ def generate_from_yaml(
         repository_path=repository_path, versions_file=repo_config.versions_file
     )
 
+    # cleanup temp output folder
+    try:
+        shutil.rmtree(Path(repo_config.output_folder))
+        print(f"Directory {repo_config.output_folder} and its contents removed.")
+    except OSError as e:
+        print(f"Error: {e} - Failed to remove directory {repo_config.output_folder}.")
+        raise
+
 
 def get_target_libraries(
     config: GenerationConfig, target_library_names: list[str] = None

hermetic_build/library_generation/tests/cli/entry_point_unit_tests.py

@@ -12,7 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import os
+import shutil
 import unittest
+from pathlib import Path
 from unittest.mock import patch, ANY
 from click.testing import CliRunner
 from library_generation.cli.entry_point import (
@@ -49,6 +51,20 @@ def test_entry_point_with_invalid_config_raise_file_exception(self):
         self.assertEqual(FileNotFoundError, result.exc_info[0])
         self.assertRegex(result.exception.args[0], "/non-existent/file does not exist.")
 
+    def test_entry_point_with_invalid_generation_input_raise_file_exception(
+        self,
+    ):
+        os.chdir(script_dir)
+        runner = CliRunner()
+        # noinspection PyTypeChecker
+        result = runner.invoke(generate, ["--generation-input=/non-existent/folder"])
+        self.assertEqual(1, result.exit_code)
+        self.assertEqual(FileNotFoundError, result.exc_info[0])
+        self.assertRegex(
+            result.exception.args[0],
+            "/non-existent/folder/generation_config.yaml does not exist.",
+        )
+
     def test_validate_generation_config_succeeds(
         self,
     ):
@@ -94,6 +110,7 @@ def test_generate_non_monorepo_without_library_names_full_generation(
         # does special handling when a method is annotated with @main.command()
         generate_impl(
             generation_config_path=config_path,
+            generation_input=None,
             library_names=None,
             repository_path=".",
             api_definitions_path=".",
@@ -122,6 +139,7 @@ def test_generate_non_monorepo_with_library_names_full_generation(
         # does special handling when a method is annotated with @main.command()
         generate_impl(
             generation_config_path=config_path,
+            generation_input=None,
             library_names="non-existent-library",
             repository_path=".",
             api_definitions_path=".",
@@ -150,6 +168,7 @@ def test_generate_monorepo_with_common_protos_without_library_names_triggers_ful
         # does special handling when a method is annotated with @main.command()
         generate_impl(
             generation_config_path=config_path,
+            generation_input=None,
             library_names=None,
             repository_path=".",
             api_definitions_path=".",
@@ -177,6 +196,7 @@ def test_generate_monorepo_with_common_protos_with_library_names_triggers_full_g
         # does special handling when a method is annotated with @main.command()
         generate_impl(
             generation_config_path=config_path,
+            generation_input=None,
             library_names="iam,non-existent-library",
             repository_path=".",
             api_definitions_path=".",
@@ -206,6 +226,7 @@ def test_generate_monorepo_without_library_names_trigger_full_generation(
         # does special handling when a method is annotated with @main.command()
         generate_impl(
             generation_config_path=config_path,
+            generation_input=None,
             library_names=None,
             repository_path=".",
             api_definitions_path=".",
@@ -235,6 +256,7 @@ def test_generate_monorepo_with_library_names_trigger_selective_generation(
         # does special handling when a method is annotated with @main.command()
         generate_impl(
             generation_config_path=config_path,
+            generation_input=None,
             library_names="asset",
             repository_path=".",
             api_definitions_path=".",
@@ -245,3 +267,44 @@ def test_generate_monorepo_with_library_names_trigger_selective_generation(
             api_definitions_path=ANY,
             target_library_names=["asset"],
         )
+
+    @patch("library_generation.cli.entry_point.from_yaml")
+    def test_generate_provide_generation_input(
+        self,
+        from_yaml,
+    ):
+        """
+        This test confirms that when no generation_config_path and
+        only generation_input is provided, it looks inside this path
+        for generation config and creates versions file when not exists
+        """
+        config_path = f"{test_resource_dir}/generation_config.yaml"
+        self._create_folder_in_current_dir("test-output")
+        # we call the implementation method directly since click
+        # does special handling when a method is annotated with @main.command()
+        generate_impl(
+            generation_config_path=None,
+            generation_input=test_resource_dir,
+            library_names="asset",
+            repository_path="./test-output",
+            api_definitions_path=".",
+        )
+        from_yaml.assert_called_with(os.path.abspath(config_path))
+        self.assertTrue(os.path.exists(f"test-output/versions.txt"))
+
+    def tearDown(self):
+        # clean up after
+        if os.path.exists("./output"):
+            shutil.rmtree(Path("./output"))
+        if os.path.exists("./test-output"):
+            shutil.rmtree(Path("./test-output"))
+
+    def _create_folder_in_current_dir(self, folder_name):
+        """Creates a folder in the current directory."""
+        try:
+            os.makedirs(
+                folder_name, exist_ok=True
+            )  # exist_ok prevents errors if folder exists
+            print(f"Folder '{folder_name}' created successfully.")
+        except OSError as e:
+            print(f"Error creating folder '{folder_name}': {e}")

hermetic_build/library_generation/utils/utilities.py

@@ -159,14 +159,14 @@ def prepare_repo(
         json_name = ".repo-metadata.json"
         if os.path.exists(f"{absolute_library_path}/{json_name}"):
             os.remove(f"{absolute_library_path}/{json_name}")
-    versions_file = f"{repo_path}/versions.txt"
-    if not Path(versions_file).exists():
-        raise FileNotFoundError(f"{versions_file} is not found.")
-
+    versions_file = Path(repo_path) / "versions.txt"
+    if not versions_file.exists():
+        versions_file.touch()
+        print(f"Created empty versions file: {versions_file}")
     return RepoConfig(
         output_folder=output_folder,
         libraries=libraries,
-        versions_file=str(Path(versions_file).resolve()),
+        versions_file=str(versions_file),
     )