py2fgen: don't recompile if unchanged (#1110)

py2fgen will avoid recompilation (and write of the f90 and py file) if
there are no changes. This allows for better integration with tools that
detect changes by timestamp.

`--regenerate` will force regeneration/compilation.

Author: havogt

tools/src/icon4py/tools/py2fgen/_cli.py

@@ -7,6 +7,7 @@
 # SPDX-License-Identifier: BSD-3-Clause
 
 import pathlib
+import sysconfig
 
 import click
 
@@ -37,12 +38,19 @@
     default="",
     help="Specify an rpath for the compiled library. If not set, no rpath is added.",
 )
+@click.option(
+    "--regenerate",
+    is_flag=True,
+    default=False,
+    help="Force regeneration of all files and recompilation, even if they are up to date.",
+)
 def main(
     module_import_path: str,
     functions: list[str],
     library_name: str,
     output_path: pathlib.Path,
     rpath: str,
+    regenerate: bool,
 ) -> None:
     """Generate C and F90 wrappers and C library for embedding a Python module in C and Fortran."""
     output_path.mkdir(exist_ok=True, parents=True)
@@ -52,15 +60,34 @@ def main(
     c_header = _codegen.generate_c_header(plugin)
     logger.info("Generating Python wrapper...")
     python_wrapper = _codegen.generate_python_wrapper(plugin)
-    _utils.write_file(python_wrapper, output_path, f"{plugin.library_name}.py")
     logger.info("Generating Fortran interface...")
     f90_interface = _codegen.generate_f90_interface(plugin)
-    _utils.write_file(f90_interface, output_path, f"{plugin.library_name}.f90")
 
-    logger.info("Compiling CFFI dynamic library...")
-    _generator.generate_and_compile_cffi_plugin(
-        plugin.library_name, c_header, python_wrapper, output_path, rpath
-    )
+    if regenerate:
+        logger.info("Force regeneration requested.")
+
+    any_changed = False
+    for content, fname, label in [
+        (python_wrapper, f"{plugin.library_name}.py", "Python wrapper"),
+        (f90_interface, f"{plugin.library_name}.f90", "Fortran interface"),
+    ]:
+        changed = _utils.write_file_if_changed(content, output_path, fname, force=regenerate)
+        logger.info("%s %s.", label, "changed" if changed else "is up to date")
+        any_changed |= changed
+
+    compilation_outputs_exist = (output_path / f"{plugin.library_name}.h").exists() and (
+        output_path / f"lib{plugin.library_name}{sysconfig.get_config_var('SHLIB_SUFFIX')}"
+    ).exists()
+    if not compilation_outputs_exist:
+        logger.info("Compilation outputs missing.")
+
+    if any_changed or not compilation_outputs_exist:
+        logger.info("Compiling CFFI dynamic library...")
+        _generator.generate_and_compile_cffi_plugin(
+            plugin.library_name, c_header, python_wrapper, output_path, rpath
+        )
+    else:
+        logger.info("All generated files are up to date. Skipping compilation.")
 
 
 if __name__ == "__main__":

tools/src/icon4py/tools/py2fgen/_utils.py

@@ -43,6 +43,24 @@ def write_file(string: str, outdir: pathlib.Path, fname: str) -> None:
         f.write(string)
 
 
+def write_file_if_changed(
+    content: str, outdir: pathlib.Path, fname: str, *, force: bool = False
+) -> bool:
+    """Write file only if its content differs from what is already on disk.
+
+    Args:
+        force: If True, always write the file regardless of current content.
+
+    Returns True if the file was written (content changed, file was new, or force=True),
+    False if the existing file already has the same content.
+    """
+    path = outdir / fname
+    if not force and path.exists() and path.read_text() == content:
+        return False
+    write_file(content, outdir, fname)
+    return True
+
+
 def setup_logger(name: str, log_level: int = logging.INFO) -> logging.Logger:
     """Set up a logger with a given name and log level."""
     logger = logging.getLogger(name)

tools/tests/tools/py2fgen/test_cli.py

@@ -6,6 +6,7 @@
 # Please, refer to the LICENSE file in the root directory.
 # SPDX-License-Identifier: BSD-3-Clause
 
+import logging
 import os
 import pathlib
 import subprocess
@@ -84,12 +85,15 @@ def run_test_case(
         )
 
 
-def invoke_cli(cli, module, function, library_name):
+def invoke_cli(cli, module, function, library_name, extra_args=None):
     rpath = utils.get_prefix_lib_path()
 
     cli_args = [module, function, library_name, "-r", rpath]
+    if extra_args:
+        cli_args.extend(extra_args)
     result = cli.invoke(main, cli_args)
     assert result.exit_code == 0, result.output
+    return result
 
 
 def compile_and_run_fortran(
@@ -284,3 +288,50 @@ def test_py2fgen_compilation_and_execution_dycore_gpu(cli_runner, samples_path,
         ("-acc", "-Minfo=acc"),
         env_vars={"ICON4PY_BACKEND": "GPU"},
     )
+
+
+def test_py2fgen_incremental_skips_compilation_when_unchanged(
+    cli_runner, square_wrapper_module, test_temp_dir, caplog
+):
+    """Test that running py2fgen twice without changes skips compilation on the second run."""
+    with cli_runner.isolated_filesystem(temp_dir=test_temp_dir):
+        # First run: should generate and compile
+        with caplog.at_level(logging.INFO, logger="py2fgen"):
+            caplog.clear()
+            invoke_cli(cli_runner, square_wrapper_module, "square_from_function", "square_plugin")
+            first_log = caplog.text
+        assert "Compiling CFFI dynamic library" in first_log
+
+        # Second run: all files should be up to date, compilation should be skipped
+        with caplog.at_level(logging.INFO, logger="py2fgen"):
+            caplog.clear()
+            invoke_cli(cli_runner, square_wrapper_module, "square_from_function", "square_plugin")
+            second_log = caplog.text
+        assert "Python wrapper is up to date" in second_log
+        assert "Fortran interface is up to date" in second_log
+        assert "Skipping compilation" in second_log
+        assert "Compiling CFFI dynamic library" not in second_log
+
+
+def test_py2fgen_regenerate_forces_recompilation(
+    cli_runner, square_wrapper_module, test_temp_dir, caplog
+):
+    """Test that --regenerate forces recompilation even if files are up to date."""
+    with cli_runner.isolated_filesystem(temp_dir=test_temp_dir):
+        # First run: generate and compile
+        invoke_cli(cli_runner, square_wrapper_module, "square_from_function", "square_plugin")
+
+        # Second run with --regenerate: should recompile
+        with caplog.at_level(logging.INFO, logger="py2fgen"):
+            caplog.clear()
+            invoke_cli(
+                cli_runner,
+                square_wrapper_module,
+                "square_from_function",
+                "square_plugin",
+                extra_args=["--regenerate"],
+            )
+            regen_log = caplog.text
+        assert "Force regeneration requested" in regen_log
+        assert "Compiling CFFI dynamic library" in regen_log
+        assert "Skipping compilation" not in regen_log

tools/tests/tools/py2fgen/test_utils.py

@@ -0,0 +1,34 @@
+# ICON4Py - ICON inspired code in Python and GT4Py
+#
+# Copyright (c) 2022-2024, ETH Zurich and MeteoSwiss
+# All rights reserved.
+#
+# Please, refer to the LICENSE file in the root directory.
+# SPDX-License-Identifier: BSD-3-Clause
+
+import pathlib
+
+from icon4py.tools.py2fgen._utils import write_file_if_changed
+
+
+def test_write_file_if_changed_creates_new_file(tmp_path: pathlib.Path):
+    assert write_file_if_changed("hello", tmp_path, "test.txt") is True
+    assert (tmp_path / "test.txt").read_text() == "hello"
+
+
+def test_write_file_if_changed_skips_unchanged(tmp_path: pathlib.Path):
+    write_file_if_changed("hello", tmp_path, "test.txt")
+    assert write_file_if_changed("hello", tmp_path, "test.txt") is False
+    assert (tmp_path / "test.txt").read_text() == "hello"
+
+
+def test_write_file_if_changed_writes_when_changed(tmp_path: pathlib.Path):
+    write_file_if_changed("hello", tmp_path, "test.txt")
+    assert write_file_if_changed("world", tmp_path, "test.txt") is True
+    assert (tmp_path / "test.txt").read_text() == "world"
+
+
+def test_write_file_if_changed_force_rewrites_unchanged(tmp_path: pathlib.Path):
+    write_file_if_changed("hello", tmp_path, "test.txt")
+    assert write_file_if_changed("hello", tmp_path, "test.txt", force=True) is True
+    assert (tmp_path / "test.txt").read_text() == "hello"