postprocess: add --exclude-file for skipping functions (#1541)

As a general-purpose solution to bypassing functions the postprocessor's
comment transferer (and future transforms, too) can't handle yet, this
adds an exclude list to skip functions in files. There is also
`--ident-filter`, which is a single regex just based on the function
name, but this is more specific and also takes into account the file
name. The file passed to `--exclude-file` is a YAML file containing file
paths (relative to the exclude file's directory) and the function names
within them to skip. This gets us around issues like
conditionally-compiled comments (#1534), the first function in a file
also including license comments and `#include`s, and anything else. I've
run this on `json-c` so far

```sh
(cd c2rust-postprocess/ && uv run postprocess $OLDPWD/tests/integration/tests/json-c/repo/lib.rs --no-update-rust --exclude-file $OLDPWD/tests/integration/tests/json-c/postprocess-exclude.yml)
```

and checked-in the LLM cache for it, but setting up the testing
infrastructure to run this automatically is still a WIP (should be the
next PR). Funnily enough, the LLM cache is going to end up behaving
similarly to snapshot tests once it's up and running.

Author: kkysen

c2rust-postprocess/README.md

@@ -17,6 +17,39 @@ accelerate the types of translation and migration that help move C code to Rust.
 - `c2rust-postprocess path/to/transpiled_rust.rs`, or
 - `uv run postprocess path/to/transpiled_rust.rs`
 
+## Excluding/Filtering
+
+`c2rust-postprocess` has a few ways to filter/exclude the function identifiers that are processed.
+
+`--ident-filter` simply takes a regex.
+Anything matching the regex is filtered out of being processed.
+This is very useful for on-the-fly filtering that's easy to change quickly.
+
+`--exclude-file` is for more granular, more permanent filtering/exclusion.
+It takes a path to an exclude file, which we tend to call `postprocess-exclude.yml`.
+This is a YAML file containing the file paths and function identifiers to exclude.
+For example,
+
+```yaml
+src/lib.rs:
+  - foo
+  - bar
+
+src/main.rs:
+  - main
+```
+
+will exclude `fn foo` and `fn bar` in `src/lib.rs`
+(resolved relative to the directory of `postprocess-exclude.yml`)
+and `fn main` in `src/main.rs`.
+
+We filter based on the Rust file instead of the C file
+because the C file may not be available and isn't read by `c2rust-postprocess`
+(only the adjacent `*.c_decls.json` file is).
+
+These file paths and function identifiers must match fully.
+They are not regexes or globs.
+
 # Testing
 
 ## Test prerequisites

c2rust-postprocess/postprocess/__init__.py

@@ -6,10 +6,12 @@
 import logging
 from argparse import BooleanOptionalAction
 from collections.abc import Sequence
+from pathlib import Path
 
 from google.genai import types
 
 from postprocess.cache import DirectoryCache, FrozenCache
+from postprocess.exclude_list import IdentifierExcludeList
 from postprocess.models import api_key_from_env, get_model_by_id
 from postprocess.models.mock import MockGenerativeModel
 from postprocess.transforms import (
@@ -45,7 +47,17 @@ def build_arg_parser() -> argparse.ArgumentParser:
         type=str,
         required=False,
         default=None,
-        help="Regular expression to filter function identifiers to process",
+        help="Regular expression to filter function identifiers to process"
+        " (see README for more info)",
+    )
+
+    parser.add_argument(
+        "--exclude-file",
+        type=Path,
+        required=False,
+        default=None,
+        help="A YAML file of file paths and identifiers within them to exclude/skip"
+        " (see README for more info)",
     )
 
     parser.add_argument(
@@ -116,6 +128,7 @@ def main(argv: Sequence[str] | None = None):
         xform = CommentTransfer(cache, model)
         xform.transfer_comments_dir(
             root_rust_source_file=args.root_rust_source_file,
+            exclude_list=IdentifierExcludeList(src_path=args.exclude_file),
             ident_filter=args.ident_filter,
             update_rust=args.update_rust,
         )

c2rust-postprocess/postprocess/cache.py

@@ -11,6 +11,8 @@
 from platformdirs import user_cache_dir
 from tomlkit.items import String
 
+from postprocess.utils import check_isinstance
+
 
 class AbstractCache(ABC):
     """
@@ -94,12 +96,13 @@ def convert_value(value: TomlValue) -> TomlValue | String:
             return {k: convert_value(v) for k, v in value.items() if v is not None}
         elif isinstance(value, list):
             return [convert_value(e) for e in value if e is not None]
+        elif value is None:
+            raise TypeError("top-level `None` `TomlValue`s are not allowed")
         else:
-            assert value is not None
             return value
 
     converted_value = convert_value(value)
-    assert isinstance(converted_value, dict)
+    converted_value = check_isinstance(converted_value, dict)
     doc = tomlkit.document()
     for k, v in converted_value.items():
         doc[k] = v
@@ -173,9 +176,16 @@ def lookup(
         try:
             toml = cache_file.read_text()
         except FileNotFoundError:
-            logging.debug(f"Cache miss: {cache_file}: {messages}")
+            data = {
+                "transform": transform,
+                "identifier": identifier,
+                "model": model,
+                "messages": messages,
+            }
+            toml = to_multiline_toml(data)
+            logging.debug(f"Cache miss: {cache_file}:\n{toml}")
             return None
-        logging.debug(f"Cache hit: {cache_file}")
+        logging.debug(f"Cache hit: {cache_file}:\n{toml}")
         data = tomli.loads(toml)
 
         return data["response"]
@@ -206,7 +216,7 @@ def update(
         response_path = cache_dir / "response.txt"
         metadata_path.write_text(toml)
         response_path.write_text(response)
-        logging.debug(f"Cache updated: {cache_dir}")
+        logging.debug(f"Cache updated: {cache_dir}:\n{toml}")
 
     def clear(self) -> None:
         self._path.unlink(missing_ok=True)

c2rust-postprocess/postprocess/definitions/__init__.py

@@ -174,6 +174,7 @@ def get_comments_text(comments: Iterable[str]) -> list[str]:
         comment_line
         for comment in comments
         for comment_line in get_comment_text(comment)
+        if comment_line
     ]
 
 

c2rust-postprocess/postprocess/exclude_list.py

@@ -0,0 +1,37 @@
+from pathlib import Path
+
+import yaml
+
+from postprocess.utils import check_isinstance
+
+
+class IdentifierExcludeList:
+    src_path: Path | None
+    paths: dict[Path, set[str]]
+
+    def __init__(self, src_path: Path | None) -> None:
+        self.src_path = src_path
+        self.paths = {}
+        if src_path is None:
+            return
+        data = yaml.safe_load(src_path.read_text())
+        data = check_isinstance(data, dict)
+        for path, identifiers in data.items():
+            path = check_isinstance(path, str)
+            identifiers = check_isinstance(identifiers, list)
+            identifiers = [check_isinstance(ident, str) for ident in identifiers]
+            path = Path(path)
+            existing_identifiers = self.paths.get(path)
+            if existing_identifiers is None:
+                self.paths[path] = set(identifiers)
+            else:
+                existing_identifiers.update(*identifiers)
+
+    def contains(self, path: Path, identifier: str) -> bool:
+        # No `src_path` means an empty exclude list.
+        if self.src_path is None:
+            return False
+        # Consider paths relative to `src_path`, the location of the exclude file.
+        rel_path: Path = path.relative_to(self.src_path.parent)
+        identifiers = self.paths.get(rel_path, set())
+        return identifier in identifiers

c2rust-postprocess/postprocess/transforms.py

@@ -11,6 +11,7 @@
     get_rust_definitions,
     update_rust_definition,
 )
+from postprocess.exclude_list import IdentifierExcludeList
 from postprocess.models import AbstractGenerativeModel
 from postprocess.utils import get_highlighted_c, get_highlighted_rust, remove_backticks
 
@@ -58,10 +59,11 @@ def transfer_comments(
         self,
         root_rust_source_file: Path,
         rust_source_file: Path,
+        exclude_list: IdentifierExcludeList,
         ident_filter: str | None = None,
         update_rust: bool = True,
     ) -> None:
-        pattern = re.compile(ident_filter) if ident_filter else None
+        ident_regex = re.compile(ident_filter) if ident_filter else None
 
         rust_definitions = get_rust_definitions(rust_source_file)
         c_definitions = get_c_definitions(rust_source_file)
@@ -71,7 +73,17 @@ def transfer_comments(
 
         prompts: list[CommentTransferPrompt] = []
         for identifier, rust_definition in rust_definitions.items():
-            if pattern and not pattern.search(identifier):
+            if exclude_list.contains(path=rust_source_file, identifier=identifier):
+                logging.info(
+                    f"Skipping Rust fn {identifier} in {rust_source_file}"
+                    f"due to exclude file {exclude_list.src_path}"
+                )
+                continue
+            if ident_regex and not ident_regex.search(identifier):
+                logging.info(
+                    f"Skipping Rust fn {identifier} in {rust_source_file}"
+                    f"due to ident filter {ident_filter}"
+                )
                 continue
 
             rust_comments = get_rust_comments(rust_definition)
@@ -170,6 +182,7 @@ def transfer_comments(
     def transfer_comments_dir(
         self,
         root_rust_source_file: Path,
+        exclude_list: IdentifierExcludeList,
         ident_filter: str | None = None,
         update_rust: bool = True,
     ):
@@ -187,6 +200,7 @@ def transfer_comments_dir(
             self.transfer_comments(
                 root_rust_source_file=root_rust_source_file,
                 rust_source_file=rs_path,
+                exclude_list=exclude_list,
                 ident_filter=ident_filter,
                 update_rust=update_rust,
             )

c2rust-postprocess/postprocess/utils.py

@@ -13,6 +13,12 @@
 from requests.exceptions import JSONDecodeError
 
 
+def check_isinstance[T](value: object, typ: type[T]) -> T:
+    if not isinstance(value, typ):
+        raise TypeError(f"Expected {typ}, got {type(value)}")
+    return value
+
+
 def get_tool_path(tool_name: str) -> Path:
     """Get the path to the given tool in the system PATH."""
 

c2rust-postprocess/pyproject.toml

@@ -8,6 +8,7 @@ dependencies = [
     "openai>=2.9.0",
     "platformdirs>=4.5.1",
     "pygments>=2.19.2",
+    "pyyaml>=6.0.3",
     "tomli>=2.3.0",
     "tomlkit>=0.13.3",
     "tree-sitter>=0.25.2",

c2rust-postprocess/tests/examples/qsort.rs

@@ -7,10 +7,7 @@
     unused_mut
 )]
 #[no_mangle]
-pub unsafe extern "C" fn swap(
-    mut a: *mut ::core::ffi::c_int,
-    mut b: *mut ::core::ffi::c_int,
-) {
+pub unsafe extern "C" fn swap(mut a: *mut ::core::ffi::c_int, mut b: *mut ::core::ffi::c_int) {
     let mut t: ::core::ffi::c_int = *a;
     *a = *b;
     *b = t;

c2rust-postprocess/tests/llm-cache/CommentTransfer/_json_c_set_last_err/9adf00d95ad0a6f053bb53b6cd6f44351e53f231893f4c9666ed52cf00d9921a/metadata.toml

@@ -0,0 +1,56 @@
+transform = "CommentTransfer"
+identifier = "_json_c_set_last_err"
+model = "gemini-3-flash-preview"
+response = """#[no_mangle]
+
+pub unsafe extern "C" fn _json_c_set_last_err(
+    mut err_fmt: *const ::core::ffi::c_char,
+    mut args: ...
+) {
+    let mut ap: ::core::ffi::VaListImpl;
+    ap = args.clone();
+    // Ignore (attempted) overruns from snprintf
+    vsnprintf(
+        &raw mut _last_err as *mut ::core::ffi::c_char,
+        ::core::mem::size_of::<[::core::ffi::c_char; 256]>() as size_t,
+        err_fmt,
+        ap.as_va_list(),
+    );
+}"""
+
+[[messages]]
+role = "user"
+content = """Transfer the comments from the following C function to the corresponding Rust function.
+Do not add any comments that are not present in the C function.
+Use Rust doc comment syntax (///) where appropriate (e.g., for function documentation).
+Respond with the Rust function definition with the transferred comments; say nothing else.
+
+C function:
+```c
+void _json_c_set_last_err(const char *err_fmt, ...)
+{
+\tva_list ap;
+\tva_start(ap, err_fmt);
+\t// Ignore (attempted) overruns from snprintf
+\t(void)vsnprintf(_last_err, sizeof(_last_err), err_fmt, ap);
+\tva_end(ap);
+}```
+
+Rust function:
+```rust
+#[no_mangle]
+
+pub unsafe extern "C" fn _json_c_set_last_err(
+    mut err_fmt: *const ::core::ffi::c_char,
+    mut args: ...
+) {
+    let mut ap: ::core::ffi::VaListImpl;
+    ap = args.clone();
+    vsnprintf(
+        &raw mut _last_err as *mut ::core::ffi::c_char,
+        ::core::mem::size_of::<[::core::ffi::c_char; 256]>() as size_t,
+        err_fmt,
+        ap.as_va_list(),
+    );
+}```
+"""