Add optional tmpfs storage for fuzzing corpus (#461)

* Add optional tmpfs storage for fuzzing corpus

Add support for storing the node-local fuzzing corpus in tmpfs (RAM-based
filesystem) for improved I/O performance during fuzzing operations.

Key changes:
- Add CORPUS_TMPFS_PATH environment variable to configure tmpfs location
- Add cross-filesystem safe operations (copy+delete fallback for EXDEV)
- Update Corpus class to use tmpfs path when enabled while maintaining
  correct remote path calculation for rsync synchronization
- Add Helm chart configuration using /dev/shm (requires no host setup)
- Update fuzzer-bot, coverage-bot, merger-bot, seed-gen deployments

The feature is disabled by default. Enable by setting
global.volumes.corpusTmpfs.enabled=true in values.yaml.

Uses hostPath to /dev/shm/buttercup-corpus which is available on
Linux systems without requiring any Kubernetes host configuration.

Author: hbrodin

common/src/buttercup/common/corpus.py

@@ -18,6 +18,37 @@
 # TODO: this file is one of the few files that uses os.path.join. Switch to Path.
 
 
+def _get_corpus_storage_path(wdir: str, corpus_subpath: str) -> tuple[str, Path]:
+    """Calculate the local storage path and remote path for corpus.
+
+    When tmpfs is enabled, the local path is on tmpfs but the remote path
+    is still calculated relative to NODE_DATA_DIR.
+
+    Args:
+        wdir: The working directory (typically NODE_DATA_DIR)
+        corpus_subpath: The corpus subdirectory (e.g., "{task_id}/buttercup_corpus_{harness}")
+
+    Returns:
+        Tuple of (local_path, remote_path)
+    """
+    tmpfs_path = node_local.get_corpus_tmpfs_path()
+
+    if tmpfs_path:
+        # Store corpus on tmpfs
+        local_path = os.path.join(str(tmpfs_path), corpus_subpath)
+        # Remote path is still calculated relative to NODE_DATA_DIR structure
+        # We compute it as if the corpus were stored in wdir
+        canonical_path = os.path.join(wdir, corpus_subpath)
+        remote_path = node_local.remote_path(Path(canonical_path))
+        logger.debug(f"Using tmpfs for corpus: local={local_path}, remote={remote_path}")
+    else:
+        # Standard behavior - store corpus in wdir
+        local_path = os.path.join(wdir, corpus_subpath)
+        remote_path = node_local.remote_path(Path(local_path))
+
+    return local_path, remote_path
+
+
 def hash_file(fl: BinaryIO) -> str:
     h = hashlib.new("sha256")
     bts = fl.read(100)
@@ -28,9 +59,34 @@ def hash_file(fl: BinaryIO) -> str:
 
 
 class InputDir:
-    def __init__(self, wdir: str, name: str, copy_corpus_max_size: int | None = None):
-        self.path = os.path.join(wdir, name)
-        self.remote_path = node_local.remote_path(Path(self.path))
+    def __init__(
+        self,
+        wdir: str,
+        name: str,
+        copy_corpus_max_size: int | None = None,
+        *,
+        override_local_path: str | None = None,
+        override_remote_path: Path | None = None,
+    ):
+        """Initialize an InputDir for corpus/crash storage.
+
+        Args:
+            wdir: Working directory (typically NODE_DATA_DIR)
+            name: Subdirectory name within wdir
+            copy_corpus_max_size: Maximum size for copied corpus files
+            override_local_path: If provided, use this as the local path instead of wdir/name
+            override_remote_path: If provided, use this as the remote path
+        """
+        if override_local_path is not None:
+            self.path = override_local_path
+        else:
+            self.path = os.path.join(wdir, name)
+
+        if override_remote_path is not None:
+            self.remote_path = override_remote_path
+        else:
+            self.remote_path = node_local.remote_path(Path(self.path))
+
         self.copy_corpus_max_size = copy_corpus_max_size
         os.makedirs(self.path, exist_ok=True)
 
@@ -110,9 +166,12 @@ def hash_corpus(cls, path: str) -> list[str]:
                 continue
             file_path = os.path.join(path, file)
             try:
+                assert os.path.isfile(file_path)
                 with open(file_path, "rb") as f:
                     hash_filename = hash_file(f)
-                os.rename(file_path, os.path.join(path, hash_filename))
+                dst_path = os.path.join(path, hash_filename)
+                # Use shutil.move which handles cross-filesystem operations
+                shutil.move(file_path, dst_path)
                 hashed_files.append(hash_filename)
             except Exception as e:
                 # Likely already hashed by another pod
@@ -224,7 +283,17 @@ def __init__(self, wdir: str, task_id: str, harness_name: str, copy_corpus_max_s
         self.task_id = task_id
         self.harness_name = harness_name
         self.corpus_dir = os.path.join(task_id, f"{CORPUS_DIR_NAME}_{harness_name}")
-        super().__init__(wdir, self.corpus_dir, copy_corpus_max_size=copy_corpus_max_size)
+
+        # Get the storage paths, potentially using tmpfs for the local path
+        local_path, remote_path = _get_corpus_storage_path(wdir, self.corpus_dir)
+
+        super().__init__(
+            wdir,
+            self.corpus_dir,
+            copy_corpus_max_size=copy_corpus_max_size,
+            override_local_path=local_path,
+            override_remote_path=remote_path,
+        )
 
     def remove_any_merged(self, redis: Redis) -> None:
         merged_corpus_set = MergedCorpusSet(redis, self.task_id, self.harness_name)

common/src/buttercup/common/node_local.py

@@ -1,4 +1,5 @@
 # Store the node local path for subsequent use
+import errno
 import logging
 import os
 import shutil
@@ -14,6 +15,10 @@
 
 node_local_path = os.getenv("NODE_DATA_DIR")
 
+# Optional tmpfs path for corpus storage
+# When set, the fuzzing corpus will be stored on this tmpfs mount for improved performance
+corpus_tmpfs_path = os.getenv("CORPUS_TMPFS_PATH")
+
 
 NodeLocalPath: TypeAlias = Path
 RemotePath: TypeAlias = Path
@@ -24,6 +29,18 @@ def _get_root_path() -> NodeLocalPath:
     return NodeLocalPath(node_local_path)
 
 
+def get_corpus_tmpfs_path() -> Path | None:
+    """Return the tmpfs path for corpus storage if configured, otherwise None."""
+    if corpus_tmpfs_path:
+        return Path(corpus_tmpfs_path)
+    return None
+
+
+def is_corpus_tmpfs_enabled() -> bool:
+    """Check if tmpfs corpus storage is enabled."""
+    return corpus_tmpfs_path is not None
+
+
 class TmpDir:
     def __init__(self, path: Path):
         self.path = path
@@ -48,7 +65,10 @@ def temp_dir(root_path: Path) -> Iterator[TmpDir]:
 
 
 def rename_atomically(src: Path, dst: Path) -> Path | None:
-    """Rename a file atomically"""
+    """Rename a file atomically.
+
+    Falls back to copy+delete for cross-filesystem operations.
+    """
     src = Path(src)
     dst = Path(dst)
     dst.parent.mkdir(parents=True, exist_ok=True)
@@ -57,13 +77,50 @@ def rename_atomically(src: Path, dst: Path) -> Path | None:
     except OSError as e:
         # If the path already exists, it means another pod already downloaded it
         # we can just ignore this error and return None to signify that the path already exists
-        if e.errno == 39:
+        if e.errno == errno.ENOTEMPTY:  # 39 = Directory not empty
             logger.debug(f"Local path {dst} already exists for {src}")
             return None
+        # Handle cross-filesystem rename (errno 18 = EXDEV = Invalid cross-device link)
+        if e.errno == errno.EXDEV:
+            logger.debug(f"Cross-filesystem rename from {src} to {dst}, using copy+delete")
+            return _copy_and_delete(src, dst)
         raise e
     return dst
 
 
+def _copy_and_delete(src: Path, dst: Path) -> Path | None:
+    """Copy a file/directory and delete the source. Used for cross-filesystem operations.
+
+    Note: Unlike os.rename, this intentionally returns None if dst already exists
+    rather than overwriting. This supports the "first pod wins" pattern used by
+    callers in distributed scenarios where concurrent pods may race to create the same file.
+    """
+    try:
+        if src.is_dir():
+            # For directories, use copytree
+            if dst.exists():
+                logger.debug(f"Destination {dst} already exists, skipping copy")
+                shutil.rmtree(src, ignore_errors=True)
+                return None
+            shutil.copytree(src, dst)
+            shutil.rmtree(src, ignore_errors=True)
+        else:
+            # For files, use copy2 to preserve metadata
+            if dst.exists():
+                logger.debug(f"Destination {dst} already exists, skipping copy")
+                try:
+                    os.unlink(src)
+                except OSError:
+                    pass
+                return None
+            shutil.copy2(src, dst)
+            os.unlink(src)
+        return dst
+    except Exception as e:
+        logger.error(f"Failed to copy {src} to {dst}: {e}")
+        raise
+
+
 def remote_path(local_path: NodeLocalPath) -> RemotePath:
     """Convert the node local path to a remote path"""
     local_path = Path(local_path)