Support writable overlay mounts on squash mounts

Operating systems like DOS, Amiga, macOS, and others expect to change the
filesystem, and this is frequently used by games to save settings, write save
game files, (among other uses).

Because squashfs is a read-only mount, this limits its use to games that never
write any data. Figuring this out for DOS games can be trial and error, and
worst case the user has to forfeit the tidy benefits of using squash.

This PR lets emulators indicate that they need write access to the ROM area, in
which case a writable overlayfs mount will be placed on top of the squashfs
mount (if squash is used), with the writes going into the usual
`/userdata/saves/<platform>/<game>` area.

The overlay feature is a follow-on context after squash:
 - It handles squash's directory or single-file and linked output
 - It logs and cleans up the virtual area on context close
 - It preserves the writeable area if changes were made, but otherwise cleans
   it up if it's empty.

Log staments:

```
DEBUG (overlayfs.py:43):_mount mounted '/var/run/overlays/biomenace-bare.pc'
      with components 'lowerdir=/var/run/squashfs/biomenace-bare.pc,
                       upperdir=/userdata/saves/dos/biomenace-bare.pc/upper,
                       workdir=/userdata/saves/dos/biomenace-bare.pc/work'

DEBUG (overlayfs.py:85):mount_overlayfs cleaning up
      '/var/run/overlays/biomenace-bare.pc'

DEBUG (overlayfs.py:90):mount_overlayfs keeping populated save directory
      '/userdata/saves/dos/biomenace-bare.pc'
```

I've tested all four types of changes:
 - Modifying an existing file in the squash base
 - Deleting an existing file in the squash base
 - Adding or changing a new file that doesn't exist in the squash base
 - Deleting a previously created file not in the squash base

Using squashfs for PC games has a nice side-effect of keeping the underlying
game in a known pristine state as any risky changes (patches, cracks, cheats
and so on that a user might employ) can be wiped clean by deleting the save
area.

As of this PR I've only toggled it on for the DOSBox Staging generator, but
could be easily be added to others if someone has working ROMs for those
platforms (Amiga, MacOS, etc..)

Author: dux119-cmd

batocera-Changelog.md

@@ -78,6 +78,7 @@
 - You can now choose to create a Win32 WINE bottle only via the option to run 32-bit Windows games.
 - DOSBox Staging's working directory is now set to the games' folder, allowing for local and relative (img)mount and conf file references.
 - DOSBox Staging will fallback to a C:\> prompt inside the games' folder if its missing dosbox.cfg/.conf/.bat files.
+- DOSBox Staging now stores DOS filesystem changes in /userdata/saves/dos/<game> for squashfs ROMs.
 - Systems like WINE and DOSBOX can now be prepared from PCManFM context menu. Right click on file items inside supported ones.
   to presetup them. This is mostly thought for startup files like dosbox.bat and autorun.cmd and for handling squashed archive files.
 - RPCS3 PS Move (light gun) mapping simplified. D-pad buttons are now PS Move face buttons. Check wiki for more info.

package/batocera/core/batocera-configgen/configgen/configgen/emulatorlauncher.py

@@ -35,7 +35,8 @@
 from .utils.evmapy import evmapy
 from .utils.hotkeygen import set_hotkeygen_context
 from .utils.logger import setup_logging
-from .utils.squashfs import squashfs_rom
+from .utils.squashfs import mount_squashfs
+from .utils.overlayfs import mount_overlayfs
 
 if TYPE_CHECKING:
     from collections.abc import Iterable
@@ -55,12 +56,23 @@
 _evmapy_instance = None
 
 def main(args: argparse.Namespace, maxnbplayers: int) -> int:
+    original_rom = args.rom
+
     # squashfs roms if squashed
-    if args.rom.suffix == ".squashfs":
-        with squashfs_rom(args.rom) as rom:
-            return start_rom(args, maxnbplayers, rom, args.rom)
+    if original_rom.suffix == ".squashfs":
+        with mount_squashfs(original_rom) as squash_rom:
+
+            # Do we need a writable overlay for the read-only squash?
+            system = Emulator(args, original_rom)
+            generator = get_generator(system.config.emulator)
+            if generator.writesToRom():
+                rom_saves_dir = SAVES / original_rom.parent.name / original_rom.stem
+                with mount_overlayfs(squash_rom, rom_saves_dir) as overlay_rom:
+                    return start_rom(args, maxnbplayers, overlay_rom, original_rom)
+
+            return start_rom(args, maxnbplayers, squash_rom, original_rom)
     else:
-        return start_rom(args, maxnbplayers, args.rom, args.rom)
+        return start_rom(args, maxnbplayers, original_rom, original_rom)
 
 def start_rom(args: argparse.Namespace, maxnbplayers: int, rom: Path, original_rom: Path) -> int:
     global _active_player_controllers, _evmapy_instance

package/batocera/core/batocera-configgen/configgen/configgen/generators/Generator.py

@@ -38,6 +38,10 @@ def getMouseMode(self, config: SystemConfig, rom: Path) -> bool:
     def executionDirectory(self, config: SystemConfig, rom: Path) -> Path | None:
         return None
 
+    # Some systems expect to write into the ROM area, for example: DOS, Amiga, and Wine
+    def writesToRom(self) -> bool:
+        return False
+
     # mame or libretro have internal bezels, don't display the one of mangohud
     def supportsInternalBezels(self) -> bool:
         return False

package/batocera/core/batocera-configgen/configgen/configgen/generators/dosboxstaging/dosboxstagingGenerator.py

@@ -25,21 +25,24 @@ class DosBoxStagingGenerator(Generator):
     # Returns a populated Command object
     def generate(self, system, rom, playersControllers, metadata, guns, wheels, gameResolution):
 
+        # Handle the single-file ROM case
+        game_dir = rom if rom.is_dir() else rom.parent
+
         # DOSBox Staging common resource data and conf file
         common_resource_dir = CONFIGS / 'dosbox'
         common_resource_conf = _find_iname(common_resource_dir, "dosbox-staging.conf")
 
-        dosbox_cfg =  _find_iname(rom, "dosbox.cfg")
-        dosbox_conf = _find_iname(rom, "dosbox.conf")
-        dosbox_bat = _find_iname(rom, "dosbox.bat")
+        dosbox_cfg  = _find_iname(game_dir, "dosbox.cfg")
+        dosbox_conf = _find_iname(game_dir, "dosbox.conf")
+        dosbox_bat  = _find_iname(game_dir, "dosbox.bat")
 
         is_configured = dosbox_cfg or dosbox_conf or dosbox_bat
 
         commandArray = [
             '/usr/bin/dosbox-staging',
             "--fullscreen",
-            "--working-dir", str(rom),
-            "-c", f"set WORKDIR={rom}",
+            "--working-dir", str(game_dir),
+            "-c", f"set WORKDIR={game_dir}",
         ]
 
         if common_resource_dir.is_dir():
@@ -85,3 +88,6 @@ def getHotkeysContext(self) -> HotkeysContext:
             "name": "dosboxstaging",
             "keys": { "exit": ["KEY_LEFTCTRL", "KEY_F9"] }
         }
+
+    def writesToRom(self) -> bool:
+        return system.config.get_bool('dosbox_staging_writes_to_rom')

package/batocera/core/batocera-configgen/configgen/configgen/utils/overlayfs.py

@@ -0,0 +1,95 @@
+from __future__ import annotations
+
+import logging
+import shutil
+import subprocess
+from contextlib import contextmanager
+from pathlib import Path
+from typing import TYPE_CHECKING, Final
+
+from ..batoceraPaths import mkdir_if_not_exists
+from ..exceptions import BatoceraException
+
+if TYPE_CHECKING:
+    from collections.abc import Iterator
+
+_logger = logging.getLogger(__name__)
+
+_OVERLAY_BASE_DIR: Final = Path("/var/run/overlays/")
+
+
+def _unmount_and_remove(mount_point: Path):
+    if mount_point.is_mount():
+        result = subprocess.run(["umount", str(mount_point)], capture_output=True, text=True)
+        if result.returncode != 0:
+            _logger.error("failed unmounting '%s' (rc=%d) because %s",
+                          mount_point, result.returncode, result.stderr.strip())
+
+            # Skip the follow-on removal if the umount failed because it might still
+            # be connected to the ROM's save area (system crashing or bad state?).
+            return
+
+    shutil.rmtree(mount_point, ignore_errors=True)
+
+
+def _mount(lower: Path, upper: Path, work: Path, mount_point: Path) -> bool:
+    components = f"lowerdir={lower},upperdir={upper},workdir={work}"
+
+    result = subprocess.run(["mount", "-t", "overlay", "overlay",
+                             "-o", components, str(mount_point)],
+                             capture_output=True, text=True)
+
+    if result.returncode == 0:
+        _logger.debug("mounted '%s' with components '%s'",
+                       mount_point, components)
+    else:
+        _logger.error("failed mounting '%s' with components '%s' (rc=%d) because %s",
+                      mount_point, components, result.returncode, result.stderr.strip())
+
+    return result.returncode == 0
+
+
+@contextmanager
+def mount_overlayfs(lower_dir: Path, writes_dir: Path, /) -> Iterator[Path]:
+    """
+    Create an overlay mount for a read-only lower directory saving writes to the
+    writes_dir. Returns the merged overlay mount point.
+    """
+
+    # If we were passed a single file, then overlay its parent directory
+    lower_file = None
+    if lower_dir.is_file():
+        _logger.debug("overlaying single-file or linked rom '%s'", lower_dir)
+        lower_dir  = lower_dir.parent
+        lower_file = lower_dir.name
+
+    # Where overlayfs keeps persistent writes
+    upper_dir = writes_dir / "upper"
+    mkdir_if_not_exists(upper_dir)
+
+    # Where overlayfs manages in-flight writes
+    work_dir = writes_dir / "work"
+    mkdir_if_not_exists(work_dir)
+
+    # Where overlayfs exposes the combined filesystem
+    mount_point = _OVERLAY_BASE_DIR / lower_dir.name
+    _unmount_and_remove(mount_point)
+    mkdir_if_not_exists(mount_point)
+
+    if not _mount(lower_dir, upper_dir, work_dir, mount_point):
+        raise BatoceraException(f"Unable to setup writable overlay for '{lower_dir}'")
+    try:
+        yield mount_point / lower_file if lower_file else mount_point
+
+    finally:
+        _logger.debug("cleaning up '%s'", mount_point)
+        _unmount_and_remove(mount_point)
+
+        has_writes = upper_dir.is_dir() and any(upper_dir.iterdir())
+
+        _logger.debug("%s save directory '%s'",
+                      "keeping populated" if has_writes else "removing empty",
+                      writes_dir)
+
+        if not has_writes:
+            shutil.rmtree(writes_dir, ignore_errors=True)

package/batocera/core/batocera-configgen/configgen/configgen/utils/squashfs.py

@@ -18,8 +18,8 @@
 
 
 @contextmanager
-def squashfs_rom(rom: Path, /) -> Iterator[Path]:
-    _logger.debug("squashfs_rom(%s)", rom)
+def mount_squashfs(rom: Path, /) -> Iterator[Path]:
+    _logger.debug("mount_squashfs(%s)", rom)
     mount_point = _SQUASHFS_DIR / rom.stem
 
     mkdir_if_not_exists(_SQUASHFS_DIR)
@@ -63,12 +63,12 @@ def squashfs_rom(rom: Path, /) -> Iterator[Path]:
                 _logger.debug("squashfs: linked rom %s", rom_linked)
                 yield rom_linked
     finally:
-        _logger.debug("squashfs_rom: cleaning up %s", mount_point)
+        _logger.debug("mount_squashfs: cleaning up %s", mount_point)
 
         # unmount
         return_code = subprocess.call(["umount", mount_point])
         if return_code != 0:
-            _logger.debug("squashfs_rom: unmounting %s failed", mount_point)
+            _logger.debug("mount_squashfs: unmounting %s failed", mount_point)
             raise BatoceraException(f"Unable to unmount the file {mount_point}")
 
         # cleaning the empty directory

package/batocera/emulators/dosbox-staging/dosbox_staging.emulator.yml

@@ -17,3 +17,10 @@ systems:
   - name: dos
     exclude_extensions:
       - zip
+custom_features:
+  dosbox_staging_writes_to_rom:
+    prompt: Enable writable overlay for squashfs ROMs
+    description: Writes are saved per-game in saves/dos/game-rom-name
+    choices:
+      Enabled: True
+      Disabled: False