esm-tools · mandresm · Feb 6, 2025 · Feb 6, 2025 · Feb 7, 2025 · Feb 14, 2025
diff --git a/configs/components/fesom/fesom-2.0.yaml b/configs/components/fesom/fesom-2.0.yaml
@@ -440,4 +440,5 @@ fesom:
         - "throw_away_some_infiles"
         - "copy_stuff_back_from_work"
         - "copy_all_results_to_exp"
+        - "dump_file_tracker_log"
         - "clean_run_dir"
diff --git a/configs/components/fesom/fesom-2.1.yaml b/configs/components/fesom/fesom-2.1.yaml
@@ -624,6 +624,7 @@ fesom:
         - "throw_away_some_infiles"
         - "copy_stuff_back_from_work"
         - "copy_all_results_to_exp"
+        - "dump_file_tracker_log"
         - "clean_run_dir"
 
 computer:

diff --git a/configs/components/fesom/fesom-2.5.yaml b/configs/components/fesom/fesom-2.5.yaml
@@ -659,4 +659,5 @@ fesom:
     - "throw_away_some_infiles"
     - "copy_stuff_back_from_work"
     - "copy_all_results_to_exp"
+    - "dump_file_tracker_log"
     - "clean_run_dir"
diff --git a/configs/components/fesom/fesom-2.6.yaml b/configs/components/fesom/fesom-2.6.yaml
@@ -712,5 +712,6 @@ fesom:
     - "throw_away_some_infiles"
     - "copy_stuff_back_from_work"
     - "copy_all_results_to_exp"
+    - "dump_file_tracker_log"
     - "clean_run_dir"
 
diff --git a/configs/components/pism/pism.recipes.yaml b/configs/components/pism/pism.recipes.yaml
@@ -30,7 +30,7 @@ pism:
 
               #- "add_batch_hostfile"
             - "copy_files_to_work"
-            - "compute_and_log_file_checksums"
+            - "log_files_in_target_to_yaml"
               #- "write_simple_runscript"
             - "report_missing_files"
             #- "add_vcs_info"

diff --git a/configs/components/pism_nh/pism_nh.recipes.yaml b/configs/components/pism_nh/pism_nh.recipes.yaml
@@ -30,7 +30,7 @@ pism_nh:
 
           #- "add_batch_hostfile"
         - "copy_files_to_work"
-        - "compute_and_log_file_checksums"
+        - "log_files_in_target_to_yaml"
           #- "write_simple_runscript"
         - "report_missing_files"
         #- "add_vcs_info"

diff --git a/configs/components/pism_sh/pism_sh.recipes.yaml b/configs/components/pism_sh/pism_sh.recipes.yaml
@@ -30,7 +30,7 @@ pism_sh:
 
           #- "add_batch_hostfile"
         - "copy_files_to_work"
-        - "compute_and_log_file_checksums"
+        - "log_files_in_target_to_yaml"
           #- "write_simple_runscript"
         - "report_missing_files"
         #- "add_vcs_info"

diff --git a/configs/esm_software/esm_runscripts/esm_plugins.yaml b/configs/esm_software/esm_runscripts/esm_plugins.yaml
@@ -23,6 +23,7 @@ core:
                         - "install_missing_plugins"
                         - "add_submission_info"
                         - "initialize_batch_system"
+                        - "initialize_file_tracker"
                         - "initialize_coupler"
                         - "set_logfile"
                         #- "add_vcs_info"
@@ -86,7 +87,8 @@ core:
                         - "log_used_files"
                         - "report_missing_files"
                         - "check_for_unknown_files"
-                        - "compute_and_log_file_checksums"
+                        - "log_files_in_target_to_yaml"
+                        - "dump_file_tracker_log"
 
                 database_actions:
                         - "database_entry"

diff --git a/configs/esm_software/esm_runscripts/esm_runscripts.yaml b/configs/esm_software/esm_runscripts/esm_runscripts.yaml
@@ -32,6 +32,7 @@ choose_job_type:
                         - "restore_protected_last_minute_changes"
                         - "add_submission_info"
                         - "initialize_batch_system"
+                        - "initialize_file_tracker"
                         - "initialize_coupler"
                         - "set_logfile"
                         - "check_config_for_warnings_errors"
@@ -71,6 +72,7 @@ choose_job_type:
                         - "throw_away_some_infiles"
                         - "copy_stuff_back_from_work"
                         - "copy_all_results_to_exp"
+                        - "dump_file_tracker_log"
                         - "clean_run_dir"
 
         prepcompute:
@@ -87,8 +89,9 @@ choose_job_type:
                         - "modify_namelists"
                         - "modify_files"
                         - "copy_files_to_work"
+                        - "dump_file_tracker_log"
                         - "report_missing_files"
-                        - "compute_and_log_file_checksums"
+                        - "log_files_in_target_to_yaml"
                         #- "add_vcs_info"
                         #- "check_vcs_info_against_last_run"
                         - "_write_finalized_config"

diff --git a/configs/machines/albedo.yaml b/configs/machines/albedo.yaml
@@ -14,10 +14,12 @@ computer:
     accounting: true
     further_reading:
         - batch_system/slurm.yaml
+    partition_cpn: 128
+
     partitions:
         compute:
             name: mpp
-            cores_per_node: 128
+            cores_per_node: ${computer.partition_cpn}
         pp:
             name: smp
             cores_per_node: 1

diff --git a/configs/setups/awiesm/awiesm-2.2.yaml b/configs/setups/awiesm/awiesm-2.2.yaml
@@ -18,7 +18,7 @@ general:
             - "modify_files"
             - "copy_files_to_work"
             - "report_missing_files"
-            - "compute_and_log_file_checksums"
+            - "log_files_in_target_to_yaml"
             #- "add_vcs_info"
             #- "check_vcs_info_against_last_run"
             - "_write_finalized_config"

diff --git a/configs/setups/awiesm/awiesm.yaml b/configs/setups/awiesm/awiesm.yaml
@@ -52,7 +52,7 @@ general:
                 - "modify_files"
                 - "copy_files_to_work"
                 - "report_missing_files"
-                - "compute_and_log_file_checksums"
+                - "log_files_in_target_to_yaml"
                 - "_write_finalized_config"
                 - "database_entry"
             "*":
@@ -71,7 +71,7 @@ general:
                 - "modify_files"
                 - "copy_files_to_work"
                 - "report_missing_files"
-                - "compute_and_log_file_checksums"
+                - "log_files_in_target_to_yaml"
                 #- "add_vcs_info"
                 #- "check_vcs_info_against_last_run"
                 - "_write_finalized_config"

diff --git a/configs/setups/focioifs/focioifs.yaml b/configs/setups/focioifs/focioifs.yaml
@@ -20,7 +20,7 @@ general:
            - "modify_files"
            - "copy_files_to_work"
            - "report_missing_files"
-           - "compute_and_log_file_checksums"
+           - "log_files_in_target_to_yaml"
            # not working, bug already reported at https://github.com/esm-tools/esm_tools/discussions/774
            # - "add_vcs_info"
            # - "check_vcs_info_against_last_run"

diff --git a/docs/esm_runscripts.rst b/docs/esm_runscripts.rst
@@ -365,6 +365,54 @@ As an example, to configure ``esm_runscripts`` for an echam-experiment to link t
 Both ways to set the entries are doing the same thing. It is possible, as in the ``input`` case, to set the file movement method independently for each of the
 directions; the setting ``all_directions`` is just a shortcut if the method is identical for all of them.
 
+File Logging and Checksums
+--------------------------
+
+During experiment execution, ``esm_runscripts`` can generate a YAML file listing all
+files that were copied or linked into the work directory, along with optional MD5
+checksums. This is useful for:
+
+* **Reproducibility**: Track exactly which files were used in a run
+* **Debugging**: Identify missing or changed files between runs
+* **Validation**: Compare file manifests between baseline and test runs
+
+The file list is written to the ``log`` directory of each run as
+``<expid>_<model>_filelist_<datestamp>.yaml`` and contains:
+
+* Source, intermediate, and target paths for each file
+* The file category (input, forcing, restart, etc.)
+* Whether the file exists in the target directory
+* MD5 checksum (if enabled)
+
+To enable checksum computation, add the following to your runscript:
+
+.. code-block:: yaml
+
+   general:
+       compute_file_checksums: True
+
+.. note::
+   Computing checksums adds overhead to the job preparation phase, especially for
+   large files or many files. For production runs where reproducibility tracking
+   is not needed, leave this option disabled (the default).
+
+The generated YAML file has the following structure:
+
+.. code-block:: yaml
+
+   fesom:
+       forcing_sss:
+           source: /pool/forcing/sss.nc
+           intermediate: /work/exp/run_123/sss.nc
+           target: /work/exp/run_123/work/sss.nc
+           kind: forcing
+           checksum: a1b2c3d4e5f6...
+           exists: true
+   general:
+       # ... general files ...
+   not_handled_by_filelists:
+       # ... files found in work but not tracked by esm_runscripts ...
+
 Parallel File Movements
 -----------------------
 

diff --git a/runscripts/fesom2/fesom2.6-albedo-test.yaml b/runscripts/fesom2/fesom2.6-albedo-test.yaml
@@ -0,0 +1,26 @@
+general:
+    user: !ENV ${USER}
+    account: "computing.computing"
+    setup_name: fesom
+    compute_time: "00:30:00"
+    initial_date: "1958-01-01"
+    final_date: "1958-04-30"
+    base_dir: "/albedo/work/user/${user}/experiments/${general.setup_name}-${general.version}/"
+    nyear: 0
+    nmonth: 1
+    use_venv: True
+    install_esm_tools_branch: "feat/write_files_yaml"
+    compute_file_checksums: True
+    parallel_file_movements: "dask"
+
+fesom:
+    version: 2.6
+    model_dir: "/albedo/work/user/${user}/Models/fesom-2.6/"
+    mesh_dir: "/albedo/pool/FESOM2/core2/"
+    forcing_data_dir: "/albedo/work/projects/p_pool_fesom1/forcing/"
+    restart_rate: 1
+    restart_first: 1
+    restart_unit: "m"
+    resolution: "CORE2"
+    lresume: false
+    time_step: 1800
diff --git a/src/esm_parser/provenance.py b/src/esm_parser/provenance.py
@@ -1089,8 +1089,10 @@ def clean_provenance(data, nested=False):
             clean_provenance(key, nested=True): clean_provenance(value, nested=True)
             for key, value in data.items()
         }
-    # Clean vars in objects
+    # Clean vars in objects (skip objects marked with _skip_provenance)
     elif hasattr(data, "__dict__"):
+        if getattr(data, "_skip_provenance", False):
+            return data
         for key, value in vars(data).items():
             setattr(data, key, clean_provenance(value, nested=True))
         return data