feat: optional spras revision

Author: tristan-f-r

spras/config/config.py

@@ -53,7 +53,17 @@ def spras_revision() -> str:
             return hashlib.file_digest(f, 'sha256').hexdigest()[:8]
     except importlib.metadata.PackageNotFoundError as err:
         raise RuntimeError('spras is not an installed pip-module: did you forget to install SPRAS as a module?') from err
-def attach_spras_revision(label: str) -> str:
+
+
+def attach_spras_revision(osdf_immutable: bool, label: str) -> str:
+    """
+    Attaches the SPRAS revision to a label.
+    This function signature may become more complex as specific labels get versioned.
+
+    @param label: The label to attach the SPRAS revision to.
+    @param osdf_immutable: if False, this function is equivalent to `id`.
+    """
+    if osdf_immutable is False: return label
     return f"{label}_{spras_revision()}"
 
 # This will get called in the Snakefile, instantiating the singleton with the raw config
@@ -117,6 +127,8 @@ def __init__(self, raw_config: dict[str, Any]):
         self.analysis_include_ml_aggregate_algo = None
         # A Boolean specifying whether to run the evaluation per algorithm analysis
         self.analysis_include_evaluation_aggregate_algo = None
+        # Specifies whether the files should be OSDF-immutable (i.e. the file names change when the file itself changes)
+        self.osdf_immutable = parsed_raw_config.osdf_immutable
 
         self.process_config(parsed_raw_config)
 
@@ -148,9 +160,9 @@ def process_datasets(self, raw_config: RawConfig):
         # Convert to dicts to simplify the yaml logging
 
         for dataset in raw_config.datasets:
-            dataset.label = attach_spras_revision(dataset.label)
+            dataset.label = attach_spras_revision(self.osdf_immutable, dataset.label)
         for gold_standard in raw_config.gold_standards:
-            gold_standard.label = attach_spras_revision(gold_standard.label)
+            gold_standard.label = attach_spras_revision(self.osdf_immutable, gold_standard.label)
 
         for dataset in raw_config.datasets:
             label = dataset.label
@@ -165,11 +177,14 @@ def process_datasets(self, raw_config: RawConfig):
         dataset_labels = set(self.datasets.keys())
         gold_standard_dataset_labels = {dataset_label for value in self.gold_standards.values() for dataset_label in value['dataset_labels']}
         for label in gold_standard_dataset_labels:
-            if attach_spras_revision(label) not in dataset_labels:
+            if attach_spras_revision(self.osdf_immutable, label) not in dataset_labels:
                 raise ValueError(f"Dataset label '{label}' provided in gold standards does not exist in the existing dataset labels.")
         # We attach the SPRAS revision to the individual dataset labels afterwards for a cleaner error message above.
         for key, gold_standard in self.gold_standards.items():
-            self.gold_standards[key]["dataset_labels"] = map(attach_spras_revision, gold_standard["dataset_labels"])
+            self.gold_standards[key]["dataset_labels"] = map(
+                functools.partial(attach_spras_revision, osdf_immutable=self.osdf_immutable),
+                gold_standard["dataset_labels"]
+            )
 
         # Code snipped from Snakefile that may be useful for assigning default labels
         # dataset_labels = [dataset.get('label', f'dataset{index}') for index, dataset in enumerate(datasets)]

spras/config/schema.py

@@ -101,6 +101,13 @@ class ReconstructionSettings(BaseModel):
 
 class RawConfig(BaseModel):
     containers: ContainerSettings
+    osdf_immutable: bool = False
+    """
+    If enabled, this tags all files with their local file version.
+    Most files do not have a specific version, and by default, this will be the hash of
+    all the SPRAS files in the PyPA installation. This option will not work if SPRAS was not installed
+    in a PyPA-compliant manner (PyPA-compliant installations include but are not limited to pip, poetry, uv, conda, pixi.)
+    """
 
     hash_length: int = DEFAULT_HASH_LENGTH
     "The length of the hash used to identify a parameter combination"