feat: Enable configurable data location (#291)

closes #269

Author: jarbesfeld

notebooks/evidence_matching/fusion_evidence_matching.ipynb

@@ -224,12 +224,17 @@
     "# Generate list of matches, report match score\n",
     "from fusor.fusion_matching import FusionMatcher\n",
     "from fusor.harvester import CIVICHarvester\n",
+    "from fusor.config import config\n",
+    "from fusor.models import save_fusions_cache\n",
     "\n",
     "# Generate categorical fusions list\n",
     "harvester = CIVICHarvester(fusor=fusor)\n",
     "harvester.fusions_list = variants\n",
     "civic_fusions = await harvester.load_records()\n",
     "\n",
+    "# Save cache for later\n",
+    "save_fusions_cache(civic_fusions, cache_dir=config.data_root, cache_name=\"civic_translated_fusions.pkl\")\n",
+    "\n",
     "# Initialize FusionMatcher and define sources to match against\n",
     "fm = FusionMatcher(assayed_fusions=assayed_fusion_star_fusion,\n",
     "                   categorical_fusions=civic_fusions)\n",
@@ -513,6 +518,44 @@
     "fusions_list = await harvester.load_records(path)"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "062b8820",
+   "metadata": {},
+   "source": [
+    "#### Load and standardize data from Molecular Oncology Almanac (MOA)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "92b5485c",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Downloading v.2025-06-12.zip...\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "519kB [00:00, 4.56MB/s]\n"
+     ]
+    }
+   ],
+   "source": [
+    "from fusor.harvester import MOAHarvester\n",
+    "harvester = MOAHarvester(fusor=fusor, cache_dir=config.data_root)\n",
+    "moa_fusions = harvester.load_records()\n",
+    "\n",
+    "# Save cache for later\n",
+    "save_fusions_cache(moa_fusions, cache_dir=config.data_root, cache_name=\"moa_translated_fusions.pkl\")"
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "6ed39fe1",
@@ -524,7 +567,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 13,
    "id": "79afb22e",
    "metadata": {},
    "outputs": [
@@ -569,10 +612,9 @@
     "from fusor.fusion_matching import FusionMatcher\n",
     "\n",
     "# Initialize FusionMatcher and define sources to match against. This time, we will use\n",
-    "# the cache_files field, using the cached pickle files provided in the src/fusor/data \n",
-    "# repo\n",
+    "# the cache_files field, using the cached pickle files that have been saved\n",
     "fm = FusionMatcher(assayed_fusions=fusions_list,\n",
-    "                   cache_dir=Path(\"../../src/fusor/data\"),\n",
+    "                   cache_dir=config.data_root,\n",
     "                   cache_files=[\"civic_translated_fusions.pkl\", \"moa_translated_fusions.pkl\"])\n",
     "\n",
     "# Generate list of matching fusions\n",
@@ -592,7 +634,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": 14,
    "id": "af5e10b1",
    "metadata": {},
    "outputs": [],

src/fusor/config.py

@@ -0,0 +1,31 @@
+"""Configure data storage variables for fusion objects"""
+
+import os
+from pathlib import Path
+from typing import NamedTuple
+
+from wags_tails.utils.storage import get_data_dir
+
+
+class _Config(NamedTuple):
+    """Define config data structure."""
+
+    data_root: Path
+
+
+def _get_configs() -> _Config:
+    """Fetch config values from environment.
+
+    Eventually this may be transformed into something using `pydantic-settings` but for
+    now it just assembles a NamedTuple.
+
+    :return: constructed config object
+    """
+    if env_var_data_dir := os.environ.get("FUSOR_DATA_DIR"):
+        data_root_location = Path(env_var_data_dir)
+    else:
+        data_root_location = get_data_dir() / "fusor"
+    return _Config(data_root=data_root_location)
+
+
+config = _get_configs()

src/fusor/data/civic_translated_fusions.pkl

@@ -3,6 +3,7 @@
 import pickle
 from pathlib import Path
 
+from fusor.config import config
 from fusor.models import (
     AssayedFusion,
     CategoricalFusion,
@@ -27,8 +28,8 @@ def __init__(
         """Initialize FusionMatcher class and comparator categorical fusion objects
 
         :param cache_dir: The directory containing the cached categorical fusions
-            files. If cached files do not exist in the directory, a cached file at
-            the provided location will be generated for each source.
+            files. If this parameter is not provided, it will be set by default
+            to be `FUSOR_DATA_DIR`.
         :param assayed_fusions: A list of AssayedFusion objects
         :param categorical_fusions: A list of CategoricalFusion objects
         :param cache_files: A list of cache file names in ``cache_dir`` containing
@@ -40,6 +41,9 @@ def __init__(
         if not categorical_fusions and (not cache_dir or not cache_files):
             msg = "Either a list of CategoricalFusion objects must be provided to `categorical_fusions` or a Path and list of file names must be provided to `cache_dir` and `cache_files`, respectively"
             raise ValueError(msg)
+        if not cache_dir:
+            cache_dir = config.data_root
+        cache_dir.mkdir(parents=True, exist_ok=True)
         self.cache_dir = cache_dir
         self.assayed_fusions = assayed_fusions
         self.cache_files = cache_files

src/fusor/data/moa_translated_fusions.pkl

@@ -12,6 +12,7 @@
 from cool_seq_tool.schemas import Assembly, CoordinateType
 from wags_tails import MoaData
 
+from fusor.config import config
 from fusor.fusion_caller_models import (
     CIVIC,
     JAFFA,
@@ -336,14 +337,14 @@ def __init__(
         :param fusor: A FUSOR object
         :param cache_dir: The path to the store the cached MOA assertions.
             This by defualt is set to None, and the MOA assertions are
-            stored in src/fusor/data
+            stored in the `FUSOR_DATA_DIR` directory.
         :paran force_refresh: A boolean indicating if the MOA assertions
             file should be regenerated. By default, this is set to ``False``.
         """
         self.translator = MOATranslator(fusor)
         if not cache_dir:
-            cache_dir = Path(__file__).resolve().parent / "data"
-            cache_dir.mkdir(parents=True, exist_ok=True)
+            cache_dir = config.data_root
+        cache_dir.mkdir(parents=True, exist_ok=True)
         moa_downloader = MoaData(data_dir=cache_dir)
         moa_file = moa_downloader.get_latest(force_refresh=force_refresh)[0]
         with moa_file.open("rb") as f:

src/fusor/fusion_matching.py

@@ -26,6 +26,8 @@
     model_validator,
 )
 
+from fusor.config import config
+
 _logger = logging.getLogger(__name__)
 
 LINKER_REGEX = r"\|([atcg]+)\|"
@@ -953,19 +955,19 @@ class CategoricalFusion(AbstractFusion):
 
 def save_fusions_cache(
     fusions_list: list[AssayedFusion | CategoricalFusion],
-    cache_dir: Path,
     cache_name: str,
+    cache_dir: Path | None = None,
 ) -> None:
     """Save a list of translated fusions as a cache
 
     :param fusions_list: A list of FUSOR-translated fusions
-    :param output_dir: The location to store the cached file. If this parameter is
-        not supplied, it will default to creating a `data` directory under
-        `src/fusor`
     :param cache_name: The name for the resultant cached file
+    :param cache_dir: The location to store the cached file. If this parameter is
+        not supplied, it will default to storing data in the `FUSOR_DATA_DIR`
+        directory
     """
-    if not Path.is_dir(cache_dir):
-        cache_dir = Path(__file__).resolve().parent / "data"
+    if not cache_dir:
+        cache_dir = config.data_root
     cache_dir.mkdir(parents=True, exist_ok=True)
     output_file = cache_dir / cache_name
     if output_file.exists():

src/fusor/harvester.py

@@ -6,12 +6,13 @@
 import pytest
 from cool_seq_tool.app import CoolSeqTool
 
+from fusor.config import config
 from fusor.fusion_matching import FusionMatcher
 from fusor.fusor import FUSOR
 
 FIXTURE_DATA_DIR = Path(__file__).parents[0].resolve() / "fixtures"
-CACHE_DATA_DIR = Path(__file__).resolve().parent.parent / "src" / "fusor" / "data"
-CACHE_DATA_DIR.mkdir(parents=True, exist_ok=True)  # Create cache data directory
+CACHE_DATA_DIR = config.data_root
+CACHE_DATA_DIR.mkdir(parents=True, exist_ok=True)
 
 
 def pytest_addoption(parser):

src/fusor/models.py

@@ -189,3 +189,115 @@ tests:
             - gene:
                 name: "ABL1"
         score: 2
+      - fields:
+          type: "CategoricalFusion"
+          structure:
+            - gene:
+                name: "BCR"
+            - gene:
+                name: "ABL1"
+        score: 2
+      - fields:
+          type: "CategoricalFusion"
+          structure:
+            - gene:
+                name: "BCR"
+            - gene:
+                name: "ABL1"
+        score: 2
+      - fields:
+          type: "CategoricalFusion"
+          structure:
+            - gene:
+                name: "BCR"
+            - gene:
+                name: "ABL1"
+        score: 2
+      - fields:
+          type: "CategoricalFusion"
+          structure:
+            - gene:
+                name: "BCR"
+            - gene:
+                name: "ABL1"
+        score: 2
+      - fields:
+          type: "CategoricalFusion"
+          structure:
+            - gene:
+                name: "BCR"
+            - gene:
+                name: "ABL1"
+        score: 2
+      - fields:
+          type: "CategoricalFusion"
+          structure:
+            - gene:
+                name: "BCR"
+            - gene:
+                name: "ABL1"
+        score: 2
+      - fields:
+          type: "CategoricalFusion"
+          structure:
+            - gene:
+                name: "BCR"
+            - gene:
+                name: "ABL1"
+        score: 2
+      - fields:
+          type: "CategoricalFusion"
+          structure:
+            - gene:
+                name: "BCR"
+            - gene:
+                name: "ABL1"
+        score: 2
+      - fields:
+          type: "CategoricalFusion"
+          structure:
+            - gene:
+                name: "BCR"
+            - gene:
+                name: "ABL1"
+        score: 2
+      - fields:
+          type: "CategoricalFusion"
+          structure:
+            - gene:
+                name: "BCR"
+            - gene:
+                name: "ABL1"
+        score: 2
+      - fields:
+          type: "CategoricalFusion"
+          structure:
+            - gene:
+                name: "BCR"
+            - gene:
+                name: "ABL1"
+        score: 2
+      - fields:
+          type: "CategoricalFusion"
+          structure:
+            - gene:
+                name: "BCR"
+            - gene:
+                name: "ABL1"
+        score: 2
+      - fields:
+          type: "CategoricalFusion"
+          structure:
+            - gene:
+                name: "BCR"
+            - gene:
+                name: "ABL1"
+        score: 2
+      - fields:
+          type: "CategoricalFusion"
+          structure:
+            - gene:
+                name: "BCR"
+            - gene:
+                name: "ABL1"
+        score: 2

tests/conftest.py

@@ -6,6 +6,7 @@
 from cool_seq_tool.schemas import Strand
 from pydantic import ValidationError
 
+from fusor.config import config
 from fusor.models import (
     AbstractFusion,
     AnchoredReads,
@@ -1067,7 +1068,7 @@ def test_model_examples():
             model(**schema["example"])
 
 
-def test_save_cache(fixture_data_dir):
+def test_save_cache():
     """Test cache saving functionality for AssayedFusion and CategoricalFusion
     objects
     """
@@ -1081,15 +1082,13 @@ def test_save_cache(fixture_data_dir):
     # Test AssayedFusion
     save_fusions_cache(
         fusions_list=[assayed_fusion],
-        cache_dir=Path(fixture_data_dir),
         cache_name="assayed_cache_test.pkl",
     )
-    assert Path.exists(fixture_data_dir / "assayed_cache_test.pkl")
+    assert Path.exists(config.data_root / "assayed_cache_test.pkl")
 
     # Test CategoricalFusion
     save_fusions_cache(
         fusions_list=[categorical_fusion],
-        cache_dir=Path(fixture_data_dir),
         cache_name="categorical_cache_test.pkl",
     )
-    assert Path.exists(fixture_data_dir / "categorical_cache_test.pkl")
+    assert Path.exists(config.data_root / "categorical_cache_test.pkl")