Merge branch 'main' of https://github.com/DeepLearnPhysics/spine

Author: francois-drielsma

.gitignore

@@ -2,6 +2,7 @@
 log*
 !logger.py
 weights*
+.cache/
 *.txt
 *.csv
 *.root

src/spine/config/README.md

@@ -109,14 +109,16 @@ model:
 ```
 
 The `!download` tag automatically downloads and caches remote files:
-- **Caching**: Files downloaded to `weights/` directory (or `$SPINE_CACHE_DIR`)
+- **Caching**: Files cached in `.cache/weights/` under `$SPINE_PROD_BASEDIR` or `$SPINE_BASEDIR`
+- **Centralized**: Single cache location regardless of execution directory
 - **Smart reuse**: Existing files with matching hash aren't re-downloaded
 - **Hash validation**: Optional SHA256 verification ensures file integrity
 - **Production ready**: Ideal for model weights and large reference files
+- **Override**: Set `$SPINE_CACHE_DIR` to use a custom cache location
 
 Benefits for production:
 - Fully reproducible: exact weight URLs in version control
-- No manual downloads: users run configs directly  
+- No manual downloads: users run configs directly
 - Cache efficient: download once, reuse everywhere
 - Integrity verified: hash validation prevents corruption
 

src/spine/config/download.py

@@ -17,7 +17,10 @@
 def get_cache_dir() -> Path:
     """Get the directory for caching downloaded files.
 
-    By default, creates a 'weights/' directory in the current working directory.
+    By default, creates a '.cache/weights/' directory in SPINE_BASEDIR or
+    SPINE_PROD_BASEDIR (if running from spine-prod). This ensures downloads
+    are cached centrally regardless of execution directory.
+
     Can be overridden with the SPINE_CACHE_DIR environment variable.
 
     Returns
@@ -28,6 +31,23 @@ def get_cache_dir() -> Path:
     cache_dir = os.environ.get("SPINE_CACHE_DIR")
     if cache_dir:
         return Path(cache_dir)
+
+    # Try SPINE_PROD_BASEDIR first (if running from spine-prod)
+    spine_prod_base = os.environ.get("SPINE_PROD_BASEDIR")
+    if spine_prod_base:
+        return Path(spine_prod_base) / ".cache" / "weights"
+
+    # Fall back to SPINE_BASEDIR
+    spine_base = os.environ.get("SPINE_BASEDIR")
+    if spine_base:
+        return Path(spine_base) / ".cache" / "weights"
+
+    # Last resort: use current directory (with warning)
+    print(
+        "WARNING: SPINE_BASEDIR and SPINE_PROD_BASEDIR not set. "
+        "Using current directory for cache. "
+        "Please source configure.sh for proper caching."
+    )
     return Path.cwd() / "weights"
 
 

test/test_download.py

@@ -22,17 +22,47 @@
 class TestDownloadUtilities:
     """Test suite for download utility functions."""
 
-    def test_get_cache_dir_default(self):
-        """Test default cache directory."""
-        with patch.dict(os.environ, {}, clear=True):
+    def test_get_cache_dir_default(self, monkeypatch):
+        """Test default cache directory with warning when env vars not set."""
+        # Clear environment
+        monkeypatch.delenv("SPINE_CACHE_DIR", raising=False)
+        monkeypatch.delenv("SPINE_PROD_BASEDIR", raising=False)
+        monkeypatch.delenv("SPINE_BASEDIR", raising=False)
+
+        with patch("builtins.print") as mock_print:
             cache_dir = get_cache_dir()
+            # Should warn about missing env vars
+            mock_print.assert_called_once()
+            assert "WARNING" in str(mock_print.call_args)
             assert cache_dir == Path.cwd() / "weights"
 
-    def test_get_cache_dir_env_override(self):
-        """Test cache directory from environment variable."""
-        with patch.dict(os.environ, {"SPINE_CACHE_DIR": "/custom/cache"}):
-            cache_dir = get_cache_dir()
-            assert cache_dir == Path("/custom/cache")
+    def test_get_cache_dir_spine_prod(self, monkeypatch):
+        """Test cache directory from SPINE_PROD_BASEDIR."""
+        monkeypatch.delenv("SPINE_CACHE_DIR", raising=False)
+        monkeypatch.setenv("SPINE_PROD_BASEDIR", "/opt/spine-prod")
+        monkeypatch.setenv("SPINE_BASEDIR", "/opt/spine")
+
+        cache_dir = get_cache_dir()
+        # Should prefer SPINE_PROD_BASEDIR
+        assert cache_dir == Path("/opt/spine-prod") / ".cache" / "weights"
+
+    def test_get_cache_dir_spine_basedir(self, monkeypatch):
+        """Test cache directory from SPINE_BASEDIR."""
+        monkeypatch.delenv("SPINE_CACHE_DIR", raising=False)
+        monkeypatch.delenv("SPINE_PROD_BASEDIR", raising=False)
+        monkeypatch.setenv("SPINE_BASEDIR", "/opt/spine")
+
+        cache_dir = get_cache_dir()
+        assert cache_dir == Path("/opt/spine") / ".cache" / "weights"
+
+    def test_get_cache_dir_env_override(self, monkeypatch):
+        """Test cache directory from environment variable override."""
+        monkeypatch.setenv("SPINE_CACHE_DIR", "/custom/cache")
+        monkeypatch.setenv("SPINE_PROD_BASEDIR", "/opt/spine-prod")
+
+        cache_dir = get_cache_dir()
+        # SPINE_CACHE_DIR should take precedence
+        assert cache_dir == Path("/custom/cache")
 
     def test_url_to_filename(self):
         """Test URL to filename conversion."""