Set xet cache dir (#3192)

set xet cache dir

Author: lhoestq

services/worker/src/worker/job_runners/_job_runner_with_datasets_cache.py

@@ -41,6 +41,8 @@ def set_datasets_cache(self, cache_subdirectory: Optional[Path]) -> None:
         )
         huggingface_hub.constants.HF_HUB_CACHE = str(cache_subdirectory / "hub")
         logging.debug(f"huggingface_hub cache set to: {huggingface_hub.constants.HF_HUB_CACHE}")
+        huggingface_hub.constants.HF_XET_CACHE = str(cache_subdirectory / "xet")
+        logging.debug(f"huggingface_hub xet cache set to: {huggingface_hub.constants.HF_XET_CACHE}")
 
     def pre_compute(self) -> None:
         super().pre_compute()

services/worker/tests/job_runners/test__job_runner_with_datasets_cache.py

@@ -77,6 +77,7 @@ def test_set_datasets_cache(app_config: AppConfig, get_job_runner: GetJobRunner)
     job_runner.set_datasets_cache(dummy_path)
     assert datasets.config.HF_DATASETS_CACHE.is_relative_to(dummy_path)
     assert Path(huggingface_hub.constants.HF_HUB_CACHE).is_relative_to(dummy_path)
+    assert Path(huggingface_hub.constants.HF_XET_CACHE).is_relative_to(dummy_path)
 
 
 def test_pre_compute_post_compute(app_config: AppConfig, get_job_runner: GetJobRunner) -> None:
@@ -100,9 +101,11 @@ def assert_datasets_cache_path(path: Optional[Path], exists: bool) -> None:
     if exists:
         datasets_cache_path = path / "datasets"
         hub_cache_path = path / "hub"
+        xet_cache_path = path / "xet"
         assert datasets.config.HF_DATASETS_CACHE == datasets_cache_path
         assert (
             datasets.config.DOWNLOADED_DATASETS_PATH == datasets_cache_path / datasets.config.DOWNLOADED_DATASETS_DIR
         )
         assert datasets.config.EXTRACTED_DATASETS_PATH == datasets_cache_path / datasets.config.EXTRACTED_DATASETS_DIR
         assert huggingface_hub.constants.HF_HUB_CACHE == str(hub_cache_path)
+        assert huggingface_hub.constants.HF_XET_CACHE == str(xet_cache_path)