Set xet cache env var (#3194)

* set xet cache env var

* style

Author: lhoestq

services/worker/src/worker/job_runners/_job_runner_with_datasets_cache.py

@@ -2,6 +2,7 @@
 # Copyright 2022 The HuggingFace Authors.
 
 import logging
+import os
 from pathlib import Path
 from typing import Optional
 
@@ -39,9 +40,12 @@ def set_datasets_cache(self, cache_subdirectory: Optional[Path]) -> None:
         datasets.config.EXTRACTED_DATASETS_PATH = (
             datasets.config.HF_DATASETS_CACHE / datasets.config.EXTRACTED_DATASETS_DIR
         )
+
         huggingface_hub.constants.HF_HUB_CACHE = str(cache_subdirectory / "hub")
+        os.environ["HF_HUB_CACHE"] = str(cache_subdirectory / "hub")
         logging.debug(f"huggingface_hub cache set to: {huggingface_hub.constants.HF_HUB_CACHE}")
         huggingface_hub.constants.HF_XET_CACHE = str(cache_subdirectory / "xet")
+        os.environ["HF_XET_CACHE"] = str(cache_subdirectory / "xet")
         logging.debug(f"huggingface_hub xet cache set to: {huggingface_hub.constants.HF_XET_CACHE}")
 
     def pre_compute(self) -> None: