[Jobs] Use current or stored token in a Job secrets (#3272)

* hf_auth in jobs secrets

* --secrets HF_TOKEN

* also in --env for consistency

* factorize

* mypy

* Apply suggestions from code review

Co-authored-by: Lucain <[email protected]>

* style

* update regex

* add test

* Update src/huggingface_hub/utils/_dotenv.py

---------

Co-authored-by: Lucain <[email protected]>

Author: lhoestq

docs/source/en/guides/cli.md

@@ -720,6 +720,14 @@ You can pass environment variables to your job using
 >>> hf jobs run --secrets-file .env.secrets python:3.12 python -c "import os; print(os.environ['MY_SECRET'])"
 ```
 
+<Tip>
+
+Use `--secrets HF_TOKEN` to pass your local Hugging Face token implicitly.
+With this syntax, the secret is retrieved from the environment variable.
+For `HF_TOKEN`, it may read the token file located in the Hugging Face home folder if the environment variable is unset.
+
+</Tip>
+
 ### Hardware
 
 Available `--flavor` options:

src/huggingface_hub/cli/jobs.py

@@ -40,7 +40,7 @@
 
 import requests
 
-from huggingface_hub import HfApi, SpaceHardware
+from huggingface_hub import HfApi, SpaceHardware, get_token
 from huggingface_hub.utils import logging
 from huggingface_hub.utils._dotenv import load_dotenv
 
@@ -75,8 +75,16 @@ class RunCommand(BaseHuggingfaceCLICommand):
     def register_subcommand(parser: _SubParsersAction) -> None:
         run_parser = parser.add_parser("run", help="Run a Job")
         run_parser.add_argument("image", type=str, help="The Docker image to use.")
-        run_parser.add_argument("-e", "--env", action="append", help="Set environment variables.")
-        run_parser.add_argument("-s", "--secrets", action="append", help="Set secret environment variables.")
+        run_parser.add_argument("-e", "--env", action="append", help="Set environment variables. E.g. --env ENV=value")
+        run_parser.add_argument(
+            "-s",
+            "--secrets",
+            action="append",
+            help=(
+                "Set secret environment variables. E.g. --secrets SECRET=value "
+                "or `--secrets HF_TOKEN` to pass your Hugging Face token."
+            ),
+        )
         run_parser.add_argument("--env-file", type=str, help="Read in a file of environment variables.")
         run_parser.add_argument("--secrets-file", type=str, help="Read in a file of secret environment variables.")
         run_parser.add_argument(
@@ -113,14 +121,15 @@ def __init__(self, args: Namespace) -> None:
         self.command: List[str] = args.command
         self.env: dict[str, Optional[str]] = {}
         if args.env_file:
-            self.env.update(load_dotenv(Path(args.env_file).read_text()))
+            self.env.update(load_dotenv(Path(args.env_file).read_text(), environ=os.environ.copy()))
         for env_value in args.env or []:
-            self.env.update(load_dotenv(env_value))
+            self.env.update(load_dotenv(env_value, environ=os.environ.copy()))
         self.secrets: dict[str, Optional[str]] = {}
+        extended_environ = _get_extended_environ()
         if args.secrets_file:
-            self.secrets.update(load_dotenv(Path(args.secrets_file).read_text()))
+            self.secrets.update(load_dotenv(Path(args.secrets_file).read_text(), environ=extended_environ))
         for secret in args.secrets or []:
-            self.secrets.update(load_dotenv(secret))
+            self.secrets.update(load_dotenv(secret, environ=extended_environ))
         self.flavor: Optional[SpaceHardware] = args.flavor
         self.timeout: Optional[str] = args.timeout
         self.detach: bool = args.detach
@@ -449,7 +458,15 @@ def register_subcommand(parser):
             help=f"Flavor for the hardware, as in HF Spaces. Defaults to `cpu-basic`. Possible values: {', '.join(SUGGESTED_FLAVORS)}.",
         )
         run_parser.add_argument("-e", "--env", action="append", help="Environment variables")
-        run_parser.add_argument("-s", "--secrets", action="append", help="Secret environment variables")
+        run_parser.add_argument(
+            "-s",
+            "--secrets",
+            action="append",
+            help=(
+                "Set secret environment variables. E.g. --secrets SECRET=value "
+                "or `--secrets HF_TOKEN` to pass your Hugging Face token."
+            ),
+        )
         run_parser.add_argument("--env-file", type=str, help="Read in a file of environment variables.")
         run_parser.add_argument(
             "--secrets-file",
@@ -480,14 +497,15 @@ def __init__(self, args: Namespace) -> None:
         self.image = args.image
         self.env: dict[str, Optional[str]] = {}
         if args.env_file:
-            self.env.update(load_dotenv(Path(args.env_file).read_text()))
+            self.env.update(load_dotenv(Path(args.env_file).read_text(), environ=os.environ.copy()))
         for env_value in args.env or []:
-            self.env.update(load_dotenv(env_value))
+            self.env.update(load_dotenv(env_value, environ=os.environ.copy()))
         self.secrets: dict[str, Optional[str]] = {}
+        extended_environ = _get_extended_environ()
         if args.secrets_file:
-            self.secrets.update(load_dotenv(Path(args.secrets_file).read_text()))
+            self.secrets.update(load_dotenv(Path(args.secrets_file).read_text(), environ=extended_environ))
         for secret in args.secrets or []:
-            self.secrets.update(load_dotenv(secret))
+            self.secrets.update(load_dotenv(secret, environ=extended_environ))
         self.flavor: Optional[SpaceHardware] = args.flavor
         self.timeout: Optional[str] = args.timeout
         self.detach: bool = args.detach
@@ -523,3 +541,10 @@ def run(self) -> None:
         # Now let's stream the logs
         for log in api.fetch_job_logs(job_id=job.id):
             print(log)
+
+
+def _get_extended_environ() -> Dict[str, str]:
+    extended_environ = os.environ.copy()
+    if (token := get_token()) is not None:
+        extended_environ["HF_TOKEN"] = token
+    return extended_environ

src/huggingface_hub/hf_api.py

@@ -132,7 +132,11 @@
     validate_hf_hub_args,
 )
 from .utils import tqdm as hf_tqdm
-from .utils._auth import _get_token_from_environment, _get_token_from_file, _get_token_from_google_colab
+from .utils._auth import (
+    _get_token_from_environment,
+    _get_token_from_file,
+    _get_token_from_google_colab,
+)
 from .utils._deprecation import _deprecate_method
 from .utils._runtime import is_xet_available
 from .utils._typing import CallableT

src/huggingface_hub/utils/_dotenv.py

@@ -1,9 +1,9 @@
 # AI-generated module (ChatGPT)
 import re
-from typing import Dict
+from typing import Dict, Optional
 
 
-def load_dotenv(dotenv_str: str) -> Dict[str, str]:
+def load_dotenv(dotenv_str: str, environ: Optional[Dict[str, str]] = None) -> Dict[str, str]:
     """
     Parse a DOTENV-format string and return a dictionary of key-value pairs.
     Handles quoted values, comments, export keyword, and blank lines.
@@ -12,17 +12,17 @@ def load_dotenv(dotenv_str: str) -> Dict[str, str]:
     line_pattern = re.compile(
         r"""
         ^\s*
-        (?:export\s+)?                    # optional export
+        (?:export[^\S\n]+)?               # optional export
         ([A-Za-z_][A-Za-z0-9_]*)          # key
-        \s*=\s*
+        [^\S\n]*(=)?[^\S\n]*
         (                                 # value group
             (?:
                 '(?:\\'|[^'])*'           # single-quoted value
-                | "(?:\\"|[^"])*"         # double-quoted value
+                | \"(?:\\\"|[^\"])*\"     # double-quoted value
                 | [^#\n\r]+?              # unquoted value
             )
         )?
-        \s*(?:\#.*)?$                     # optional inline comment
+        [^\S\n]*(?:\#.*)?$                # optional inline comment
     """,
         re.VERBOSE,
     )
@@ -33,19 +33,23 @@ def load_dotenv(dotenv_str: str) -> Dict[str, str]:
             continue  # Skip comments and empty lines
 
         match = line_pattern.match(line)
-        if not match:
-            continue  # Skip malformed lines
-
-        key, raw_val = match.group(1), match.group(2) or ""
-        val = raw_val.strip()
-
-        # Remove surrounding quotes if quoted
-        if (val.startswith('"') and val.endswith('"')) or (val.startswith("'") and val.endswith("'")):
-            val = val[1:-1]
-            val = val.replace(r"\n", "\n").replace(r"\t", "\t").replace(r"\"", '"').replace(r"\\", "\\")
-            if raw_val.startswith('"'):
-                val = val.replace(r"\$", "$")  # only in double quotes
-
-        env[key] = val
+        if match:
+            key = match.group(1)
+            val = None
+            if match.group(2):  # if there is '='
+                raw_val = match.group(3) or ""
+                val = raw_val.strip()
+                # Remove surrounding quotes if quoted
+                if (val.startswith('"') and val.endswith('"')) or (val.startswith("'") and val.endswith("'")):
+                    val = val[1:-1]
+                    val = val.replace(r"\n", "\n").replace(r"\t", "\t").replace(r"\"", '"').replace(r"\\", "\\")
+                    if raw_val.startswith('"'):
+                        val = val.replace(r"\$", "$")  # only in double quotes
+            elif environ is not None:
+                # Get it from the current environment
+                val = environ.get(key)
+
+            if val is not None:
+                env[key] = val
 
     return env

tests/test_utils_dotenv.py

@@ -62,3 +62,15 @@ def test_multiple_lines():
     D=4
     """
     assert load_dotenv(data) == {"A": "1", "B": "two", "C": "three", "D": "4"}
+
+
+def test_environ():
+    data = """
+    A=1
+    B
+    C=3
+    MISSING
+    EMPTY
+    """
+    environ = {"A": "one", "B": "two", "D": "four", "EMPTY": ""}
+    assert load_dotenv(data, environ=environ) == {"A": "1", "B": "two", "C": "3", "EMPTY": ""}