Fix dotenv parser truncating unquoted values containing '#' (#4842)

* Fix dotenv parser truncating unquoted values containing '#'

* fix empty values swallowing inline comments and quoted values absoprtion bug

* bare keys with comments dropped bug fixed

* bare key alternative fix

Author: BenYang12

src/huggingface_hub/utils/_dotenv.py

@@ -29,15 +29,26 @@ def load_dotenv(dotenv_str: str, environ: dict[str, str] | None = None) -> dict[
         ^\s*
         (?:export[^\S\n]+)?               # optional export
         ([A-Za-z_][A-Za-z0-9_]*)          # key
-        [^\S\n]*(=)?[^\S\n]*
-        (                                 # value group
+        (?:
+            [^\S\n]*
+            (=)                           # equal sign
             (?:
-                '(?:\\'|[^'])*'           # single-quoted value
-                | \"(?:\\\"|[^\"])*\"     # double-quoted value
-                | [^#\n\r]+?              # unquoted value
+                [^\S\n]*
+                (                         # quoted value
+                    '(?:\\'|[^'])*'       # single-quoted
+                    | \"(?:\\\"|[^\"])*\" # double-quoted
+                )
+                [^\S\n]*(?:\#[^\n\r]*)?   # inline comment (needs no preceding whitespace after a quote)
+                |
+                (?:[^\S\n]+(?!\#))?       # whitespace after '=' also separates an inline comment
+                (                         # unquoted value (may contain '#')
+                    [^\n\r]*?
+                )
+                (?:[^\S\n]+\#[^\n\r]*)?   # inline comment (must be preceded by whitespace)
             )
-        )?
-        [^\S\n]*(?:\#.*)?$                # optional inline comment
+            |
+            [^\S\n]*(?:\#[^\n\r]*)?       # bare key (no '='), with an optional inline comment
+        )$
     """,
         re.VERBOSE,
     )
@@ -52,7 +63,7 @@ def load_dotenv(dotenv_str: str, environ: dict[str, str] | None = None) -> dict[
             key = match.group(1)
             val = None
             if match.group(2):  # if there is '='
-                raw_val = match.group(3) or ""
+                raw_val = match.group(3) or match.group(4) or ""
                 val = raw_val.strip()
                 # Remove surrounding quotes if quoted
                 if val.startswith('"') and val.endswith('"'):

tests/test_utils_dotenv.py

@@ -107,3 +107,78 @@ def test_single_quoted_values_are_literal():
         "ESCAPED_QUOTE": r"a\"b",
     }
     assert load_dotenv(r'DQ="line1\nline2"') == {"DQ": "line1\nline2"}
+
+
+def test_hash_in_unquoted_value_is_kept():
+    # A "#" only starts an inline comment when preceded by whitespace. A "#" that is part of an
+    # unquoted value (e.g. in a password, token or URL fragment) must be preserved, not truncated.
+    data = """
+    PASSWORD=p@ss#word
+    TOKEN=abc#123
+    URL=http://example.com/x#frag
+    LEADING=#notacomment
+    COMMENTED=value  # actual comment
+    """
+    assert load_dotenv(data) == {
+        "PASSWORD": "p@ss#word",
+        "TOKEN": "abc#123",
+        "URL": "http://example.com/x#frag",
+        "LEADING": "#notacomment",
+        "COMMENTED": "value",
+    }
+
+
+def test_empty_value_with_inline_comment():
+    # Whitespace right after "=" also separates an inline comment: the value is empty, not the
+    # comment text. Otherwise a comment would leak into env vars/secrets (e.g. `hf jobs --env-file`).
+    data = """
+    EMPTY= # comment
+    EMPTY_MULTI_SPACE=   # comment
+    EMPTY_NO_COMMENT=
+    LEADING=#notacomment
+    """
+    assert load_dotenv(data) == {
+        "EMPTY": "",
+        "EMPTY_MULTI_SPACE": "",
+        "EMPTY_NO_COMMENT": "",
+        "LEADING": "#notacomment",
+    }
+
+
+def test_comment_attached_to_closing_quote():
+    # After a closing quote, a "#" starts a comment even without preceding whitespace.
+    data = """
+    DQ="value"# comment
+    SQ='value'#comment
+    SPACED="value" # comment
+    HASH_INSIDE="a#b"
+    """
+    assert load_dotenv(data) == {
+        "DQ": "value",
+        "SQ": "value",
+        "SPACED": "value",
+        "HASH_INSIDE": "a#b",
+    }
+
+
+def test_bare_key_with_inline_comment():
+    # A bare key (no "=") is resolved from the environment. A trailing comment must not prevent the
+    # line from matching, otherwise the key is silently dropped by `--env-file` / `--secrets-file`.
+    data = """
+    BARE # comment
+    BARE_NO_SPACE#comment
+    BARE_PLAIN
+    """
+    environ = {"BARE": "1", "BARE_NO_SPACE": "2", "BARE_PLAIN": "3"}
+    assert load_dotenv(data, environ=environ) == {"BARE": "1", "BARE_NO_SPACE": "2", "BARE_PLAIN": "3"}
+
+
+def test_invalid_line_does_not_import_from_environ():
+    # A key followed by arbitrary text is not a valid line: it must be ignored rather than treated
+    # as a bare key, which would pull the host value in and clobber an explicit assignment above.
+    data = """
+    SECRET=explicit_value
+    SECRET is documented above
+    OTHER not an assignment
+    """
+    assert load_dotenv(data, environ={"SECRET": "HOST_ENV", "OTHER": "HOST_ENV"}) == {"SECRET": "explicit_value"}