From 92a3f7a3b4e5c7b7f6d11dddd8065332aee83281 Mon Sep 17 00:00:00 2001 From: BichengWang Date: Thu, 15 Oct 2026 10:23:23 -0700 Subject: [PATCH 1/2] fix(link-checker): preserve changed filenames with special characters --- tools/check_all_links.py | 4 +-- tools/test_check_all_links.py | 52 +++++++++++++++++++++++++++++++---- 2 files changed, 49 insertions(+), 7 deletions(-) diff --git a/tools/check_all_links.py b/tools/check_all_links.py index eea9d2027..203f655ab 100755 --- a/tools/check_all_links.py +++ b/tools/check_all_links.py @@ -305,9 +305,9 @@ def get_changed_files_from_git(repo_root: pathlib.Path) -> Optional[List[str]]: base_refs = ["upstream/main", "origin/main", "main", "HEAD~1"] for base in base_refs: try: - cmd = ["git", "diff", "--name-only", f"{base}...HEAD"] + cmd = ["git", "diff", "--name-only", "-z", f"{base}...HEAD"] out = subprocess.check_output(cmd, cwd=repo_root, text=True, stderr=subprocess.DEVNULL) - lines = [line.strip() for line in out.splitlines() if line.strip().endswith((".md", ".ipynb"))] + lines = [path for path in out.split("\0") if path.endswith((".md", ".ipynb"))] # A successful empty diff still establishes the comparison base. logger.info("Detected %d changed file(s) against %s", len(lines), base) return lines diff --git a/tools/test_check_all_links.py b/tools/test_check_all_links.py index 7cbf7f325..0eec13a7a 100644 --- a/tools/test_check_all_links.py +++ b/tools/test_check_all_links.py @@ -351,11 +351,11 @@ def test_github_blob_url_with_query_params_and_casing(self) -> None: def test_changed_files_stops_at_available_base_without_documents(self) -> None: """An empty document diff must not select changes from an older base.""" - for diff in ("", "tools/check_all_links.py\n"): + for diff in ("", "tools/check_all_links.py\0"): with self.subTest(diff=diff): with mock.patch( "tools.check_all_links.subprocess.check_output", - side_effect=[diff, "quickstarts/guide.md\n"], + side_effect=[diff, "quickstarts/guide.md\0"], ): self.assertEqual(get_changed_files_from_git(self.repo_root), []) @@ -365,7 +365,7 @@ def test_changed_files_falls_back_when_base_is_unavailable(self) -> None: "tools.check_all_links.subprocess.check_output", side_effect=[ subprocess.CalledProcessError(128, "git diff"), - "README.md\ntools/check_all_links.py\nquickstarts/sample.ipynb\n", + "README.md\0tools/check_all_links.py\0quickstarts/sample.ipynb\0", ], ): self.assertEqual( @@ -375,7 +375,7 @@ def test_changed_files_falls_back_when_base_is_unavailable(self) -> None: def test_changed_cli_skips_unrelated_broken_links(self) -> None: """No document changes against origin must not audit an older broken guide.""" - for diff in ("", "tools/check_all_links.py\n"): + for diff in ("", "tools/check_all_links.py\0"): with self.subTest(diff=diff): output = io.StringIO() with ( @@ -384,7 +384,7 @@ def test_changed_cli_skips_unrelated_broken_links(self) -> None: side_effect=[ subprocess.CalledProcessError(128, "git diff"), diff, - "quickstarts/guide.md\n", + "quickstarts/guide.md\0", ], ), mock.patch.object( @@ -399,6 +399,48 @@ def test_changed_cli_skips_unrelated_broken_links(self) -> None: self.assertEqual(exit_context.exception.code, 0, output.getvalue()) self.assertIn("No modified .md or .ipynb files detected", output.getvalue()) + def test_changed_files_preserves_special_characters(self) -> None: + """Git paths must not be quoted, split on newlines, or stripped.""" + paths = [ + "quickstarts/指南.ipynb", + "quickstarts/line\nbreak.md", + "quickstarts/tab\tname.md", + 'quickstarts/quoted"name.md', + " leading-space.md", + ] + with mock.patch( + "tools.check_all_links.subprocess.check_output", + return_value="\0".join(paths + ["tools/check_all_links.py", ""]), + ) as git_diff: + self.assertEqual(get_changed_files_from_git(self.repo_root), paths) + self.assertEqual( + git_diff.call_args.args[0], + ["git", "diff", "--name-only", "-z", "upstream/main...HEAD"], + ) + + def test_changed_cli_audits_unicode_filename(self) -> None: + """A changed Unicode filename must not hide a broken link from the CLI.""" + target = self.quickstarts_dir / "指南.md" + target.write_text("[Broken](missing.md)\n", encoding="utf-8") + output = io.StringIO() + with ( + mock.patch( + "tools.check_all_links.subprocess.check_output", + return_value="quickstarts/指南.md\0", + ), + mock.patch.object( + sys, + "argv", + ["check_all_links.py", "--repo-dir", str(self.repo_root), "--changed"], + ), + contextlib.redirect_stdout(output), + self.assertRaises(SystemExit) as exit_context, + ): + main() + self.assertEqual(exit_context.exception.code, 1, output.getvalue()) + self.assertIn("quickstarts/指南.md", output.getvalue()) + self.assertIn("File does not exist: missing.md", output.getvalue()) + def test_changed_files_returns_none_when_no_base_is_available(self) -> None: """Unavailable comparison bases must be distinct from an empty diff.""" with ( From 7bf7fa86b13597fabbb5fe1dabc9fa6478c6f100 Mon Sep 17 00:00:00 2001 From: BichengWang Date: Fri, 16 Oct 2026 10:04:56 -0700 Subject: [PATCH 2/2] fix(link-checker): decode changed paths as UTF-8 --- tools/check_all_links.py | 4 +++- tools/test_check_all_links.py | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/tools/check_all_links.py b/tools/check_all_links.py index 203f655ab..2a339542b 100755 --- a/tools/check_all_links.py +++ b/tools/check_all_links.py @@ -306,7 +306,9 @@ def get_changed_files_from_git(repo_root: pathlib.Path) -> Optional[List[str]]: for base in base_refs: try: cmd = ["git", "diff", "--name-only", "-z", f"{base}...HEAD"] - out = subprocess.check_output(cmd, cwd=repo_root, text=True, stderr=subprocess.DEVNULL) + out = subprocess.check_output( + cmd, cwd=repo_root, encoding="utf-8", stderr=subprocess.DEVNULL + ) lines = [path for path in out.split("\0") if path.endswith((".md", ".ipynb"))] # A successful empty diff still establishes the comparison base. logger.info("Detected %d changed file(s) against %s", len(lines), base) diff --git a/tools/test_check_all_links.py b/tools/test_check_all_links.py index 0eec13a7a..8a0ae12b7 100644 --- a/tools/test_check_all_links.py +++ b/tools/test_check_all_links.py @@ -18,6 +18,7 @@ import contextlib import io import json +import locale import pathlib import subprocess import sys @@ -441,6 +442,26 @@ def test_changed_cli_audits_unicode_filename(self) -> None: self.assertIn("quickstarts/指南.md", output.getvalue()) self.assertIn("File does not exist: missing.md", output.getvalue()) + def test_changed_files_decodes_utf8_with_non_utf8_locale(self) -> None: + """Unicode paths must survive subprocess decoding under an ASCII locale.""" + path = "quickstarts/指南.md" + raw_output = (path + "\0").encode("utf-8") + check_output = subprocess.check_output + original_locale = locale.setlocale(locale.LC_CTYPE) + self.addCleanup(locale.setlocale, locale.LC_CTYPE, original_locale) + locale.setlocale(locale.LC_CTYPE, "C") + + def git_diff_output(cmd, **kwargs): + return check_output( + [sys.executable, "-c", f"import sys; sys.stdout.buffer.write({raw_output!r})"], + **kwargs, + ) + + with mock.patch( + "tools.check_all_links.subprocess.check_output", side_effect=git_diff_output + ): + self.assertEqual(get_changed_files_from_git(self.repo_root), [path]) + def test_changed_files_returns_none_when_no_base_is_available(self) -> None: """Unavailable comparison bases must be distinct from an empty diff.""" with (