From 53ce29631adcf8b4452c7e473373becf1baeb9d8 Mon Sep 17 00:00:00 2001 From: Atsuo Ishimoto Date: Sat, 25 Feb 2023 00:00:09 +0900 Subject: [PATCH 1/3] fix Japanese PDF geneeration error --- build_docs.py | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/build_docs.py b/build_docs.py index aded2d2..11b7ca1 100755 --- a/build_docs.py +++ b/build_docs.py @@ -226,9 +226,28 @@ class Language: ) PLATEX_DEFAULT = ( - "-D latex_engine=platex", + "-D latex_engine=lualatex", "-D latex_elements.inputenc=", "-D latex_elements.fontenc=", + "-D latex_docclass.manual=ltjsbook", + "-D latex_docclass.howto=ltjsarticle", + "-D latex_elements.polyglossia=", + "-D latex_elements.fontpkg=", + "-D 'latex_elements.preamble=" + r"\setlength{\footskip}{16.4pt}" # warning: (fancyhdr)Make it at least 16.4pt + r"\usepackage[noto-otf]{luatexja-preset}" + r"\usepackage{newunicodechar,luacode}" + r"\newunicodechar{^^^^212a}{K}" + r"\newfontface{\fRepC}{DejaVu Sans Mono}" + r'\newunicodechar{^^^^f8fd}{{\fRepC\ltjalchar"FFFD}}' + r"\makeatletter" + r"\titleformat{\subsubsection}{\normalsize\py@HeaderFamily}" + r"{\py@TitleColor\thesubsubsection}{0.5em}{\py@TitleColor}" + r"\titleformat{\paragraph}{\normalsize\py@HeaderFamily}" + r"{\py@TitleColor\theparagraph}{0.5em}{\py@TitleColor}" + r"\titleformat{\subparagraph}{\normalsize\py@HeaderFamily}" + r"{\py@TitleColor\thesubparagraph}{0.5em}{\py@TitleColor}" + r"\makeatother'" ) XELATEX_WITH_FONTSPEC = ( @@ -257,7 +276,7 @@ class Language: Language("fr", "fr", "French", True, XELATEX_WITH_FONTSPEC), Language("id", "id", "Indonesian", False, XELATEX_DEFAULT), Language("it", "it", "Italian", False, XELATEX_DEFAULT), - Language("ja", "ja", "Japanese", True, PLATEX_DEFAULT, html_only=True), # See https://github.com/python/python-docs-ja/issues/35 + Language("ja", "ja", "Japanese", True, PLATEX_DEFAULT, html_only=False), # See https://github.com/python/python-docs-ja/issues/35 Language("ko", "ko", "Korean", True, XELATEX_FOR_KOREAN), Language("pl", "pl", "Polish", False, XELATEX_DEFAULT), Language("pt-br", "pt_BR", "Brazilian Portuguese", True, XELATEX_DEFAULT), @@ -726,6 +745,13 @@ def build(self): "-D gettext_compact=0", ) ) + if self.language.tag == "ja": + subprocess.check_output("sed -i s/\N{REPLACEMENT CHARACTER}/\uf8fd/g " + f"{locale_dirs}/ja/LC_MESSAGES/**/*.po", + shell=True) + subprocess.check_output("sed -i s/\N{REPLACEMENT CHARACTER}/\uf8fd/g " + f"{self.checkout}/Doc/**/*.rst", shell=True) + if self.version.status == "EOL": sphinxopts.append("-D html_context.outdated=1") maketarget = ( From 41b939825540e7849d812a14ce983b4b51a50258 Mon Sep 17 00:00:00 2001 From: Atsuo Ishimoto Date: Sat, 25 Feb 2023 15:55:18 +0900 Subject: [PATCH 2/3] various change for ja/lualatex --- build_docs.py | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/build_docs.py b/build_docs.py index 11b7ca1..5337010 100755 --- a/build_docs.py +++ b/build_docs.py @@ -225,21 +225,33 @@ class Language: "-D latex_elements.fontenc=", ) -PLATEX_DEFAULT = ( +LUALATEX_FOR_JP = ( "-D latex_engine=lualatex", "-D latex_elements.inputenc=", "-D latex_elements.fontenc=", "-D latex_docclass.manual=ltjsbook", "-D latex_docclass.howto=ltjsarticle", + + # supress polyglossia warnings "-D latex_elements.polyglossia=", "-D latex_elements.fontpkg=", + + # preamble "-D 'latex_elements.preamble=" - r"\setlength{\footskip}{16.4pt}" # warning: (fancyhdr)Make it at least 16.4pt r"\usepackage[noto-otf]{luatexja-preset}" r"\usepackage{newunicodechar,luacode}" r"\newunicodechar{^^^^212a}{K}" + + # Since luatex doesn't support \ufffd, replace \ufffd with \uf8fd and retore later. + # https://gist.github.com/zr-tex8r/e0931df922f38fbb67634f05dfdaf66b + # Luatex already fixed this issue, so we can remove this once Texlive is updated. + # (https://github.com/TeX-Live/luatex/commit/eaa95ce0a141eaf7a02) r"\newfontface{\fRepC}{DejaVu Sans Mono}" r'\newunicodechar{^^^^f8fd}{{\fRepC\ltjalchar"FFFD}}' + + # Workaround for the luatex-ja issue (Thanks to @jfbu) + # https://github.com/sphinx-doc/sphinx/issues/11179#issuecomment-1420715092 + # https://osdn.net/projects/luatex-ja/ticket/47321 r"\makeatletter" r"\titleformat{\subsubsection}{\normalsize\py@HeaderFamily}" r"{\py@TitleColor\thesubsubsection}{0.5em}{\py@TitleColor}" @@ -247,7 +259,11 @@ class Language: r"{\py@TitleColor\theparagraph}{0.5em}{\py@TitleColor}" r"\titleformat{\subparagraph}{\normalsize\py@HeaderFamily}" r"{\py@TitleColor\thesubparagraph}{0.5em}{\py@TitleColor}" - r"\makeatother'" + r"\makeatother" + + # subpress warning: (fancyhdr)Make it at least 16.4pt + r"\setlength{\footskip}{16.4pt}" + "'" ) XELATEX_WITH_FONTSPEC = ( @@ -276,7 +292,7 @@ class Language: Language("fr", "fr", "French", True, XELATEX_WITH_FONTSPEC), Language("id", "id", "Indonesian", False, XELATEX_DEFAULT), Language("it", "it", "Italian", False, XELATEX_DEFAULT), - Language("ja", "ja", "Japanese", True, PLATEX_DEFAULT, html_only=False), # See https://github.com/python/python-docs-ja/issues/35 + Language("ja", "ja", "Japanese", True, LUALATEX_FOR_JP), Language("ko", "ko", "Korean", True, XELATEX_FOR_KOREAN), Language("pl", "pl", "Polish", False, XELATEX_DEFAULT), Language("pt-br", "pt_BR", "Brazilian Portuguese", True, XELATEX_DEFAULT), @@ -746,6 +762,10 @@ def build(self): ) ) if self.language.tag == "ja": + # Since luatex doesn't support \ufffd, replace \ufffd with \uf8fd and retore later. + # https://gist.github.com/zr-tex8r/e0931df922f38fbb67634f05dfdaf66b + # Luatex already fixed this issue, so we can remove this once Texlive is updated. + # (https://github.com/TeX-Live/luatex/commit/eaa95ce0a141eaf7a02) subprocess.check_output("sed -i s/\N{REPLACEMENT CHARACTER}/\uf8fd/g " f"{locale_dirs}/ja/LC_MESSAGES/**/*.po", shell=True) From f56da505d88774eaf295524daa4fdf599b2782f7 Mon Sep 17 00:00:00 2001 From: Atsuo Ishimoto Date: Sun, 26 Feb 2023 22:47:14 +0900 Subject: [PATCH 3/3] Replace U+FFFD with '?' --- build_docs.py | 39 +++++++++++++++++---------------------- 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/build_docs.py b/build_docs.py index 5337010..c31826a 100755 --- a/build_docs.py +++ b/build_docs.py @@ -237,33 +237,28 @@ class Language: "-D latex_elements.fontpkg=", # preamble - "-D 'latex_elements.preamble=" - r"\usepackage[noto-otf]{luatexja-preset}" - r"\usepackage{newunicodechar,luacode}" - r"\newunicodechar{^^^^212a}{K}" + "-D latex_elements.preamble=" - # Since luatex doesn't support \ufffd, replace \ufffd with \uf8fd and retore later. + # Render non-Japanese letters with luatex # https://gist.github.com/zr-tex8r/e0931df922f38fbb67634f05dfdaf66b - # Luatex already fixed this issue, so we can remove this once Texlive is updated. - # (https://github.com/TeX-Live/luatex/commit/eaa95ce0a141eaf7a02) - r"\newfontface{\fRepC}{DejaVu Sans Mono}" - r'\newunicodechar{^^^^f8fd}{{\fRepC\ltjalchar"FFFD}}' + r"\\usepackage[noto-otf]{luatexja-preset}" + r"\\usepackage{newunicodechar}" + r"\\newunicodechar{^^^^212a}{K}" # Workaround for the luatex-ja issue (Thanks to @jfbu) # https://github.com/sphinx-doc/sphinx/issues/11179#issuecomment-1420715092 # https://osdn.net/projects/luatex-ja/ticket/47321 - r"\makeatletter" - r"\titleformat{\subsubsection}{\normalsize\py@HeaderFamily}" - r"{\py@TitleColor\thesubsubsection}{0.5em}{\py@TitleColor}" - r"\titleformat{\paragraph}{\normalsize\py@HeaderFamily}" - r"{\py@TitleColor\theparagraph}{0.5em}{\py@TitleColor}" - r"\titleformat{\subparagraph}{\normalsize\py@HeaderFamily}" - r"{\py@TitleColor\thesubparagraph}{0.5em}{\py@TitleColor}" - r"\makeatother" + r"\\makeatletter" + r"\\titleformat{\\subsubsection}{\\normalsize\\py@HeaderFamily}" + r"{\\py@TitleColor\\thesubsubsection}{0.5em}{\\py@TitleColor}" + r"\\titleformat{\\paragraph}{\\normalsize\\py@HeaderFamily}" + r"{\\py@TitleColor\\theparagraph}{0.5em}{\\py@TitleColor}" + r"\\titleformat{\\subparagraph}{\\normalsize\\py@HeaderFamily}" + r"{\\py@TitleColor\\thesubparagraph}{0.5em}{\\py@TitleColor}" + r"\\makeatother" # subpress warning: (fancyhdr)Make it at least 16.4pt - r"\setlength{\footskip}{16.4pt}" - "'" + r"\\setlength{\\footskip}{16.4pt}" ) XELATEX_WITH_FONTSPEC = ( @@ -762,14 +757,14 @@ def build(self): ) ) if self.language.tag == "ja": - # Since luatex doesn't support \ufffd, replace \ufffd with \uf8fd and retore later. + # Since luatex doesn't support \ufffd, replace \ufffd with '?'. # https://gist.github.com/zr-tex8r/e0931df922f38fbb67634f05dfdaf66b # Luatex already fixed this issue, so we can remove this once Texlive is updated. # (https://github.com/TeX-Live/luatex/commit/eaa95ce0a141eaf7a02) - subprocess.check_output("sed -i s/\N{REPLACEMENT CHARACTER}/\uf8fd/g " + subprocess.check_output("sed -i s/\N{REPLACEMENT CHARACTER}/?/g " f"{locale_dirs}/ja/LC_MESSAGES/**/*.po", shell=True) - subprocess.check_output("sed -i s/\N{REPLACEMENT CHARACTER}/\uf8fd/g " + subprocess.check_output("sed -i s/\N{REPLACEMENT CHARACTER}/?/g " f"{self.checkout}/Doc/**/*.rst", shell=True) if self.version.status == "EOL":