bpo-45949: Pure Python freeze module for cross builds (GH-29899)

Doc/using/configure.rst

@@ -518,9 +518,8 @@ Cross Compiling Options
 
 Cross compiling, also known as cross building, can be used to build Python
 for another CPU architecture or platform. Cross compiling requires a Python
-interpreter and the :program:`_freeze_module` binary from another build. The
-version of the build Python and :program:`_freeze_module` command must be
-the same as the cross compiled host Python.
+interpreter for the build platform. The version of the build Python must match
+the version of the cross compiled host Python.
 
 .. cmdoption:: --build=BUILD
 
@@ -530,13 +529,7 @@ the same as the cross compiled host Python.
 
    cross-compile to build programs to run on HOST (target platform)
 
-.. cmdoption:: --with-freeze-module=Programs/_freeze_module
-
-   path to ``_freeze_module`` binary for cross compiling.
-
-   .. versionadded:: 3.11
-
-.. cmdoption:: --with-build-python=python3.xx
+.. cmdoption:: --with-build-python=path/to/python
 
    path to build ``python`` binary for cross compiling
 
@@ -559,7 +552,6 @@ Cross compiling example::
    CONFIG_SITE=config.site-aarch64 ../configure \
        --build=x86_64-pc-linux-gnu \
        --host=aarch64-unknown-linux-gnu \
-       --with-freeze-module=../x86_64/Programs/_freeze_module \
        --with-build-python=../x86_64/python
 
 

Makefile.pre.in

@@ -283,14 +283,19 @@ BUILDPYTHON=	python$(BUILDEXE)
 PYTHON_FOR_REGEN?=@PYTHON_FOR_REGEN@
 UPDATE_FILE=$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/update_file.py
 PYTHON_FOR_BUILD=@PYTHON_FOR_BUILD@
-# Standard builds use _bootstrap_python for freezing, cross compiling
-# uses build Python, which must have the same version and bytecode,
-PYTHON_FOR_FREEZE?=@PYTHON_FOR_FREEZE@
+
+# Normal builds use Programs/_freeze_module.c for bootstrapping and
+# ./_bootstrap_python Programs/_freeze_module.py for remaining modules
+# Cross builds use an external "build Python" for all modules.
+PYTHON_FOR_FREEZE=@PYTHON_FOR_FREEZE@
+FREEZE_MODULE_BOOTSTRAP=@FREEZE_MODULE_BOOTSTRAP@
+FREEZE_MODULE_BOOTSTRAP_DEPS=@FREEZE_MODULE_BOOTSTRAP_DEPS@
+FREEZE_MODULE=@FREEZE_MODULE@
+FREEZE_MODULE_DEPS=@FREEZE_MODULE_DEPS@
+
 _PYTHON_HOST_PLATFORM=@_PYTHON_HOST_PLATFORM@
 BUILD_GNU_TYPE=	@build@
 HOST_GNU_TYPE=	@host@
-# Allow developers to override freeze_module command for cross building (bpo-45886)
-FREEZE_MODULE?=@FREEZE_MODULE@
 
 # Tcl and Tk config info from --with-tcltk-includes and -libs options
 TCLTK_INCLUDES=	@TCLTK_INCLUDES@
@@ -967,7 +972,7 @@ _bootstrap_python: $(LIBRARY_OBJS_OMIT_FROZEN) Programs/_bootstrap_python.o Modu
 .PHONY: regen-deepfreeze
 regen-deepfreeze: $(DEEPFREEZE_OBJS)
 
-DEEPFREEZE_DEPS=$(srcdir)/Tools/scripts/deepfreeze.py _bootstrap_python
+DEEPFREEZE_DEPS=$(srcdir)/Tools/scripts/deepfreeze.py $(FREEZE_MODULE_DEPS)
 
 # BEGIN: deepfreeze modules
 
@@ -1044,6 +1049,30 @@ Python/deepfreeze/frozen_only.c: Python/frozen_modules/frozen_only.h $(DEEPFREEZ
 
 ############################################################################
 # frozen modules (including importlib)
+#
+# Freezing is a multi step process. It works differently for standard builds
+# and cross builds. Standard builds use Programs/_freeze_module and
+# _bootstrap_python for freezing and deepfreezing, so users can build Python
+# without an existing Python installation. Cross builds cannot execute
+# compiled binaries and therefore rely on an external build Python
+# interpreter. The build interpreter must have same version and same bytecode
+# as the host (target) binary.
+#
+# Standard build process:
+# 1) compile minimal core objects for Py_Compile*() and PyMarshal_Write*().
+# 2) build Programs/_freeze_module binary.
+# 3) create frozen module headers for importlib and getpath.
+# 4) build _bootstrap_python binary.
+# 5) create remaining frozen module headers with
+#    ``./_bootstrap_python Programs/_freeze_module.py``. The pure Python
+#    script is used to test the cross compile code path.
+# 6) deepfreeze modules with _bootstrap_python
+#
+# Cross compile process:
+# 1) create all frozen module headers with external build Python and
+#    Programs/_freeze_module.py script.
+# 2) deepfreeze modules with external build Python.
+#
 
 # FROZEN_FILES_* are auto-generated by Tools/scripts/freeze_modules.py.
 FROZEN_FILES_IN = \
@@ -1104,83 +1133,83 @@ Modules/getpath_noop.o: $(srcdir)/Modules/getpath_noop.c Makefile
 Programs/_freeze_module: Programs/_freeze_module.o Modules/getpath_noop.o $(LIBRARY_OBJS_OMIT_FROZEN)
 	$(LINKCC) $(PY_CORE_LDFLAGS) -o $@ Programs/_freeze_module.o Modules/getpath_noop.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LIBS) $(MODLIBS) $(SYSLIBS)
 
+# We manually freeze getpath.py rather than through freeze_modules
+Python/frozen_modules/getpath.h: Modules/getpath.py $(FREEZE_MODULE_BOOTSTRAP_DEPS)
+	$(FREEZE_MODULE_BOOTSTRAP) getpath $(srcdir)/Modules/getpath.py Python/frozen_modules/getpath.h
+
 # BEGIN: freezing modules
 
-Python/frozen_modules/importlib._bootstrap.h: $(FREEZE_MODULE) Lib/importlib/_bootstrap.py
-	$(FREEZE_MODULE) importlib._bootstrap $(srcdir)/Lib/importlib/_bootstrap.py Python/frozen_modules/importlib._bootstrap.h
+Python/frozen_modules/importlib._bootstrap.h: Lib/importlib/_bootstrap.py $(FREEZE_MODULE_BOOTSTRAP_DEPS)
+	$(FREEZE_MODULE_BOOTSTRAP) importlib._bootstrap $(srcdir)/Lib/importlib/_bootstrap.py Python/frozen_modules/importlib._bootstrap.h
 
-Python/frozen_modules/importlib._bootstrap_external.h: $(FREEZE_MODULE) Lib/importlib/_bootstrap_external.py
-	$(FREEZE_MODULE) importlib._bootstrap_external $(srcdir)/Lib/importlib/_bootstrap_external.py Python/frozen_modules/importlib._bootstrap_external.h
+Python/frozen_modules/importlib._bootstrap_external.h: Lib/importlib/_bootstrap_external.py $(FREEZE_MODULE_BOOTSTRAP_DEPS)
+	$(FREEZE_MODULE_BOOTSTRAP) importlib._bootstrap_external $(srcdir)/Lib/importlib/_bootstrap_external.py Python/frozen_modules/importlib._bootstrap_external.h
 
-Python/frozen_modules/zipimport.h: $(FREEZE_MODULE) Lib/zipimport.py
-	$(FREEZE_MODULE) zipimport $(srcdir)/Lib/zipimport.py Python/frozen_modules/zipimport.h
+Python/frozen_modules/zipimport.h: Lib/zipimport.py $(FREEZE_MODULE_BOOTSTRAP_DEPS)
+	$(FREEZE_MODULE_BOOTSTRAP) zipimport $(srcdir)/Lib/zipimport.py Python/frozen_modules/zipimport.h
 
-Python/frozen_modules/abc.h: $(FREEZE_MODULE) Lib/abc.py
+Python/frozen_modules/abc.h: Lib/abc.py $(FREEZE_MODULE_DEPS)
 	$(FREEZE_MODULE) abc $(srcdir)/Lib/abc.py Python/frozen_modules/abc.h
 
-Python/frozen_modules/codecs.h: $(FREEZE_MODULE) Lib/codecs.py
+Python/frozen_modules/codecs.h: Lib/codecs.py $(FREEZE_MODULE_DEPS)
 	$(FREEZE_MODULE) codecs $(srcdir)/Lib/codecs.py Python/frozen_modules/codecs.h
 
-Python/frozen_modules/io.h: $(FREEZE_MODULE) Lib/io.py
+Python/frozen_modules/io.h: Lib/io.py $(FREEZE_MODULE_DEPS)
 	$(FREEZE_MODULE) io $(srcdir)/Lib/io.py Python/frozen_modules/io.h
 
-Python/frozen_modules/_collections_abc.h: $(FREEZE_MODULE) Lib/_collections_abc.py
+Python/frozen_modules/_collections_abc.h: Lib/_collections_abc.py $(FREEZE_MODULE_DEPS)
 	$(FREEZE_MODULE) _collections_abc $(srcdir)/Lib/_collections_abc.py Python/frozen_modules/_collections_abc.h
 
-Python/frozen_modules/_sitebuiltins.h: $(FREEZE_MODULE) Lib/_sitebuiltins.py
+Python/frozen_modules/_sitebuiltins.h: Lib/_sitebuiltins.py $(FREEZE_MODULE_DEPS)
 	$(FREEZE_MODULE) _sitebuiltins $(srcdir)/Lib/_sitebuiltins.py Python/frozen_modules/_sitebuiltins.h
 
-Python/frozen_modules/genericpath.h: $(FREEZE_MODULE) Lib/genericpath.py
+Python/frozen_modules/genericpath.h: Lib/genericpath.py $(FREEZE_MODULE_DEPS)
 	$(FREEZE_MODULE) genericpath $(srcdir)/Lib/genericpath.py Python/frozen_modules/genericpath.h
 
-Python/frozen_modules/ntpath.h: $(FREEZE_MODULE) Lib/ntpath.py
+Python/frozen_modules/ntpath.h: Lib/ntpath.py $(FREEZE_MODULE_DEPS)
 	$(FREEZE_MODULE) ntpath $(srcdir)/Lib/ntpath.py Python/frozen_modules/ntpath.h
 
-Python/frozen_modules/posixpath.h: $(FREEZE_MODULE) Lib/posixpath.py
+Python/frozen_modules/posixpath.h: Lib/posixpath.py $(FREEZE_MODULE_DEPS)
 	$(FREEZE_MODULE) posixpath $(srcdir)/Lib/posixpath.py Python/frozen_modules/posixpath.h
 
-Python/frozen_modules/os.h: $(FREEZE_MODULE) Lib/os.py
+Python/frozen_modules/os.h: Lib/os.py $(FREEZE_MODULE_DEPS)
 	$(FREEZE_MODULE) os $(srcdir)/Lib/os.py Python/frozen_modules/os.h
 
-Python/frozen_modules/site.h: $(FREEZE_MODULE) Lib/site.py
+Python/frozen_modules/site.h: Lib/site.py $(FREEZE_MODULE_DEPS)
 	$(FREEZE_MODULE) site $(srcdir)/Lib/site.py Python/frozen_modules/site.h
 
-Python/frozen_modules/stat.h: $(FREEZE_MODULE) Lib/stat.py
+Python/frozen_modules/stat.h: Lib/stat.py $(FREEZE_MODULE_DEPS)
 	$(FREEZE_MODULE) stat $(srcdir)/Lib/stat.py Python/frozen_modules/stat.h
 
-Python/frozen_modules/importlib.util.h: $(FREEZE_MODULE) Lib/importlib/util.py
+Python/frozen_modules/importlib.util.h: Lib/importlib/util.py $(FREEZE_MODULE_DEPS)
 	$(FREEZE_MODULE) importlib.util $(srcdir)/Lib/importlib/util.py Python/frozen_modules/importlib.util.h
 
-Python/frozen_modules/importlib.machinery.h: $(FREEZE_MODULE) Lib/importlib/machinery.py
+Python/frozen_modules/importlib.machinery.h: Lib/importlib/machinery.py $(FREEZE_MODULE_DEPS)
 	$(FREEZE_MODULE) importlib.machinery $(srcdir)/Lib/importlib/machinery.py Python/frozen_modules/importlib.machinery.h
 
-Python/frozen_modules/runpy.h: $(FREEZE_MODULE) Lib/runpy.py
+Python/frozen_modules/runpy.h: Lib/runpy.py $(FREEZE_MODULE_DEPS)
 	$(FREEZE_MODULE) runpy $(srcdir)/Lib/runpy.py Python/frozen_modules/runpy.h
 
-Python/frozen_modules/__hello__.h: $(FREEZE_MODULE) Lib/__hello__.py
+Python/frozen_modules/__hello__.h: Lib/__hello__.py $(FREEZE_MODULE_DEPS)
 	$(FREEZE_MODULE) __hello__ $(srcdir)/Lib/__hello__.py Python/frozen_modules/__hello__.h
 
-Python/frozen_modules/__phello__.h: $(FREEZE_MODULE) Lib/__phello__/__init__.py
+Python/frozen_modules/__phello__.h: Lib/__phello__/__init__.py $(FREEZE_MODULE_DEPS)
 	$(FREEZE_MODULE) __phello__ $(srcdir)/Lib/__phello__/__init__.py Python/frozen_modules/__phello__.h
 
-Python/frozen_modules/__phello__.ham.h: $(FREEZE_MODULE) Lib/__phello__/ham/__init__.py
+Python/frozen_modules/__phello__.ham.h: Lib/__phello__/ham/__init__.py $(FREEZE_MODULE_DEPS)
 	$(FREEZE_MODULE) __phello__.ham $(srcdir)/Lib/__phello__/ham/__init__.py Python/frozen_modules/__phello__.ham.h
 
-Python/frozen_modules/__phello__.ham.eggs.h: $(FREEZE_MODULE) Lib/__phello__/ham/eggs.py
+Python/frozen_modules/__phello__.ham.eggs.h: Lib/__phello__/ham/eggs.py $(FREEZE_MODULE_DEPS)
 	$(FREEZE_MODULE) __phello__.ham.eggs $(srcdir)/Lib/__phello__/ham/eggs.py Python/frozen_modules/__phello__.ham.eggs.h
 
-Python/frozen_modules/__phello__.spam.h: $(FREEZE_MODULE) Lib/__phello__/spam.py
+Python/frozen_modules/__phello__.spam.h: Lib/__phello__/spam.py $(FREEZE_MODULE_DEPS)
 	$(FREEZE_MODULE) __phello__.spam $(srcdir)/Lib/__phello__/spam.py Python/frozen_modules/__phello__.spam.h
 
-Python/frozen_modules/frozen_only.h: $(FREEZE_MODULE) Tools/freeze/flag.py
+Python/frozen_modules/frozen_only.h: Tools/freeze/flag.py $(FREEZE_MODULE_DEPS)
 	$(FREEZE_MODULE) frozen_only $(srcdir)/Tools/freeze/flag.py Python/frozen_modules/frozen_only.h
 
 # END: freezing modules
 
-# We manually freeze getpath.py rather than through freeze_modules
-Python/frozen_modules/getpath.h: $(FREEZE_MODULE) Modules/getpath.py
-	$(FREEZE_MODULE) getpath $(srcdir)/Modules/getpath.py Python/frozen_modules/getpath.h
-
 Tools/scripts/freeze_modules.py: $(FREEZE_MODULE)
 
 .PHONY: regen-frozen

Misc/NEWS.d/next/Build/2021-12-02-23-21-18.bpo-45949.OTSo9X.rst

@@ -0,0 +1,3 @@
+Use pure Python ``freeze_module`` for all but importlib bootstrap files.
+``--with-freeze-module`` :program:`configure` option is no longer needed for
+cross builds.

Programs/_freeze_module.c

@@ -5,6 +5,8 @@
    This is used directly by Tools/scripts/freeze_modules.py, and indirectly by "make regen-frozen".
 
    See Python/frozen.c for more info.
+
+   Keep this file in sync with Programs/_freeze_module.py.
 */
 
 #include <Python.h>

Programs/_freeze_module.py

@@ -0,0 +1,68 @@
+"""Python implementation of Programs/_freeze_module.c
+
+The pure Python implementation uses same functions and arguments as the C
+implementation.
+
+The generated byte code is slightly different because
+compile() sets the PyCF_SOURCE_IS_UTF8 flag and objects have a
+reference count > 1. Marshal adds the `FLAG_REF` flag and creates a
+reference `hashtable`.
+"""
+
+import marshal
+import sys
+
+header = "/* Auto-generated by Programs/_freeze_module.py */"
+
+
+def read_text(inpath: str) -> bytes:
+    with open(inpath, "rb") as f:
+        return f.read()
+
+
+def compile_and_marshal(name: str, text: bytes) -> bytes:
+    filename = f"<frozen {name}>"
+    # exec == Py_file_input
+    code = compile(text, filename, "exec", optimize=0, dont_inherit=True)
+    return marshal.dumps(code)
+
+
+def get_varname(name: str, prefix: str) -> str:
+    return f"{prefix}{name.replace('.', '_')}"
+
+
+def write_code(outfile, marshalled: bytes, varname: str) -> None:
+    data_size = len(marshalled)
+
+    outfile.write(f"const unsigned char {varname}[] = {{\n")
+
+    for n in range(0, data_size, 16):
+        outfile.write("    ")
+        outfile.write(",".join(str(i) for i in marshalled[n : n + 16]))
+        outfile.write(",\n")
+    outfile.write("};\n")
+
+
+def write_frozen(outpath: str, inpath: str, name: str, marshalled: bytes) -> None:
+    with open(outpath, "w") as outfile:
+        outfile.write(header)
+        outfile.write("\n")
+        arrayname = get_varname(name, "_Py_M__")
+        write_code(outfile, marshalled, arrayname)
+
+
+def main():
+    if len(sys.argv) != 4:
+        sys.exit("need to specify the name, input and output paths\n")
+
+    name = sys.argv[1]
+    inpath = sys.argv[2]
+    outpath = sys.argv[3]
+
+    text = read_text(inpath)
+    marshalled = compile_and_marshal(name, text)
+    write_frozen(outpath, inpath, name, marshalled)
+
+
+if __name__ == "__main__":
+    main()

Tools/scripts/deepfreeze.py

@@ -393,13 +393,14 @@ def generate(self, name: str, obj: object) -> str:
 }
 """
 
-FROZEN_COMMENT = "/* Auto-generated by Programs/_freeze_module.c */"
+FROZEN_COMMENT_C = "/* Auto-generated by Programs/_freeze_module.c */"
+FROZEN_COMMENT_PY = "/* Auto-generated by Programs/_freeze_module.py */"
 
 FROZEN_DATA_LINE = r"\s*(\d+,\s*)+\s*"
 
 
 def is_frozen_header(source: str) -> bool:
-    return source.startswith(FROZEN_COMMENT)
+    return source.startswith((FROZEN_COMMENT_C, FROZEN_COMMENT_PY))
 
 
 def decode_frozen_data(source: str) -> types.CodeType:

Tools/scripts/freeze_modules.py

@@ -264,6 +264,10 @@ def ispkg(self):
         else:
             return os.path.basename(self.pyfile) == '__init__.py'
 
+    @property
+    def isbootstrap(self):
+        return self.id in BOOTSTRAP
+
 
 def resolve_frozen_file(frozenid, destdir):
     """Return the filename corresponding to the given frozen ID.
@@ -476,7 +480,7 @@ def regen_frozen(modules):
     indent = '    '
     lastsection = None
     for mod in modules:
-        if mod.frozenid in BOOTSTRAP:
+        if mod.isbootstrap:
             lines = bootstraplines
         elif mod.section == TESTS_SECTION:
             lines = testlines
@@ -585,10 +589,17 @@ def regen_makefile(modules):
         pyfile = relpath_for_posix_display(src.pyfile, ROOT_DIR)
         pyfiles.append(f'\t\t{pyfile} \\')
 
-        freeze = (f'$(FREEZE_MODULE) {src.frozenid} '
-                  f'$(srcdir)/{pyfile} {frozen_header}')
+        if src.isbootstrap:
+            freezecmd = '$(FREEZE_MODULE_BOOTSTRAP)'
+            freezedep = '$(FREEZE_MODULE_BOOTSTRAP_DEPS)'
+        else:
+            freezecmd = '$(FREEZE_MODULE)'
+            freezedep = '$(FREEZE_MODULE_DEPS)'
+
+        freeze = (f'{freezecmd} {src.frozenid} '
+                    f'$(srcdir)/{pyfile} {frozen_header}')
         rules.extend([
-            f'{frozen_header}: $(FREEZE_MODULE) {pyfile}',
+            f'{frozen_header}: {pyfile} {freezedep}',
             f'\t{freeze}',
             '',
         ])