From b0ad28c39c2078cb46ed1201505225c8d15697ae Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Mon, 18 Mar 2024 15:38:10 -0500 Subject: [PATCH 1/2] Allow building CPython from a local source directory --- cpython-unix/Makefile | 5 ++++ cpython-unix/build-main.py | 27 +++++++++++++++++--- cpython-unix/build.py | 51 +++++++++++++++++++++++++++++--------- cpython-windows/build.py | 1 + pythonbuild/buildenv.py | 10 ++++++-- pythonbuild/utils.py | 13 +++++++++- 6 files changed, 88 insertions(+), 19 deletions(-) diff --git a/cpython-unix/Makefile b/cpython-unix/Makefile index 7d534899..08488956 100644 --- a/cpython-unix/Makefile +++ b/cpython-unix/Makefile @@ -17,6 +17,10 @@ ifndef PYBUILD_HOST_PLATFORM $(error PYBUILD_HOST_PLATFORM not defined) endif +ifndef PYBUILD_PYTHON_SOURCE + $(error PYBUILD_PYTHON_SOURCE not defined) +endif + ifndef PYBUILD_PYTHON_VERSION $(error PYBUILD_PYTHON_VERSION not defined) endif @@ -33,6 +37,7 @@ RUN_BUILD = $(BUILD) \ --host-platform $(HOST_PLATFORM) \ --target-triple $(TARGET_TRIPLE) \ --optimizations $(PYBUILD_OPTIMIZATIONS) \ + --python-source $(PYBUILD_PYTHON_SOURCE) \ --dest-archive $@ \ $(NULL) diff --git a/cpython-unix/build-main.py b/cpython-unix/build-main.py index bc09733a..4f68e568 100755 --- a/cpython-unix/build-main.py +++ b/cpython-unix/build-main.py @@ -71,6 +71,11 @@ def main(): default="cpython-3.11", help="Python distribution to build", ) + parser.add_argument( + "--python-source", + default=None, + help="A custom path to CPython source files to use", + ) parser.add_argument( "--break-on-failure", action="store_true", @@ -118,6 +123,12 @@ def main(): ) return 1 + python_source = ( + (str(pathlib.Path(args.python_source).resolve())) + if args.python_source + else "null" + ) + musl = "musl" in target_triple env = dict(os.environ) @@ -125,6 +136,7 @@ def main(): env["PYBUILD_HOST_PLATFORM"] = host_platform env["PYBUILD_TARGET_TRIPLE"] = target_triple env["PYBUILD_OPTIMIZATIONS"] = args.optimizations + env["PYBUILD_PYTHON_SOURCE"] = python_source if musl: env["PYBUILD_MUSL"] = "1" if args.break_on_failure: @@ -132,9 +144,16 @@ def main(): if args.no_docker: env["PYBUILD_NO_DOCKER"] = "1" - entry = DOWNLOADS[args.python] - env["PYBUILD_PYTHON_VERSION"] = entry["version"] - env["PYBUILD_PYTHON_MAJOR_VERSION"] = ".".join(entry["version"].split(".")[0:2]) + if not args.python_source: + entry = DOWNLOADS[args.python] + env["PYBUILD_PYTHON_VERSION"] = cpython_version = entry["version"] + else: + if "PYBUILD_PYTHON_VERSION" not in env: + print("PYBUILD_PYTHON_VERSION must be set when using `--python-source`") + return 1 + cpython_version = env["PYBUILD_PYTHON_VERSION"] + + env["PYBUILD_PYTHON_MAJOR_VERSION"] = ".".join(cpython_version.split(".")[0:2]) if "PYBUILD_RELEASE_TAG" in os.environ: release_tag = os.environ["PYBUILD_RELEASE_TAG"] @@ -142,7 +161,7 @@ def main(): release_tag = release_tag_from_git() archive_components = [ - "cpython-%s" % entry["version"], + "cpython-%s" % cpython_version, target_triple, args.optimizations, ] diff --git a/cpython-unix/build.py b/cpython-unix/build.py index 8eff1977..1a5bb07a 100755 --- a/cpython-unix/build.py +++ b/cpython-unix/build.py @@ -32,12 +32,14 @@ add_env_common, add_licenses_to_extension_entry, clang_toolchain, + create_tar_from_directory, download_entry, get_targets, get_target_settings, target_needs, validate_python_json, write_package_versions, + write_cpython_version, write_target_settings, write_triples_makefiles, ) @@ -62,8 +64,7 @@ def install_sccache(build_env): """ candidates = [ # Prefer a binary in the project itself. - ROOT - / "sccache", + ROOT / "sccache", ] # Look for sccache in $PATH, but only if the build environment @@ -255,10 +256,9 @@ def simple_build( build_env.copy_file(SUPPORT / ("build-%s.sh" % entry)) env = { - "%s_VERSION" - % entry.upper() - .replace("-", "_") - .replace(".", "_"): DOWNLOADS[entry]["version"], + "%s_VERSION" % entry.upper().replace("-", "_").replace(".", "_"): DOWNLOADS[ + entry + ]["version"], } add_target_env(env, host_platform, target_triple, build_env) @@ -684,13 +684,23 @@ def build_cpython( optimizations, dest_archive, version=None, + python_source=None, ): """Build CPython in a Docker image'""" entry_name = "cpython-%s" % version entry = DOWNLOADS[entry_name] - python_version = entry["version"] + if not python_source: + python_version = entry["version"] + python_archive = download_entry(entry_name, DOWNLOADS_PATH) + else: + python_version = os.environ["PYBUILD_PYTHON_VERSION"] + python_archive = DOWNLOADS_PATH / ("Python-%s.tar.xz" % python_version) + print("Compressing %s to %s" % (python_source, python_archive)) + with python_archive.open("wb") as fh: + create_tar_from_directory( + fh, python_source, path_prefix="Python-%s" % python_version + ) - python_archive = download_entry(entry_name, DOWNLOADS_PATH) setuptools_archive = download_entry("setuptools", DOWNLOADS_PATH) pip_archive = download_entry("pip", DOWNLOADS_PATH) @@ -726,7 +736,9 @@ def build_cpython( for p in sorted(packages): build_env.install_artifact_archive(BUILD, p, target_triple, optimizations) - build_env.install_toolchain_archive(BUILD, entry_name, host_platform) + build_env.install_toolchain_archive( + BUILD, entry_name, host_platform, version=python_version + ) for p in ( python_archive, @@ -762,8 +774,8 @@ def build_cpython( env = { "PIP_VERSION": DOWNLOADS["pip"]["version"], - "PYTHON_VERSION": entry["version"], - "PYTHON_MAJMIN_VERSION": ".".join(entry["version"].split(".")[0:2]), + "PYTHON_VERSION": python_version, + "PYTHON_MAJMIN_VERSION": ".".join(python_version.split(".")[0:2]), "SETUPTOOLS_VERSION": DOWNLOADS["setuptools"]["version"], "TOOLCHAIN": "clang-%s" % host_platform, } @@ -824,7 +836,7 @@ def build_cpython( "target_triple": target_triple, "optimizations": optimizations, "python_tag": entry["python_tag"], - "python_version": entry["version"], + "python_version": python_version, "python_stdlib_test_packages": sorted(STDLIB_TEST_PACKAGES), "python_symbol_visibility": python_symbol_visibility, "python_extension_module_loading": extension_module_loading, @@ -924,6 +936,11 @@ def main(): "--dest-archive", required=True, help="Path to archive that we are producing" ) parser.add_argument("--docker-image", help="Docker image to use for building") + parser.add_argument( + "--python-source", + default=None, + help="A custom path to CPython source files to use", + ) parser.add_argument("action") args = parser.parse_args() @@ -933,6 +950,9 @@ def main(): target_triple = args.target_triple host_platform = args.host_platform optimizations = args.optimizations + python_source = ( + pathlib.Path(args.python_source) if args.python_source != "null" else None + ) dest_archive = pathlib.Path(args.dest_archive) docker_image = args.docker_image @@ -969,6 +989,12 @@ def main(): write_target_settings(targets, BUILD / "targets") write_package_versions(BUILD / "versions") + # Override the DOWNLOADS package entry for CPython for the local build + if python_source: + write_cpython_version( + BUILD / "versions", os.environ["PYBUILD_PYTHON_VERSION"] + ) + elif action.startswith("image-"): image_name = action[6:] image_path = BUILD / ("%s.Dockerfile" % image_name) @@ -1179,6 +1205,7 @@ def main(): optimizations=optimizations, dest_archive=dest_archive, version=action.split("-")[1], + python_source=python_source, ) else: diff --git a/cpython-windows/build.py b/cpython-windows/build.py index 5167e47c..5f1aefd5 100644 --- a/cpython-windows/build.py +++ b/cpython-windows/build.py @@ -2656,6 +2656,7 @@ def main(): "cpython-3.10", "cpython-3.11", "cpython-3.12", + "cpython-3.13", }, default="cpython-3.11", help="Python distribution to build", diff --git a/pythonbuild/buildenv.py b/pythonbuild/buildenv.py index 8716bc11..6b0605df 100644 --- a/pythonbuild/buildenv.py +++ b/pythonbuild/buildenv.py @@ -152,9 +152,15 @@ def copy_file(self, source: pathlib.Path, dest_path=None, dest_name=None): log("copying %s to %s/%s" % (source, dest_dir, dest_name)) shutil.copy(source, dest_dir / dest_name) - def install_toolchain_archive(self, build_dir, package_name, host_platform): + def install_toolchain_archive( + self, build_dir, package_name, host_platform, version=None + ): entry = DOWNLOADS[package_name] - basename = "%s-%s-%s.tar" % (package_name, entry["version"], host_platform) + basename = "%s-%s-%s.tar" % ( + package_name, + version or entry["version"], + host_platform, + ) p = build_dir / basename dest_path = self.td / "tools" diff --git a/pythonbuild/utils.py b/pythonbuild/utils.py index 09f90c43..68c64bf1 100644 --- a/pythonbuild/utils.py +++ b/pythonbuild/utils.py @@ -182,6 +182,17 @@ def write_package_versions(dest_path: pathlib.Path): write_if_different(p, content.encode("ascii")) +def write_cpython_version(dest_path: pathlib.Path, version: str): + """Write a CPython version in a directory.""" + dest_path.mkdir(parents=True, exist_ok=True) + + major_minor = ".".join(version.split(".")[:2]) + k = "cpython-%s" % major_minor + p = dest_path / ("VERSION.%s" % k) + content = "%s_VERSION := %s\n" % (k.upper().replace("-", "_"), version) + write_if_different(p, content.encode("ascii")) + + def write_target_settings(targets, dest_path: pathlib.Path): dest_path.mkdir(parents=True, exist_ok=True) @@ -621,7 +632,7 @@ def release_download_statistics(mode="by_asset"): print("%d\t%s" % (count, build)) elif mode == "by_tag": for tag, count in sorted(by_tag.items()): - print("%d\t%s"% (count, tag)) + print("%d\t%s" % (count, tag)) elif mode == "total": print("%d" % by_tag.total()) else: From b65100062ead981d4b7c62a4778f2568a9c30b95 Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Tue, 19 Mar 2024 20:58:12 -0500 Subject: [PATCH 2/2] Allow `version` override in `ContainerContext.install_toolchain_archive` as well --- pythonbuild/buildenv.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pythonbuild/buildenv.py b/pythonbuild/buildenv.py index 6b0605df..0ce3cd03 100644 --- a/pythonbuild/buildenv.py +++ b/pythonbuild/buildenv.py @@ -38,9 +38,15 @@ def copy_file(self, source: pathlib.Path, dest_path=None, dest_name=None): dest_path = dest_path or "/build" copy_file_to_container(source, self.container, dest_path, dest_name) - def install_toolchain_archive(self, build_dir, package_name, host_platform): + def install_toolchain_archive( + self, build_dir, package_name, host_platform, version=None + ): entry = DOWNLOADS[package_name] - basename = "%s-%s-%s.tar" % (package_name, entry["version"], host_platform) + basename = "%s-%s-%s.tar" % ( + package_name, + version or entry["version"], + host_platform, + ) p = build_dir / basename self.copy_file(p)