Add automation for lint, format, and type check

.github/workflows/check.yml

@@ -0,0 +1,20 @@
+
+name: Check
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  check:
+    runs-on: "ubuntu-latest"
+    name: "check"
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: "Run checks"
+        run: |
+          python ./check.py

build-linux.py

@@ -9,7 +9,6 @@
 import sys
 import venv
 
-
 ROOT = pathlib.Path(os.path.abspath(__file__)).parent
 BUILD = ROOT / "build"
 VENV = BUILD / "venv.linux"

build-macos.py

@@ -9,7 +9,6 @@
 import sys
 import venv
 
-
 ROOT = pathlib.Path(os.path.abspath(__file__)).parent
 BUILD = ROOT / "build"
 VENV = BUILD / "venv.macos"

build-windows.py

@@ -9,7 +9,6 @@
 import sys
 import venv
 
-
 ROOT = pathlib.Path(os.path.abspath(__file__)).parent
 BUILD = ROOT / "build"
 DIST = ROOT / "dist"

check.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+from __future__ import annotations
+
+import argparse
+import os
+import pathlib
+import subprocess
+import sys
+import venv
+
+ROOT = pathlib.Path(os.path.abspath(__file__)).parent
+VENV = ROOT / "venv.dev"
+PIP = VENV / "bin" / "pip"
+PYTHON = VENV / "bin" / "python"
+REQUIREMENTS = ROOT / "requirements.dev.txt"
+
+
+def bootstrap():
+    venv.create(VENV, with_pip=True)
+
+    subprocess.run([str(PIP), "install", "-r", str(REQUIREMENTS)], check=True)
+
+    os.environ["PYBUILD_BOOTSTRAPPED"] = "1"
+    os.environ["PATH"] = "%s:%s" % (str(VENV / "bin"), os.environ["PATH"])
+    os.environ["PYTHONPATH"] = str(ROOT)
+
+    args = [str(PYTHON), __file__, *sys.argv[1:]]
+
+    os.execv(str(PYTHON), args)
+
+
+def run_command(command: list[str]) -> int:
+    print("$ " + " ".join(command))
+    returncode = subprocess.run(
+        command, stdout=sys.stdout, stderr=sys.stderr
+    ).returncode
+    print()
+    return returncode
+
+
+def run():
+    env = dict(os.environ)
+    env["PYTHONUNBUFFERED"] = "1"
+
+    parser = argparse.ArgumentParser(description="Check code.")
+    parser.add_argument(
+        "--fix",
+        action="store_true",
+        help="Fix problems",
+    )
+    args = parser.parse_args()
+
+    # Lints:
+    #   Sort imports
+    #   Unused import
+    #   Unused variable
+    check_args = ["--select", "I,F401,F841"]
+    format_args = []
+    mypy_args = [
+        "pythonbuild",
+        "check.py",
+        "build-linux.py",
+        "build-macos.py",
+        "build-windows.py",
+    ]
+
+    if args.fix:
+        check_args.append("--fix")
+    else:
+        format_args.append("--check")
+
+    check_result = run_command(["ruff", "check"] + check_args)
+    format_result = run_command(["ruff", "format"] + format_args)
+    mypy_result = run_command(["mypy"] + mypy_args)
+
+    if check_result + format_result + mypy_result:
+        print("Checks failed!")
+        sys.exit(1)
+    else:
+        print("Checks passed!")
+
+
+if __name__ == "__main__":
+    try:
+        if "PYBUILD_BOOTSTRAPPED" not in os.environ:
+            bootstrap()
+        else:
+            run()
+    except subprocess.CalledProcessError as e:
+        sys.exit(e.returncode)

cpython-unix/build-main.py

@@ -4,8 +4,8 @@
 # file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
 import argparse
-import os
 import multiprocessing
+import os
 import pathlib
 import platform
 import subprocess

cpython-unix/build.py

@@ -18,12 +18,12 @@
 
 from pythonbuild.buildenv import build_environment
 from pythonbuild.cpython import (
+    STDLIB_TEST_PACKAGES,
     derive_setup_local,
     extension_modules_config,
-    meets_python_minimum_version,
     meets_python_maximum_version,
+    meets_python_minimum_version,
     parse_setup_line,
-    STDLIB_TEST_PACKAGES,
 )
 from pythonbuild.docker import build_docker_image, get_image, write_dockerfiles
 from pythonbuild.downloads import DOWNLOADS
@@ -34,12 +34,12 @@
     clang_toolchain,
     create_tar_from_directory,
     download_entry,
-    get_targets,
     get_target_settings,
+    get_targets,
     target_needs,
     validate_python_json,
-    write_package_versions,
     write_cpython_version,
+    write_package_versions,
     write_target_settings,
     write_triples_makefiles,
 )
@@ -490,10 +490,10 @@ def python_build_info(
     binary_suffix = ""
 
     if platform == "linux64":
-        bi["core"][
-            "static_lib"
-        ] = "install/lib/python{version}/config-{version}{binary_suffix}-x86_64-linux-gnu/libpython{version}{binary_suffix}.a".format(
-            version=version, binary_suffix=binary_suffix
+        bi["core"]["static_lib"] = (
+            "install/lib/python{version}/config-{version}{binary_suffix}-x86_64-linux-gnu/libpython{version}{binary_suffix}.a".format(
+                version=version, binary_suffix=binary_suffix
+            )
         )
 
         if not musl:
@@ -513,10 +513,10 @@ def python_build_info(
         else:
             object_file_format = "elf"
     elif platform == "macos":
-        bi["core"][
-            "static_lib"
-        ] = "install/lib/python{version}/config-{version}{binary_suffix}-darwin/libpython{version}{binary_suffix}.a".format(
-            version=version, binary_suffix=binary_suffix
+        bi["core"]["static_lib"] = (
+            "install/lib/python{version}/config-{version}{binary_suffix}-darwin/libpython{version}{binary_suffix}.a".format(
+                version=version, binary_suffix=binary_suffix
+            )
         )
         bi["core"]["shared_lib"] = "install/lib/libpython%s%s.dylib" % (
             version,

cpython-windows/build.py

@@ -7,6 +7,7 @@
 import concurrent.futures
 import io
 import json
+import multiprocessing
 import os
 import pathlib
 import re
@@ -15,20 +16,19 @@
 import sys
 import tempfile
 import zipfile
-import multiprocessing
 
-from pythonbuild.downloads import DOWNLOADS
 from pythonbuild.cpython import (
-    parse_config_c,
     STDLIB_TEST_PACKAGES,
     meets_python_minimum_version,
+    parse_config_c,
 )
+from pythonbuild.downloads import DOWNLOADS
 from pythonbuild.utils import (
+    compress_python_archive,
     create_tar_from_directory,
     download_entry,
     extract_tar_to_directory,
     extract_zip_to_directory,
-    compress_python_archive,
     normalize_tar_archive,
     release_tag_from_git,
     validate_python_json,

pythonbuild/docker.py

@@ -9,7 +9,7 @@
 import pathlib
 import tarfile
 
-import docker
+import docker  # type: ignore
 import jinja2
 
 from .logging import log, log_raw
@@ -85,7 +85,7 @@ def get_image(client, source_dir: pathlib.Path, image_dir: pathlib.Path, name):
             return image_id
 
         else:
-            return build_docker_image(client, source_dir, image_dir, name)
+            return build_docker_image(client, str(source_dir).encode(), image_dir, name)
 
 
 def copy_file_to_container(path, container, container_path, archive_path=None):

pythonbuild/utils.py

@@ -17,9 +17,9 @@
 import sys
 import tarfile
 import time
-import zipfile
 import urllib.error
 import urllib.request
+import zipfile
 
 import yaml
 import zstandard
@@ -202,9 +202,9 @@ def write_target_settings(targets, dest_path: pathlib.Path):
         for key in ("host_cc", "host_cxx", "target_cc", "target_cflags"):
             payload[key] = settings.get(key)
 
-        payload = json.dumps(payload, indent=4).encode("utf-8")
+        serialized_payload = json.dumps(payload, indent=4).encode("utf-8")
 
-        write_if_different(dest_path / triple, payload)
+        write_if_different(dest_path / triple, serialized_payload)
 
 
 class IntegrityError(Exception):
@@ -298,11 +298,17 @@ def download_to_path(url: str, path: pathlib.Path, size: int, sha256: str):
 def download_entry(key: str, dest_path: pathlib.Path, local_name=None) -> pathlib.Path:
     entry = DOWNLOADS[key]
     url = entry["url"]
+    size = entry["size"]
+    sha256 = entry["sha256"]
+
+    assert isinstance(url, str)
+    assert isinstance(size, int)
+    assert isinstance(sha256, str)
 
     local_name = local_name or url[url.rindex("/") + 1 :]
 
     local_path = dest_path / local_name
-    download_to_path(url, local_path, entry["size"], entry["sha256"])
+    download_to_path(url, local_path, size, sha256)
 
     return local_path
 

requirements.dev.in

@@ -0,0 +1,7 @@
+-r requirements.txt
+
+ruff
+mypy
+types-jsonschema
+types-PyYAML
+types-jinja2