Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/CODEOWNERS
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
/charts/ @lexfrei
172 changes: 172 additions & 0 deletions .github/scripts/assert_imports.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,172 @@
#!/usr/bin/env python3
"""Import every module in an image that ships compiled code.

A build proves pip exited zero. It does not prove the result loads: an
extension can install cleanly and then fail at import against the base
image's C library. Nothing else in CI would notice, because the app is
never started without a database behind it.

Written for the web image: it also imports that container's entrypoints, which
ship no compiled code of their own. Run it inside the image under test, naming
packages that image is known to have compiled code for — the scan silently
derives nothing if it looks at the wrong directory, and those names are what
turns that into a failure:

docker run --rm -v "$PWD/.github/scripts:/s:ro" IMAGE python /s/assert_imports.py psycopg2 PIL

Run --self-test on the runner to check the derivation itself.
"""
import importlib
import pathlib
import sys
import sysconfig
import tempfile

# The packages the web container's entrypoints live in — gunicorn, daphne and
# celery all import their way here. Pure Python, so they ship no compiled
# object of their own and the scan below would miss them. Web-specific, which
# is why the docstring says which image this script is for.
ENTRYPOINTS = {"django", "channels", "celery", "daphne"}

# Directories a wheel places beside its package. For .libs and .dist-info the
# name is simply not a module and attributing anything to it would be wrong.
# .data is different and worth knowing: a real importable extension can live
# under it, so skipping it is a false negative — the scan then reports a clean
# pass on a module it never imported. Name such a package in the caller's
# expected list if an image ever grows one. Sidecar libraries observed in practice carry a soname
# (libjpeg-ad9713a2.so.62.4.0) and so would not match *.so anyway, but .data
# layouts do produce a bare directory name, and relying on a naming habit of
# one packaging tool is not the same as excluding by construction.
NOT_PACKAGES = (".libs", ".dist-info", ".data")

# A bare .so at the root of site-packages is treated as a module, which is
# correct for extensions like _cffi_backend and zmq.abi3.so. A wheel that
# instead drops a loader shim there (libtorch_global_deps.so is the canonical
# example) would make this try to import it and fail loudly. Nothing in the
# current dependency set does that; the next compiled dependency might. The
# same applies to a package whose import has side effects — importing every
# extension is the point, but a dependency that talks to the network or reads
# a config at import time would fail here for reasons unrelated to the change
# being reviewed.


def derive(site_packages):
"""Top-level importable names that own a compiled extension."""
names = set()
for so in pathlib.Path(site_packages).rglob("*.so"):
top = so.relative_to(site_packages).parts[0]
if top.endswith(NOT_PACKAGES):
continue
names.add(top.split(".")[0])
return names


def self_test():
with tempfile.TemporaryDirectory() as tmp:
root = pathlib.Path(tmp)
for path in (
"PIL/_imaging.cpython-310-x86_64-linux-gnu.so",
"pillow.libs/libjpeg.so",
"psycopg2_binary.libs/libpq.so",
"psycopg2/_psycopg.cpython-310.so",
"opencv_python-4.9.0.data/purelib/cv2/cv2.so",
"numpy-1.26.3.dist-info/whatever.so",
# Extensions living directly in site-packages rather than inside a
# package directory, which is what the split on "." is for.
"_cffi_backend.cpython-38-x86_64-linux-gnu.so",
"zmq.abi3.so",
):
f = root / path
f.parent.mkdir(parents=True, exist_ok=True)
f.touch()
got = derive(root)
want = {"PIL", "psycopg2", "_cffi_backend", "zmq"}
assert got == want, f"derived {sorted(got)}, wanted {sorted(want)}"

# A directory with nothing to find must be a failure, not a quiet pass.
with tempfile.TemporaryDirectory() as empty:
assert derive(empty) == set(), "an empty tree should derive nothing"
try:
check_floor(set(), {"psycopg2"})
except SystemExit:
pass
else:
raise AssertionError("an empty derivation should have been rejected")
# And that a satisfied floor passes: the reject case alone would be met by
# a check_floor that always failed.
check_floor({"psycopg2", "PIL", "numpy"}, {"psycopg2", "PIL"})

# main() itself. Deriving the names proves nothing if nothing imports
# them, and this is the only thing between a wheel whose compiled half
# does not load on arm64 and a published image.
previous_path = list(sys.path)
original = sysconfig.get_paths
# The entrypoints are the real image's pure-Python modules; the fixture
# has none of them and importing them here would fail for a reason this
# case is not about.
global ENTRYPOINTS
previous_entrypoints = ENTRYPOINTS
ENTRYPOINTS = frozenset()
with tempfile.TemporaryDirectory() as tmp:
root = pathlib.Path(tmp) / "site-packages"
for name, body in (("importable", ""), ("broken", "raise ImportError('compiled half missing')\n")):
(root / name).mkdir(parents=True)
(root / name / "__init__.py").write_text(body)
(root / name / "_ext.cpython-310-x86_64-linux-gnu.so").write_bytes(b"")
sys.path.insert(0, str(root))
sysconfig.get_paths = lambda *a, **k: {"platlib": str(root)}
try:
try:
main(["importable", "broken"])
except (SystemExit, ImportError):
pass
else:
raise AssertionError("a module that cannot import was reported as loaded")
# And the healthy one on its own still passes, so the case above
# is about the import and not about main() refusing everything.
(root / "broken" / "__init__.py").write_text("")
main(["importable", "broken"])
finally:
sysconfig.get_paths = original
sys.path[:] = previous_path
ENTRYPOINTS = previous_entrypoints

print("self-test ok:", sorted(got))


def check_floor(names, expected):
"""Fail unless the scan found what the caller says the image contains.

An unreadable or wrong site-packages path yields an empty set rather than
an error, which would otherwise leave this asserting nothing beyond the
pure-Python entrypoints.
"""
missing = set(expected) - names
if missing:
sys.exit(
f"found {len(names)} modules with compiled code, but not "
f"{sorted(missing)} — the scan is looking at the wrong place, so "
f"it is proving nothing"
)


def main(expected):
if not expected:
sys.exit(f"usage: {sys.argv[0]} MODULE [MODULE...] | --self-test")
# platlib, not purelib: compiled extensions belong to the platform-
# specific scheme. The two are the same directory on these images, which
# is exactly why reading the wrong one would go unnoticed.
site_packages = pathlib.Path(sysconfig.get_paths()["platlib"])
native = derive(site_packages)
check_floor(native, expected)
modules = sorted(native | ENTRYPOINTS)
for name in modules:
importlib.import_module(name)
print("loaded", len(modules), "modules:", " ".join(modules))


if __name__ == "__main__":
if "--self-test" in sys.argv:
self_test()
else:
main(sys.argv[1:])
54 changes: 54 additions & 0 deletions .github/scripts/assert_inference_healthy.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
#!/usr/bin/env bash
# Start an ml_api image and wait for it to report a loaded model.
#
# Usage: assert_inference_healthy.sh IMAGE_REF [COMMAND...]
#
# The failure this exists for is quiet: lib/detection_model.py swallows an
# ImportError from the inference backend, load_net() then raises at import
# time, and the gunicorn worker dies at boot. A build that merely finished
# proves nothing.
set -euo pipefail

if [ "$#" -lt 1 ]; then
echo "usage: $(basename "$0") IMAGE_REF [COMMAND...]" >&2
exit 2
fi

ref="$1"
shift

# Overridable so the failing path can be exercised without waiting out the
# real deadline. The cases that do live in the workflow rather than here,
# because both of them need a container to run: see the two negative-path
# steps in .github/workflows/ci.yaml.
attempts="${ATTEMPTS:-45}"

# No --rm: a container that dies at boot would be gone before the trap could
# show why, which is the one case this script exists for.
cid="$(docker run --detach --publish 3333:3333 "${ref}" "$@")"
trap 'docker rm --force "${cid}" > /dev/null 2>&1 || true' EXIT

for _ in $(seq "${attempts}"); do
# A worker that dies at import takes the container with it. Waiting out the
# full deadline for that reads as slowness rather than as the failure it is.
if [ "$(docker inspect --format '{{.State.Running}}' "${cid}")" != "true" ]; then
echo "::error::${ref} exited before answering on /hc/"
docker logs "${cid}" || true
exit 1
fi
# --max-time matters more than it looks: a native crash inside the inference
# runtime leaves gunicorn's master holding the listening socket while it
# respawns workers forever, so the connection is accepted by the kernel and a
# request without a deadline hangs until the job's own timeout. The body is
# what is checked, not the status: /hc/ is a plain 200, and a worker that died
# loading the model answers nothing at all.
if [ "$(curl --silent --fail --connect-timeout 2 --max-time 5 http://127.0.0.1:3333/hc/ || true)" = "ok" ]; then
echo "inference API is up"
exit 0
fi
sleep 2
done

echo "::error::${ref} did not report a loaded model on /hc/ in time"
docker logs "${cid}" || true
exit 1
156 changes: 156 additions & 0 deletions .github/scripts/assert_probes_match_loaders.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,156 @@
#!/usr/bin/env python3
"""Each weights download is gated on a probe; the probe has to ask what the
loader asks.

`ml_api/Dockerfile` downloads a 200 MB model only when the base can use it,
and decides that with a shell test per format. The decision is only as good as
what it looks for: probing for a directory while the loader dlopens a file
inside it, or for a shared library while the loader imports a Python package,
passes on a base where the model is unreadable and 200 MB is wasted, or skips
on one where it would have worked.

The names cannot be compared automatically to a loader's behaviour, so this
compares them to the loader's source: whatever the probe names must appear in
the module that does the loading.

assert_probes_match_loaders.py DOCKERFILE MODULE_DIR
assert_probes_match_loaders.py --self-test
"""
import re
import sys
from pathlib import Path

# (format, what the probe must name, the module that loads it). The probe line
# is found by the format's model path, which is what the download writes.
PAIRS = (
("darknet", ("libdarknet_gpu.so", "libdarknet_cpu.so"), "darknet.py"),
("onnx", ("onnxruntime",), "onnx.py"),
("rknn", ("rknnlite",), "rknn.py"),
)


def probe_for(dockerfile_text, fmt):
"""The `if` line guarding this format's download, or None."""
lines = dockerfile_text.splitlines()
for number, line in enumerate(lines):
if f"/model_cache/ml_api/{fmt}/model-weights.{fmt}" not in line:
continue
# The guard is the nearest `if` above the download it guards.
for candidate in reversed(lines[:number]):
if re.search(r"(^|&&)\s*if\s", candidate):
return candidate
return None


def mismatched(dockerfile_text, module_text_by_name):
"""(format, name) for every probe naming something its loader does not."""
found = []
for fmt, names, module in PAIRS:
probe = probe_for(dockerfile_text, fmt)
if probe is None:
found.append((fmt, "no probe guards this download"))
continue
source = module_text_by_name.get(module, "")
for name in names:
if name in probe and name not in source:
found.append((fmt, f"{name} is not in {module}"))
if not any(name in probe for name in names):
found.append((fmt, f"names none of {list(names)}, which {module} uses"))
return found


def self_test():
def dockerfile(darknet_probe):
return (
"RUN mkdir -p /model_cache \\\n"
f" && if {darknet_probe}; then \\\n"
" fetch_verified x /model_cache/ml_api/darknet/model-weights.darknet 1 y; \\\n"
" fi \\\n"
" && if python3 -c 'import onnxruntime'; then \\\n"
" fetch_verified x /model_cache/ml_api/onnx/model-weights.onnx 1 y; \\\n"
" fi \\\n"
" && if python3 -c 'import rknnlite.api'; then \\\n"
" fetch_verified x /model_cache/ml_api/rknn/model-weights.rknn 1 y; \\\n"
" fi\n"
)

modules = {
"darknet.py": "so_path = os.path.join('/darknet', 'libdarknet_gpu.so')\nlibdarknet_cpu.so\n",
"onnx.py": "import onnxruntime\n",
"rknn.py": "from rknnlite.api import RKNNLite\n",
}
good = dockerfile("[ -e /darknet/libdarknet_gpu.so ] || [ -e /darknet/libdarknet_cpu.so ]")
assert mismatched(good, modules) == [], mismatched(good, modules)

# The shape that shipped: a directory test standing in for the library the
# loader opens. An empty or half-copied /darknet passes it.
directory = dockerfile("[ -d /darknet ]")
assert [f for f, _ in mismatched(directory, modules)] == ["darknet"], mismatched(directory, modules)

# And the other direction: probing for a library while the loader imports
# a package. This is what made the rk3588 download land on a base that
# could not read it.
library = (
"RUN mkdir -p /model_cache \\\n"
" && if [ -e /usr/lib/librknnrt.so ]; then \\\n"
" fetch_verified x /model_cache/ml_api/rknn/model-weights.rknn 1 y; \\\n"
" fi\n"
)
# Only the rknn download is present here, so the other two report as
# unguarded; what this case pins is the reason given for rknn.
assert ("rknn", "names none of ['rknnlite'], which rknn.py uses") in mismatched(library, modules), mismatched(library, modules)

# A download with no guard at all is the same failure without the excuse.
ungated = "RUN fetch_verified x /model_cache/ml_api/onnx/model-weights.onnx 1 y\n"
assert ("onnx", "no probe guards this download") in mismatched(ungated, modules)

# main() itself: without driving it, replacing its sys.exit(1) with a
# return leaves every case above green while CI prints the errors and
# passes. Every sibling checker here drives main() for this reason.
import os
import tempfile

previous = os.getcwd()
try:
with tempfile.TemporaryDirectory() as tmp:
os.chdir(tmp)
os.makedirs("lib")
for name, body in modules.items():
open(f"lib/{name}", "w").write(body)
open("Dockerfile", "w").write(good)
main(["Dockerfile", "lib"])
open("Dockerfile", "w").write(directory)
try:
main(["Dockerfile", "lib"])
except SystemExit:
pass
else:
raise AssertionError("a probe that asks the wrong question did not stop main()")
finally:
os.chdir(previous)

print("self-test ok")


def main(argv):
if len(argv) != 2:
sys.exit(f"usage: {sys.argv[0]} DOCKERFILE MODULE_DIR | --self-test")
dockerfile, module_dir = Path(argv[0]), Path(argv[1])
modules = {name: (module_dir / name).read_text() for _, _, name in PAIRS}
bad = mismatched(dockerfile.read_text(), modules)
for fmt, why in bad:
print(
f"::error::the {fmt} probe in {dockerfile} {why}. It decides "
"whether to download 200 MB, so asking a different question than "
"the loader means the answer is right by luck."
)
if bad:
sys.exit(1)
print(f"all {len(PAIRS)} weight probes name what their loader uses")


if __name__ == "__main__":
if "--self-test" in sys.argv:
self_test()
else:
main(sys.argv[1:])
Loading