feat: pre-solved environments for mulled tests (#1082)

## Summary
- Pre-solves the mulled test environment on the host using `conda create
--dry-run --json`, then passes an `@EXPLICIT` spec file to the container
- The container installs from the explicit spec (no solver needed), runs
`create-env` post-processing, then executes tests
- Graceful fallback: if pre-solve fails for any reason, falls back to
the original `mulled-build build-and-test` path
- Automatically disabled when `mulled_upload_target` is set (upload
needs the Docker image produced by `mulled-build`)
- Adds `--no-presolved-mulled-test` CLI flag for opt-out

Related: #817

## Savings
60-180s per mulled test by eliminating a redundant solver run inside the
container.

## Risks
- **Medium**: `conda create --dry-run --json` output format varies
across conda versions. Robust fallback to original path mitigates this.
- **Medium**: Bypassing `mulled-build` means maintaining a parallel test
execution path. The `create-env --conda=: /usr/local` POSTINSTALL step
is replicated from the involucro wrapper.
- Backward compatible: graceful fallback ensures original behavior is
preserved on any failure.

## Test plan
- [x] Run existing test suite: `pytest test/`
- [x] Integration test: `bioconda-utils build recipes/ config.yml
--docker --mulled-test` on pyfaidx
- [x] Verify tests pass identically with and without
`--no-presolved-mulled-test`
- [x] Test fallback: presolved path failed (host/container platform
mismatch on macOS) and correctly fell back to mulled-build
- [x] Verify `mulled_upload_target` correctly disables pre-solved path
(confirmed: goes straight to mulled-build)

---------

Co-authored-by: Elmar Pruesse <epruesse@users.noreply.github.com>

Author: nh13

bioconda_utils/build.py

@@ -62,6 +62,8 @@ def build(
     dag: Optional[nx.DiGraph] = None,
     skiplist_leafs: bool = False,
     live_logs: bool = True,
+    presolved_mulled_test: bool = True,
+    mulled_upload_target=None,
 ) -> BuildResult:
     """
     Build a single recipe for a single env
@@ -198,6 +200,8 @@ def build(
     if mulled_test:
         logger.info("TEST START via mulled-build %s", recipe)
         mulled_images = []
+        # Use pre-solved test env unless we need the mulled-build image for upload
+        use_presolved = presolved_mulled_test and not mulled_upload_target
         for pkg_path in pkg_paths:
             try:
                 report_resources(f"Starting mulled build for {pkg_path}")
@@ -206,6 +210,7 @@ def build(
                     base_image=base_image,
                     conda_image=mulled_conda_image,
                     live_logs=live_logs,
+                    presolved=use_presolved,
                 )
             except sp.CalledProcessError:
                 logger.error("TEST FAILED: %s", recipe)
@@ -366,6 +371,7 @@ def build_recipes(
     live_logs: bool = True,
     exclude: List[str] = None,
     subdag_depth: int = None,
+    presolved_mulled_test: bool = True,
     fast_resolve: bool = True,
 ):
     """
@@ -532,6 +538,8 @@ def build_recipes(
             record_build_failure=record_build_failures,
             skiplist_leafs=skiplist_leafs,
             live_logs=live_logs,
+            presolved_mulled_test=presolved_mulled_test,
+            mulled_upload_target=mulled_upload_target,
         )
 
         if not res.success:

bioconda_utils/cli.py

@@ -619,6 +619,12 @@ def do_lint(
     action="store_true",
     help="Disable live logging during the build process",
 )
+@arg(
+    "--no-presolved-mulled-test",
+    action="store_true",
+    help="Disable pre-solved mulled tests: always use mulled-build to solve and install "
+    "the test environment from scratch.",
+)
 @arg(
     "--no-fast-resolve",
     action="store_true",
@@ -658,6 +664,7 @@ def build(
     record_build_failures=False,
     skiplist_leafs=False,
     disable_live_logs=False,
+    no_presolved_mulled_test=False,
     no_fast_resolve=False,
     exclude=None,
     subdag_depth=None,
@@ -733,6 +740,7 @@ def build(
         live_logs=(not disable_live_logs),
         exclude=exclude,
         subdag_depth=subdag_depth,
+        presolved_mulled_test=not no_presolved_mulled_test,
         fast_resolve=not no_fast_resolve,
     )
     exit(0 if success else 1)

bioconda_utils/pkg_test.py

@@ -2,6 +2,8 @@
 Mulled Tests
 """
 
+import json
+import subprocess as sp
 import tempfile
 import os
 import shlex
@@ -87,6 +89,163 @@ def get_image_name(path):
     return spec
 
 
+def _generate_explicit_spec(spec, channels, conda_bld_dir, tmpdir):
+    """Generate an @EXPLICIT spec file by dry-running conda create.
+
+    Parameters
+    ----------
+    spec : str
+        Package spec in name=version--build format
+    channels : list
+        List of resolved channel URLs (no "local")
+    conda_bld_dir : str
+        Path to local conda-bld directory
+    tmpdir : str
+        Directory to write the spec file into
+
+    Returns
+    -------
+    str or None
+        Path to explicit spec file, or None on failure
+    """
+    # Convert spec from mulled format (name=version--build) to conda format (name=version=build)
+    pkg_spec = spec.replace("--", "=")
+
+    channel_args = []
+    for ch in channels:
+        channel_args += ["-c", ch]
+
+    cmd = (
+        [
+            "conda",
+            "create",
+            "--dry-run",
+            "--json",
+            "--override-channels",
+        ]
+        + channel_args
+        + [pkg_spec]
+    )
+
+    logger.debug("Generating explicit spec: %s", cmd)
+    try:
+        result = sp.run(
+            cmd,
+            capture_output=True,
+            text=True,
+            timeout=300,
+        )
+        if result.returncode != 0:
+            logger.debug("conda create --dry-run failed: %s", result.stderr)
+            return None
+
+        data = json.loads(result.stdout)
+        actions = data.get("actions", {})
+        link_actions = actions.get("LINK", [])
+        if not link_actions:
+            logger.debug("No LINK actions in dry-run output")
+            return None
+
+        # Build @EXPLICIT spec file
+        spec_path = os.path.join(tmpdir, "explicit_spec.txt")
+        with open(spec_path, "w") as f:
+            f.write("@EXPLICIT\n")
+            for pkg in link_actions:
+                url = pkg.get("url")
+                if not url:
+                    # Build URL from channel + subdir + fn
+                    channel = pkg.get("channel", "")
+                    subdir = pkg.get("platform", pkg.get("subdir", ""))
+                    fn = pkg.get("dist_name", "")
+                    if fn and not fn.endswith((".tar.bz2", ".conda")):
+                        fn += ".conda"
+                    url = f"{channel}/{subdir}/{fn}"
+                md5 = pkg.get("md5", "")
+                line = url
+                if md5:
+                    line += f"#{md5}"
+                f.write(line + "\n")
+
+        logger.debug("Generated explicit spec with %d packages", len(link_actions))
+        return spec_path
+
+    except (sp.TimeoutExpired, json.JSONDecodeError, KeyError) as exc:
+        logger.debug("Failed to generate explicit spec: %s", exc)
+        return None
+
+
+def _test_with_explicit_spec(
+    spec_path, tests, base_image, conda_image, conda_bld_dir, live_logs
+):
+    """Run mulled test using a pre-solved explicit spec file.
+
+    Parameters
+    ----------
+    spec_path : str
+        Path to @EXPLICIT spec file
+    tests : str
+        Test commands to run
+    base_image : str
+        Base image for the test container
+    conda_image : str
+        Conda image used to install packages
+    conda_bld_dir : str
+        Path to local conda-bld directory (needed for file:// URLs)
+    live_logs : bool
+        Enable live log output
+    """
+    # Build a test script that:
+    # 1. Creates env from explicit spec
+    # 2. Runs create-env post-processing (activation/entrypoint scripts)
+    # 3. Runs the test commands
+    test_script = f"""#!/bin/bash
+set -eo pipefail
+
+# Install from pre-solved explicit spec (no solver needed)
+conda create --name test --file /opt/explicit_spec.txt --yes --quiet
+
+# Run create-env to set up activation/entrypoint scripts
+# This replicates the POSTINSTALL step from involucro
+create-env --conda=: /usr/local
+
+# Run tests
+{tests}
+"""
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        script_path = os.path.join(tmpdir, "test_script.bash")
+        with open(script_path, "w") as f:
+            f.write(test_script)
+
+        cmd = [
+            "docker",
+            "run",
+            "-t",
+            "--net",
+            "host",
+            "--rm",
+            "-v",
+            f"{script_path}:/opt/test_script.bash:ro",
+            "-v",
+            f"{spec_path}:/opt/explicit_spec.txt:ro",
+            "-v",
+            f"{conda_bld_dir}:{conda_bld_dir}:ro",
+        ]
+
+        env_args = []
+        if base_image is not None:
+            env_args += ["-e", f"DEST_BASE_IMAGE={base_image}"]
+
+        cmd += env_args
+        cmd += [conda_image]
+        cmd += ["/bin/bash", "/opt/test_script.bash"]
+
+        logger.debug("Pre-solved mulled test command: %s", cmd)
+        with utils.Progress():
+            p = utils.run(cmd, mask=False, live=live_logs)
+        return p
+
+
 def test_package(
     path,
     name_override=None,
@@ -95,6 +254,7 @@ def test_package(
     base_image=None,
     conda_image=MULLED_CONDA_IMAGE,
     live_logs=True,
+    presolved=True,
 ):
     """
     Tests a built package in a minimal docker container.
@@ -125,6 +285,11 @@ def test_package(
 
     live_logs : True | bool
         If True, enable live logging during the build process
+
+    presolved : bool
+        If True, attempt to pre-solve the test environment on the host and
+        pass an @EXPLICIT spec file to the container, avoiding a redundant
+        solver run. Falls back to the original mulled-build path on failure.
     """
 
     assert path.endswith((".tar.bz2", ".conda")), "Unrecognized path {0}".format(path)
@@ -139,16 +304,44 @@ def test_package(
     if "local" not in channels:
         raise ValueError('"local" must be in channel list')
 
-    channels = [
+    resolved_channels = [
         "file://{0}".format(conda_bld_dir) if channel == "local" else channel
         for channel in channels
     ]
 
-    channel_args = ["--channels", ",".join(channels)]
-
     tests = get_tests(path)
     logger.debug("Tests to run: %s", tests)
 
+    # Try the pre-solved path first for faster testing
+    if presolved:
+        try:
+            with tempfile.TemporaryDirectory() as tmpdir:
+                spec_path = _generate_explicit_spec(
+                    spec,
+                    resolved_channels,
+                    conda_bld_dir,
+                    tmpdir,
+                )
+                if spec_path is not None:
+                    logger.info("Using pre-solved explicit spec for mulled test")
+                    return _test_with_explicit_spec(
+                        spec_path,
+                        tests,
+                        base_image,
+                        conda_image,
+                        conda_bld_dir,
+                        live_logs,
+                    )
+                else:
+                    logger.info("Pre-solve failed, falling back to mulled-build")
+        except Exception as exc:
+            logger.info(
+                "Pre-solved test failed (%s), falling back to mulled-build", exc
+            )
+
+    # Original mulled-build path
+    channel_args = ["--channels", ",".join(resolved_channels)]
+
     cmd = [
         "mulled-build",
         "build-and-test",