diff --git a/README.md b/README.md index 9826b92..41c4f0c 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ aiida-agents chat # interactive, and the only mode that can approve a w `ask` is one-shot and cannot approve anything, so a request that wants to submit will tell you to use `chat`. -Other commands: `check` (provider reachability), `config` (effective settings), `rag search`, `mcp` (serve the read-only tools over the Model Context Protocol). +Other commands: `doctor` (diagnose the whole setup; `--warm` also proves the model generates), `config` (effective settings), `rag search`, `mcp` (serve the read-only tools over the Model Context Protocol). ## Where to look diff --git a/docs/gsoc/manual-test-plan.md b/docs/gsoc/manual-test-plan.md index 43f80c7..b23de4b 100644 --- a/docs/gsoc/manual-test-plan.md +++ b/docs/gsoc/manual-test-plan.md @@ -19,9 +19,9 @@ ______________________________________________________________________ ## Phase 0: setup and sanity ```bash -aiida-agents doctor # profile, model reachability, RAG index, docs toolchain +aiida-agents doctor # profile, daemon, model reachability, docs toolchain, RAG index, sandbox +aiida-agents doctor --warm # the same, plus one generation to prove the model serves aiida-agents config show # every setting, its env var, and where the value came from -aiida-agents check # provider reachability, without loading the model aiida-agents --help ``` diff --git a/src/aiida_agents/cli/agent.py b/src/aiida_agents/cli/agent.py index 75d9be2..69bf19b 100644 --- a/src/aiida_agents/cli/agent.py +++ b/src/aiida_agents/cli/agent.py @@ -1,9 +1,10 @@ """Agent lifecycle for the CLI: build it, run it, probe it. Turns resolved settings into a ready agent (``_build_agent``), runs a one-shot -query (``ask``), and checks/warms the configured model (``_probe_model`` + -``_diagnose_probe_failure``). The heavy aiida / agent-stack imports stay local so -``--help`` and shell completion never load AiiDA. +query (``ask``), and probes the configured model for ``doctor`` +(``_probe_reachable``, ``_probe_model``, ``_probe_failure_hint``). The heavy +aiida / agent-stack imports stay local so ``--help`` and shell completion never +load AiiDA. """ from __future__ import annotations @@ -17,11 +18,10 @@ from pydantic_ai import Agent from pydantic_ai.exceptions import UserError from pydantic_ai.messages import ModelMessage -from typing_extensions import assert_never from aiida_agents._settings import ModelSettings, _Provider, _format_validation_error from aiida_agents.agents.planner import Specialist, Step, _as_specialist -from aiida_agents.cli.ollama import _ensure_ollama_model, _ollama_pull +from aiida_agents.cli.ollama import _ensure_ollama_model from aiida_agents.agents.handoff import Handoff, NodeReference from aiida_agents.cli.output import _trace_tool_calls, console @@ -53,13 +53,6 @@ class _StepResult(NamedTuple): #: not an agent, so it is never passed to ``get_agent``. _SPECIALISTS = ("analysis", "execution", "codegen") -# Colored status glyphs so ``check`` marks success/failure the same green/red as -# ``doctor`` (which renders its rows through rich). ``click.echo`` strips the -# color automatically when the output is not a terminal. -_OK = click.style("✓", fg="green") -_FAIL = click.style("✗", fg="red") -_WARN = click.style("!", fg="yellow") - async def ask( agent: Agent, @@ -100,10 +93,9 @@ def _resolve_settings_or_fail(provider: str | None, model: str | None) -> ModelS """Resolve model settings, turning an invalid value into a clean CLI error. ``_resolve_model_settings`` raises pydantic's ``ValidationError`` (a raw - traceback) on a bad ``AIIDA_AGENTS_*`` value. The read-only diagnostics - commands (``check``/``warm``/``doctor``) build settings outside a ``try``, so - this converts it to a ``ClickException`` the way ``_build_agent`` does for its - other configuration errors. + traceback) on a bad ``AIIDA_AGENTS_*`` value. ``doctor`` builds settings + outside a ``try``, so this converts it to a ``ClickException`` the way + ``_build_agent`` does for its other configuration errors. """ try: return _resolve_model_settings(provider, model) @@ -154,8 +146,8 @@ def _probe_model(settings: ModelSettings) -> None: # pragma: no cover A failure surfaces here (before a session), and for a local Ollama model the call loads it into memory so the first real query isn't a cold start. This is - the heavy path behind ``warm``; ``check`` uses :func:`_check_reachable`, which - never generates. + the heavy path behind ``doctor --warm``; the default report uses + :func:`_probe_reachable`, which never generates. """ from aiida_agents.agents._models import get_model @@ -182,7 +174,7 @@ async def _list_model_ids(client: Any) -> set[str]: # ``wait_for`` raises the asyncio one, a distinct class from the builtin # ``TimeoutError`` (they were only merged into aliases in 3.11), so # catching the builtin would let the timeout escape uncaught there. - # Phrase the message so ``_diagnose_probe_failure`` routes it to the + # Phrase the message so ``_probe_failure_hint`` routes it to the # "unreachable" branch (it matches on "connect"). msg = f"could not connect within {_REACHABILITY_TIMEOUT:.0f}s" raise ConnectionError(msg) from exc @@ -203,8 +195,8 @@ def _probe_reachable(settings: ModelSettings) -> _Reachability: Builds the model (validating provider / base_url / key presence), then lists the endpoint's models (one cheap GET under a short timeout). ``model_ok`` is whether the configured model is among those advertised. Raises on an - unreachable endpoint, bad key, or bad config. Shared by ``check`` and - ``doctor``; never loads or runs the model. + unreachable endpoint, bad key, or bad config. Behind ``doctor``'s model + row; never loads or runs the model. """ from aiida_agents.agents._models import get_model @@ -224,8 +216,8 @@ def _probe_reachable(settings: ModelSettings) -> _Reachability: ) -# How a probed model's availability maps to a single policy decision, shared by -# ``check`` and ``doctor`` so the "is it usable?" rule lives in one place. +# How a probed model's availability maps to a single policy decision, kept apart +# from the rendering so the "is it usable?" rule lives in one place. _ModelAvailability: TypeAlias = Literal["available", "not_pulled", "unlisted"] @@ -236,68 +228,45 @@ def _model_availability( Ollama's ``/models`` listing is authoritative, so a model absent from it is genuinely not pulled; a cloud (or openai-compatible) endpoint's listing may be - partial, so an absent model is only "unlisted" and may still work. Callers - render this as they need (``check`` echoes and exits; ``doctor`` builds a row). + partial, so an absent model is only "unlisted" and may still work. ``doctor`` + renders the outcome as a row. """ if reach.model_ok: return "available" return "not_pulled" if provider == "ollama" else "unlisted" -def _check_reachable(settings: ModelSettings) -> None: - """Print ``check``'s reachability report; exit non-zero on a real problem. +def _not_pulled_detail(settings: ModelSettings) -> str: + """The one sentence for "this Ollama model is not installed". - Reachability / auth failures raise for the caller to diagnose. A configured - model the endpoint doesn't advertise is fatal for Ollama (its listing is - authoritative, so it means "not pulled"), but only a warning for cloud or - openai-compatible endpoints, whose ``/models`` listing may be partial. + Reached two ways a user cannot tell apart and should not have to: the + endpoint's listing not carrying the model, and the probe raising a 404 for + it. Both render this, so the report does not word one condition twice. """ - reach = _probe_reachable(settings) - click.echo(f"Endpoint: {reach.endpoint}") - click.echo(f"{_OK} reachable ({reach.n_models} models advertised)") - availability = _model_availability(reach, settings.provider) - if availability == "available": - click.echo(f"{_OK} model '{settings.model}' is available") - return - if availability == "not_pulled": - click.echo(f"{_FAIL} model '{settings.model}' is not pulled.", err=True) - click.echo(f" Pull it with: ollama pull {settings.model}", err=True) - raise SystemExit(1) - if availability == "unlisted": - click.echo( - f"{_WARN} model '{settings.model}' is not in this endpoint's list " - "(it may be partial; the model may still work).", - err=True, - ) - return - assert_never(availability) # pragma: no cover + return f"model not pulled (ollama pull {settings.model})" + +def _probe_failure_hint(settings: ModelSettings, exc: Exception) -> str | None: + """A model-probe failure phrased as what to do about it, or ``None``. -def _diagnose_probe_failure(settings: ModelSettings, exc: Exception) -> None: - """Turn a probe failure into an actionable message, offering an Ollama pull.""" + The three failures a user can actually fix (an unpulled Ollama model, a bad + or absent API key, a server that is not up) each surface as provider-specific + SDK wording that says what broke and never what to do. Classification is by + message text because that is the only thing those exceptions have in common. + + ``None`` means the failure is not one of the three, so the caller should fall + back to the exception's own message rather than print a guess. + """ msg = str(exc).lower() if settings.provider == "ollama" and ("not found" in msg or "404" in msg): - click.echo(f"{_FAIL} Ollama model '{settings.model}' is not pulled.", err=True) - if click.confirm(f"Pull it now (ollama pull {settings.model})?", default=True): - _ollama_pull(settings.model) - return + return _not_pulled_detail(settings) if ("api" in msg and "key" in msg) or "401" in msg or "403" in msg: if "not set" in msg or "environment variable" in msg or "set the" in msg: - click.echo( - f"{_FAIL} API key not set: set the provider's API key.", err=True - ) - else: - click.echo( - f"{_FAIL} Authentication failed: check the provider's API key.", - err=True, - ) - return + return "API key not set; set the provider's API key" + return "authentication failed; check the provider's API key" if "connect" in msg or "connection" in msg: - click.echo( - f"{_FAIL} Could not reach the endpoint. Is the server running?", err=True - ) - return - click.echo(f"{_FAIL} {exc}", err=True) + return "could not reach the endpoint; is the server running?" + return None def _resolve_plan( diff --git a/src/aiida_agents/cli/commands.py b/src/aiida_agents/cli/commands.py index 24dbbf2..25e9508 100644 --- a/src/aiida_agents/cli/commands.py +++ b/src/aiida_agents/cli/commands.py @@ -4,14 +4,12 @@ and the ``doctor`` command live in their own modules and are registered onto the root group at the bottom of this file. What stays here is the root group (the global ``--provider`` / ``--model`` / ``--profile`` overrides and logging -setup) plus the small model-facing commands: ``chat``, ``ask``, ``check``, and -``warm``. +setup) plus the two model-facing commands: ``chat`` and ``ask``. """ from __future__ import annotations import asyncio -import time import rich_click as click from pydantic_ai.tools import DeferredToolRequests @@ -26,7 +24,6 @@ from aiida_agents.cli.mcp import mcp from aiida_agents.cli.output import ( _warn_ungrounded, - _format_duration, _print_reply, _render_tool_calls, console, @@ -40,9 +37,6 @@ _StepResult, _AGENT_CHOICES, _build_agent, - _check_reachable, - _diagnose_probe_failure, - _probe_model, _resolve_settings_or_fail, ask, ) @@ -189,44 +183,6 @@ def ask_cmd(ctx: click.Context, question: str, raw: bool) -> None: ) -@cli.command() -@click.pass_context -@_needs_recognized_settings -def check(ctx: click.Context) -> None: - """Verify config, endpoint reachability, and model availability. - - Fast and read-only: it never loads or runs the model. Use `warm` to - pre-load a local model before an interactive session. - """ - settings = _resolve_settings_or_fail(ctx.obj["provider"], ctx.obj["model"]) - click.echo(f"Checking {settings.provider}:{settings.model} ...") - try: - _check_reachable(settings) - except Exception as exc: - _diagnose_probe_failure(settings, exc) - raise SystemExit(1) from exc - - -@cli.command() -@click.pass_context -@_needs_recognized_settings -def warm(ctx: click.Context) -> None: - """Warm the model with one tiny generation, so the first query isn't cold. - - Mainly useful for a local Ollama model before a session; for a cloud model - it is just a round-trip that also confirms generation works. - """ - settings = _resolve_settings_or_fail(ctx.obj["provider"], ctx.obj["model"]) - click.echo(f"Warming {settings.provider}:{settings.model} ...") - start = time.monotonic() - try: - _probe_model(settings) - except Exception as exc: - _diagnose_probe_failure(settings, exc) - raise SystemExit(1) from exc - click.echo(f"✓ warmed in {_format_duration(time.monotonic() - start)}") - - # Register the commands and groups that live in their own modules onto the root. cli.add_command(config) cli.add_command(sandbox) diff --git a/src/aiida_agents/cli/config.py b/src/aiida_agents/cli/config.py index f5516f5..f1073d6 100644 --- a/src/aiida_agents/cli/config.py +++ b/src/aiida_agents/cli/config.py @@ -114,7 +114,7 @@ def _invalid_fields(exc: ValidationError) -> dict[str, str]: Whole-model errors (a cross-field ``model_validator``, which carry no field location) are omitted; those fields still render as ``(invalid)`` and the - full reason is available from ``aiida-agents check``. + full reason is available from ``aiida-agents doctor``. """ return { str(err["loc"][-1]): err["msg"].removeprefix("Value error, ") diff --git a/src/aiida_agents/cli/doctor.py b/src/aiida_agents/cli/doctor.py index af65f26..859f821 100644 --- a/src/aiida_agents/cli/doctor.py +++ b/src/aiida_agents/cli/doctor.py @@ -1,14 +1,15 @@ """The ``aiida-agents doctor`` command and the health checks behind it. -Each subsystem (AiiDA profile, model reachability, RAG index, codegen sandbox, -docs toolchain) is probed in its own ``try`` so one failure never aborts the -rest of the report. Only ``doctor`` runs these checks (``check``/``warm`` probe the model -through ``agent.py``), so the command lives here with its logic rather than in -``commands.py``. +Each subsystem (AiiDA profile, daemon, model reachability, docs toolchain, RAG +index, codegen sandbox) is probed in its own ``try`` so one failure never +aborts the rest of the report. This is the only command that diagnoses a setup: +the model probes live in ``agent.py`` and are rendered here, so the logic sits +with the command rather than in ``commands.py``. """ from __future__ import annotations +import time from typing import NamedTuple import rich_click as click @@ -19,7 +20,7 @@ from aiida_agents._settings import ModelSettings, _format_validation_error from aiida_agents.cli._guards import _needs_recognized_settings from aiida_agents.cli.agent import _resolve_settings_or_fail -from aiida_agents.cli.output import console +from aiida_agents.cli.output import _format_duration, console from aiida_agents.cli.rag import _module_missing @@ -44,6 +45,20 @@ def _short_reason(exc: Exception) -> str: return lines[0][:100] if lines else "" +def _probe_reason(settings: ModelSettings, exc: Exception) -> str: + """A failed model probe as a row detail: the actionable hint, or the message. + + An unpulled model, a bad key and a dead endpoint are the failures a user can + fix, and :func:`~aiida_agents.cli.agent._probe_failure_hint` phrases those as + the fix. Anything else has no known remedy, so the exception speaks for + itself rather than being dressed up as advice. + """ + from aiida_agents.cli.agent import _probe_failure_hint + + hint = _probe_failure_hint(settings, exc) + return hint if hint is not None else _short_reason(exc) + + def _check_profile(profile: str | None) -> _DiagnosticRow: """Whether the AiiDA profile loads.""" label = "AiiDA profile loads" @@ -56,15 +71,102 @@ def _check_profile(profile: str | None) -> _DiagnosticRow: return _DiagnosticRow(label, False, _short_reason(exc)) +# Written twice: by the check below, and by the skip that stands in for it when +# the profile it needs did not load. Named, so the two cannot drift into looking +# like two different rows. +_DAEMON_LABEL = "Daemon running and reachable" + + +def _check_daemon(profile: str | None) -> _DiagnosticRow: + """Whether the AiiDA daemon for ``profile`` is running and reachable. + + A loading profile is not enough: a submitted process only progresses if the + daemon is up *and* its workers can be reached. The two differ, the circus + endpoint can be briefly unreachable mid-restart while the daemon is + technically alive, so this asks for the worker count to confirm reachability + rather than trusting liveness alone. + + There is one daemon per profile, so the name is passed through rather than + left to whichever profile happens to be loaded: this row and the profile row + above it must be talking about the same profile. ``None`` means the default, + which by then ``_check_profile`` has loaded. + """ + label = _DAEMON_LABEL + # Two ways to reach one condition (see the handler below), so one sentence. + stopped = "not running; run `verdi daemon start`" + try: + from aiida.engine.daemon.client import ( + DaemonNotRunningException, + DaemonStalePidException, + DaemonTimeoutException, + get_daemon_client, + ) + + client = get_daemon_client(profile) + if not client.is_daemon_running: + return _DiagnosticRow(label, False, stopped) + # Running is not reachable. get_numprocesses() round-trips to the daemon, + # so a raise here means up-but-unreachable. + try: + response = client.get_numprocesses() + except DaemonNotRunningException: + # `is_daemon_running` only reads the PID file, so the daemon can stop + # between the two calls. Same condition as the branch above, so it + # gets that branch's sentence rather than a second one for one fault. + return _DiagnosticRow(label, False, stopped) + except DaemonStalePidException: + # A stale PID file is how `is_daemon_running` says yes while the + # round-trip fails, so it is this row's main non-trivial path. Named + # here because AiiDA puts the remedy in a second sentence 100 + # characters in, past where `_short_reason` cuts the detail off. + # `start` alone clears the file (AiiDA: "Either stop or start"). + return _DiagnosticRow( + label, + False, + "stale PID file; run `verdi daemon start`, which clears it", + ) + except DaemonTimeoutException: + return _DiagnosticRow( + label, + False, + "up but not answering; run `verdi daemon restart`", + ) + workers = response.get("numprocesses") + if workers is None: + # circus reports a command-level failure in the payload rather than + # by raising (``circus.commands.base.error``), and that payload + # carries no count. Treating a missing count as "reachable" passed + # the row for a daemon that had just answered with an error, which + # is the one thing this check exists to catch. + reason = str(response.get("reason") or response.get("status") or "no count") + return _DiagnosticRow( + label, + False, + f"answered without a worker count ({reason[:40]}); " + "run `verdi daemon restart`", + ) + if workers == 0: + return _DiagnosticRow( + label, False, "running but 0 workers; run `verdi daemon incr 1`" + ) + return _DiagnosticRow(label, True, f"{workers} worker(s)") + except Exception as exc: + return _DiagnosticRow(label, False, _short_reason(exc)) + + def _check_model(settings: ModelSettings) -> _DiagnosticRow: """Whether the configured model is reachable and advertised (no generation). - Uses the no-generation reachability probe, so ``doctor`` never warms the - model. An unadvertised model is fatal for Ollama (its listing is - authoritative) but only a note for a cloud endpoint (its listing may be - partial). + Uses the no-generation reachability probe, so the default report never + generates; ``--warm`` adds the row that does. An unadvertised model is fatal + for Ollama (its listing is authoritative) but only a note for a cloud + endpoint (its listing may be partial). """ - from aiida_agents.cli.agent import _model_availability, _probe_reachable + from aiida_agents.cli.agent import ( + _model_availability, + _not_pulled_detail, + _probe_reachable, + ) label = f"Model reachable ({settings.provider}:{settings.model})" try: @@ -73,16 +175,43 @@ def _check_model(settings: ModelSettings) -> _DiagnosticRow: if availability == "available": return _DiagnosticRow(label, True, reach.endpoint) if availability == "not_pulled": - return _DiagnosticRow( - label, False, f"model not pulled (ollama pull {settings.model})" - ) + return _DiagnosticRow(label, False, _not_pulled_detail(settings)) if availability == "unlisted": return _DiagnosticRow( label, True, "reachable; model not listed (may still work)" ) assert_never(availability) # pragma: no cover except Exception as exc: - return _DiagnosticRow(label, False, _short_reason(exc)) + return _DiagnosticRow(label, False, _probe_reason(settings, exc)) + + +# Written twice, like the daemon label: by the check below, and by the skip that +# stands in for it when the model is unreachable. +_WARM_LABEL = "Model generates" + + +def _warm_model(settings: ModelSettings) -> _DiagnosticRow: + """Whether the model actually generates, and how long the first call takes. + + Reachability is not generation: an endpoint can advertise a model it then + fails to serve, and only a real call finds that out. For a local Ollama model + the call also loads it into memory, so the first query of a session is not a + cold start. + + The only check that spends tokens, which is why it is behind ``--warm`` + instead of part of the default report: running ``doctor`` against a paid + provider should not cost anything. + """ + from aiida_agents.cli.agent import _probe_model + + label = _WARM_LABEL + start = time.monotonic() + try: + _probe_model(settings) + except Exception as exc: + return _DiagnosticRow(label, False, _probe_reason(settings, exc)) + elapsed = _format_duration(time.monotonic() - start) + return _DiagnosticRow(label, True, f"warmed in {elapsed}") def _check_rag_index() -> _DiagnosticRow: @@ -175,31 +304,79 @@ def _check_docs_toolchain() -> _DiagnosticRow: def _run_diagnostics( - settings: ModelSettings, profile: str | None + settings: ModelSettings, profile: str | None, *, warm: bool = False ) -> list[_DiagnosticRow]: """Run each health check, one :class:`_DiagnosticRow` per check. Each check catches its own failure and reports it as a failed row, so one broken check (an unreachable model, an unloadable profile) never aborts the rest of the report. + + ``warm`` appends the generation check. It is skipped when the model is not + reachable in the first place: the call would fail for the reason already on + the row above it, so the row still appears and says it was not attempted, + rather than repeating that reason or paying for a call that cannot work. + + The checks run in the order they are displayed. Only ``_check_profile`` + calls ``load_profile``, and ``_check_daemon`` and ``_check_sandbox`` need a + loaded profile, so evaluating the rows out of order would break them; the + model row is bound to a name because ``warm`` branches on it, not to move it. """ - return [ - _check_profile(profile), - _check_model(settings), + profile_row = _check_profile(profile) + # A profile that would not load leaves nothing to ask about: the daemon is + # per profile, and probing anyway reported AiiDA's "consider loading a + # profile using `aiida.load_profile()`" at a researcher, as a second red row + # for the one fault the row above already names. + daemon_row = ( + _check_daemon(profile) + if profile_row.ok + else _DiagnosticRow( + _DAEMON_LABEL, False, "not attempted; the profile did not load" + ) + ) + model = _check_model(settings) + rows = [ + profile_row, + daemon_row, + model, + # Before the index it builds: with neither present, a report that asks + # for `rag build` first sends the user at a command that cannot run yet. + _check_docs_toolchain(), _check_rag_index(), _check_sandbox(), - _check_docs_toolchain(), ] + if warm: + rows.append( + _warm_model(settings) + if model.ok + else _DiagnosticRow( + _WARM_LABEL, False, "not attempted; the model is unreachable" + ) + ) + return rows @click.command() +@click.option( + "--warm", + is_flag=True, + help=( + "Also send one tiny generation, which proves the model serves and " + "pre-loads a local Ollama model so the first query is not a cold start. " + "Costs a request on a paid provider, so it is off by default." + ), +) @click.pass_context @_needs_recognized_settings -def doctor(ctx: click.Context) -> None: - """Diagnose the setup: profile, model, RAG index, sandbox, docs toolchain.""" +def doctor(ctx: click.Context, warm: bool) -> None: + """Diagnose the setup: profile, daemon, model, docs toolchain, RAG index, sandbox. + + Read-only and free by default: nothing here generates. Pass `--warm` to + also check that the model serves, and to pre-load a local one. + """ settings = _resolve_settings_or_fail(ctx.obj["provider"], ctx.obj["model"]) click.echo("Running diagnostics ...\n") - rows = _run_diagnostics(settings, ctx.obj["profile"]) + rows = _run_diagnostics(settings, ctx.obj["profile"], warm=warm) for row in rows: # escape() the dynamic label/detail (a model name, or an error message # from _short_reason) so a stray bracket can't be swallowed as Rich diff --git a/tests/cli/test_agent.py b/tests/cli/test_agent.py index c994960..66ad8fb 100644 --- a/tests/cli/test_agent.py +++ b/tests/cli/test_agent.py @@ -5,7 +5,6 @@ import asyncio import pytest -import rich_click from click.testing import CliRunner from aiida_agents._settings import _Provider @@ -152,109 +151,71 @@ def __init__(self) -> None: pytest.param( "openrouter", "API key not set for the provider", - "API key not set", + "API key not set; set the provider's API key", id="key-not-set", ), pytest.param( - "openrouter", "401 Unauthorized", "Authentication failed", id="auth-failed" + "openrouter", + "401 Unauthorized", + "authentication failed; check the provider's API key", + id="auth-failed", ), pytest.param( "ollama", "connection refused", - "Could not reach the endpoint", + "could not reach the endpoint; is the server running?", id="unreachable", ), pytest.param( - "openrouter", "something odd", "something odd", id="generic-passthrough" + "ollama", + "model 'qwen3' not found (status 404)", + "model not pulled (ollama pull qwen3)", + id="ollama-not-pulled", ), + pytest.param("openrouter", "something odd", None, id="unrecognised-is-none"), ], ) -def test_diagnose_probe_failure_routes_message( - capsys: pytest.CaptureFixture[str], provider: _Provider, message: str, expected: str +def test_probe_failure_hint_routes_message( + provider: _Provider, message: str, expected: str | None ) -> None: - """A probe failure is turned into an actionable message routed by its text: - a missing key, an auth failure, an unreachable endpoint, or a passed-through - fallback for anything unrecognised. - """ - from aiida_agents._settings import ModelSettings - from aiida_agents.cli.agent import _diagnose_probe_failure - - _diagnose_probe_failure( - ModelSettings(provider=provider, model="m"), RuntimeError(message) - ) - assert expected in capsys.readouterr().err + """A probe failure the user can fix is phrased as the fix, routed by its text. - -@pytest.mark.parametrize("accept, pulled", [(True, ["qwen3"]), (False, [])]) -def test_diagnose_probe_failure_offers_ollama_pull( - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], - accept: bool, - pulled: list[str], -) -> None: - """A missing Ollama model offers a pull; accepting runs it, declining does not.""" + ``None`` for anything unrecognised, so the caller falls back to the + exception's own words instead of inventing advice for a failure nobody + classified. + """ from aiida_agents._settings import ModelSettings - from aiida_agents.cli import agent - - ran: list[str] = [] - monkeypatch.setattr(rich_click, "confirm", lambda *a, **k: accept) - monkeypatch.setattr(agent, "_ollama_pull", lambda model: ran.append(model)) + from aiida_agents.cli.agent import _probe_failure_hint - agent._diagnose_probe_failure( - ModelSettings(provider="ollama", model="qwen3"), - RuntimeError("model 'qwen3' not found (status 404)"), + hint = _probe_failure_hint( + ModelSettings(provider=provider, model="qwen3"), RuntimeError(message) ) - assert ran == pulled - assert "is not pulled" in capsys.readouterr().err + assert hint == expected @pytest.mark.parametrize( - "provider, model_ok, raises, expected", + "provider, model_ok, expected", [ - pytest.param("ollama", True, False, "is available", id="available"), - pytest.param("ollama", False, True, "is not pulled", id="ollama-missing-fatal"), - pytest.param( - "openrouter", - False, - False, - "not in this endpoint's list", - id="cloud-unlisted-warns", - ), + pytest.param("ollama", True, "available", id="available"), + pytest.param("ollama", False, "not_pulled", id="ollama-missing-is-fatal"), + pytest.param("openrouter", False, "unlisted", id="cloud-missing-is-a-note"), ], ) -def test_check_reachable_availability_policy( - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], - provider: _Provider, - model_ok: bool, - raises: bool, - expected: str, +def test_model_availability_policy( + provider: _Provider, model_ok: bool, expected: str ) -> None: - """A model the endpoint doesn't advertise is fatal for Ollama (its listing is - authoritative) but only a warning for a cloud endpoint (its listing may be - partial); an advertised model just reports available. + """A model the endpoint doesn't advertise is "not pulled" for Ollama (its + listing is authoritative) but only "unlisted" for a cloud endpoint (whose + listing may be partial); an advertised model is available either way. """ - from aiida_agents._settings import ModelSettings - from aiida_agents.cli import agent - from aiida_agents.cli.agent import _Reachability + from aiida_agents.cli.agent import _model_availability, _Reachability - monkeypatch.setattr( - agent, - "_probe_reachable", - lambda settings: _Reachability("http://endpoint", 2, model_ok), + availability = _model_availability( + _Reachability("http://endpoint", 2, model_ok), provider ) - settings = ModelSettings(provider=provider, model="m") - - if raises: - with pytest.raises(SystemExit) as exc_info: - agent._check_reachable(settings) - assert exc_info.value.code == 1 - else: - agent._check_reachable(settings) - captured = capsys.readouterr() - assert expected in captured.out + captured.err + assert availability == expected @pytest.mark.parametrize("name", _SPECIALISTS) diff --git a/tests/cli/test_commands.py b/tests/cli/test_commands.py index afc40c1..9930608 100644 --- a/tests/cli/test_commands.py +++ b/tests/cli/test_commands.py @@ -14,7 +14,7 @@ def test_cli_exposes_expected_commands() -> None: """The top-level group lists every subcommand we ship.""" result = CliRunner().invoke(cli, ["--help"]) assert result.exit_code == 0 - for command in ("ask", "chat", "check", "config", "doctor", "mcp", "rag", "warm"): + for command in ("ask", "chat", "config", "doctor", "mcp", "rag"): assert command in result.output @@ -34,35 +34,6 @@ def test_dash_h_is_a_help_alias(args: list[str]) -> None: assert "Show this message and exit" in result.output -@pytest.mark.parametrize( - "command, expected_calls", - [ - pytest.param("check", ["reachable"], id="check-probes-never-generates"), - pytest.param("warm", ["generate"], id="warm-generates"), - ], -) -def test_check_and_warm_use_distinct_probes( - monkeypatch: pytest.MonkeyPatch, command: str, expected_calls: list[str] -) -> None: - """`check` probes reachability and never generates; `warm` runs the generation - probe that loads the model. Pins the deliberate split (a check stays cheap and - side-effect-free): both probes are stubbed, so the recorded calls prove which - one each command drives. - """ - calls: list[str] = [] - monkeypatch.setattr( - "aiida_agents.cli.commands._check_reachable", - lambda settings: calls.append("reachable"), - ) - monkeypatch.setattr( - "aiida_agents.cli.commands._probe_model", - lambda settings: calls.append("generate"), - ) - result = CliRunner().invoke(cli, [command]) - assert result.exit_code == 0 - assert calls == expected_calls - - def test_version_option_prints_version() -> None: """`--version` reports the installed package version and exits cleanly.""" from importlib.metadata import version @@ -72,19 +43,19 @@ def test_version_option_prints_version() -> None: assert version("aiida-agents") in result.output -def test_check_reports_invalid_setting_value_cleanly( +def test_doctor_reports_invalid_setting_value_cleanly( monkeypatch: pytest.MonkeyPatch, ) -> None: """A bad setting *value* is a clean CLI error, not a raw pydantic traceback. - Regression: `check`/`warm`/`doctor` build settings outside their try, so an - invalid `AIIDA_AGENTS_*` value used to surface as an uncaught - `ValidationError`; now `_resolve_settings_or_fail` converts it. + Regression: `doctor` builds settings outside its try, so an invalid + `AIIDA_AGENTS_*` value used to surface as an uncaught `ValidationError`; + now `_resolve_settings_or_fail` converts it. """ from pydantic import ValidationError monkeypatch.setenv("AIIDA_AGENTS_PROVIDER", "bogus") - result = CliRunner().invoke(cli, ["check"]) + result = CliRunner().invoke(cli, ["doctor"]) assert result.exit_code == 1 assert "Invalid configuration" in result.output assert "provider" in result.output @@ -96,44 +67,12 @@ def test_provider_flag_rejects_unknown_choice() -> None: bad value is a clean 'invalid choice' usage error listing the real providers (complements the env-var path above, which pydantic validates). """ - result = CliRunner().invoke(cli, ["--provider", "bogus", "check"]) + result = CliRunner().invoke(cli, ["--provider", "bogus", "doctor"]) assert result.exit_code == 2 # Click usage error, before any work assert "bogus" in result.output assert "ollama" in result.output # the derived choices are listed -@pytest.mark.parametrize( - "command, probe", - [ - pytest.param("check", "_check_reachable", id="check"), - pytest.param("warm", "_probe_model", id="warm"), - ], -) -def test_probe_failure_is_diagnosed_and_exits( - monkeypatch: pytest.MonkeyPatch, command: str, probe: str -) -> None: - """A probe failure routes through `_diagnose_probe_failure` and exits 1, - never a raw traceback. - """ - from aiida_agents.cli import commands - - def _boom(settings: object) -> None: - raise RuntimeError("endpoint down") - - diagnosed: list[str] = [] - monkeypatch.setattr(f"aiida_agents.cli.commands.{probe}", _boom) - monkeypatch.setattr( - commands, - "_diagnose_probe_failure", - lambda settings, exc: diagnosed.append(str(exc)), - ) - - result = CliRunner().invoke(cli, [command]) - - assert result.exit_code == 1 - assert diagnosed == ["endpoint down"] - - class _FakeResult: def __init__(self, output: object) -> None: self.output = output diff --git a/tests/cli/test_doctor.py b/tests/cli/test_doctor.py index 3f221ba..7ae2b68 100644 --- a/tests/cli/test_doctor.py +++ b/tests/cli/test_doctor.py @@ -7,6 +7,12 @@ import pytest from click.testing import CliRunner +from aiida.engine.daemon.client import ( + DaemonNotRunningException, + DaemonStalePidException, + DaemonTimeoutException, +) + from aiida_agents._settings import ModelSettings, _Provider from aiida_agents.cli import cli from aiida_agents.cli.doctor import _DiagnosticRow @@ -46,6 +52,49 @@ def __init__(self, built: bool, stale: tuple[object, ...] = ()) -> None: self.stale = stale +# Verbatim from ``aiida.engine.daemon.client.call_client``. 162 characters, with +# the remedy in the second sentence, so ``_short_reason``'s 100-char cut lands +# mid-word and drops it. Copied rather than imported because AiiDA builds it +# inline at the raise site. +_STALE_PID_MSG = ( + "The daemon could not be reached, seemingly because of a stale PID file. " + "Either stop or start the daemon to remove it and restore the daemon to a " + "functional state." +) + + +class _DaemonClient: + """Stub AiiDA daemon client exposing what the doctor check reads.""" + + def __init__( + self, + *, + running: bool = True, + workers: int = 1, + raises: Exception | None = None, + response: dict[str, object] | None = None, + ) -> None: + self._running = running + self._workers = workers + self._raises = raises + self._response = response + + @property + def is_daemon_running(self) -> bool: + return self._running + + # Widened from AiiDA's own `dict[str, t.Any]`: a case returns an error + # payload, which carries no int at all. + def get_numprocesses(self) -> dict[str, object]: + # Running is not reachable: mid-restart the circus endpoint can be down + # while the daemon is alive, which the check reports as a failed row. + if self._raises is not None: + raise self._raises + if self._response is not None: + return self._response + return {"numprocesses": self._workers} + + def _patch_all_checks_passing(monkeypatch: pytest.MonkeyPatch) -> None: """Stub every subsystem probe to succeed, so a test can then break exactly one and assert the others are unaffected. @@ -54,6 +103,10 @@ def _patch_all_checks_passing(monkeypatch: pytest.MonkeyPatch) -> None: from aiida_agents.cli.agent import _Reachability monkeypatch.setattr("aiida.load_profile", lambda profile: _Profile()) + monkeypatch.setattr( + "aiida.engine.daemon.client.get_daemon_client", + lambda profile_name=None: _DaemonClient(), + ) monkeypatch.setattr( "aiida_agents.cli.agent._probe_reachable", lambda settings: _Reachability("http://endpoint", 3, model_ok=True), @@ -66,11 +119,13 @@ def _patch_all_checks_passing(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(doctor, "_module_missing", lambda name: False) -def _rows_by_label(profile: str | None = None) -> dict[str, _DiagnosticRow]: +def _rows_by_label( + profile: str | None = None, *, warm: bool = False +) -> dict[str, _DiagnosticRow]: from aiida_agents.cli.doctor import _run_diagnostics settings = ModelSettings(provider="ollama", model="m") - return {row.label: row for row in _run_diagnostics(settings, profile)} + return {row.label: row for row in _run_diagnostics(settings, profile, warm=warm)} @pytest.mark.parametrize( @@ -101,38 +156,152 @@ def test_run_diagnostics_all_checks_pass(monkeypatch: pytest.MonkeyPatch) -> Non assert list(rows) == [ "AiiDA profile loads", + "Daemon running and reachable", "Model reachable (ollama:m)", + # The docs toolchain precedes the index it builds: `rag build` needs + # sphinx, so a user missing both should read the cause before the effect. + "Docs toolchain (sphinx)", "RAG index built", "Codegen sandbox (disposable copy)", - "Docs toolchain (sphinx)", ] assert all(row.ok for row in rows.values()) @pytest.mark.parametrize( - "target, failing_label", + "client, expected_ok, needle", + [ + pytest.param( + _DaemonClient(running=True, workers=2), True, "2 worker(s)", id="reachable" + ), + pytest.param( + _DaemonClient(running=False), False, "verdi daemon start", id="not-running" + ), + pytest.param( + _DaemonClient(running=True, workers=0), + False, + "verdi daemon incr", + id="zero-workers", + ), + pytest.param( + _DaemonClient(running=True, raises=RuntimeError("circus unreachable")), + False, + "circus unreachable", + id="running-but-unreachable", + ), + # AiiDA's own exception types carrying AiiDA's own wording, because what + # is being pinned is that the remedy survives the row. `is_daemon_running` + # reads the PID file, so a stale one is precisely how it says yes while + # the round-trip fails: this row's main real failure, not an edge case. + pytest.param( + _DaemonClient(running=True, raises=DaemonStalePidException(_STALE_PID_MSG)), + False, + "stale PID file; run `verdi daemon start`", + id="stale-pid-file", + ), + # One condition, one sentence: the daemon can stop between the PID-file + # read and the round-trip, and that must not become a second wording. + pytest.param( + _DaemonClient( + running=True, + raises=DaemonNotRunningException("The daemon is not running."), + ), + False, + "not running; run `verdi daemon start`", + id="stopped-mid-check", + ), + pytest.param( + _DaemonClient( + running=True, + raises=DaemonTimeoutException("Connection to the daemon timed out."), + ), + False, + "verdi daemon restart", + id="timed-out", + ), + # circus answers a command-level failure in the payload instead of + # raising (circus.commands.base.error), and that payload has no + # `numprocesses`. Reading the count off it finds nothing, which must not + # read as a healthy daemon. Caught by CodeRabbit on the PR. + pytest.param( + _DaemonClient( + running=True, + response={"status": "error", "reason": "command not supported"}, + ), + False, + "command not supported", + id="error-payload", + ), + pytest.param( + _DaemonClient(running=True, response={"status": "ok"}), + False, + "verdi daemon restart", + id="no-worker-count", + ), + ], +) +def test_check_daemon( + monkeypatch: pytest.MonkeyPatch, + client: _DaemonClient, + expected_ok: bool, + needle: str, +) -> None: + """The daemon check separates running-and-reachable from every way it is not: + stopped, zero workers, or up but with an unreachable circus endpoint. + """ + from aiida_agents.cli.doctor import _check_daemon + + monkeypatch.setattr( + "aiida.engine.daemon.client.get_daemon_client", + lambda profile_name=None: client, + ) + row = _check_daemon(None) + + assert row.ok is expected_ok + assert needle in row.detail + # AiiDA phrases the stale-PID remedy in a second sentence, 162 characters in, + # so routing it through `_short_reason` (which truncates at 100) used to cut + # the row off mid-word with the fix still to come. + assert len(row.detail) < 100 + + +@pytest.mark.parametrize( + "target, failing_label, dependent", [ - pytest.param("aiida.load_profile", "AiiDA profile loads", id="profile"), + # The daemon is per profile, so it is the one row a failed profile takes + # with it. Named rather than tolerated, so a future check that quietly + # starts depending on another shows up here. + pytest.param( + "aiida.load_profile", + "AiiDA profile loads", + ("Daemon running and reachable",), + id="profile", + ), pytest.param( "aiida_agents.cli.agent._probe_reachable", "Model reachable (ollama:m)", + (), id="model", ), pytest.param( - "aiida_agents.rag.store.index_status", "RAG index built", id="rag" + "aiida_agents.rag.store.index_status", "RAG index built", (), id="rag" ), pytest.param( "aiida_agents.sandbox.copy.profiles_sharing_storage", "Codegen sandbox (disposable copy)", + (), id="sandbox", ), ], ) def test_run_diagnostics_isolates_one_failure( - monkeypatch: pytest.MonkeyPatch, target: str, failing_label: str + monkeypatch: pytest.MonkeyPatch, + target: str, + failing_label: str, + dependent: tuple[str, ...], ) -> None: """One failing check yields a failed row carrying the reason, and never - aborts the report: every other check still runs and passes. + aborts the report: every check that does not depend on it still runs and + passes, and every one that does says so rather than inventing a reason. """ _patch_all_checks_passing(monkeypatch) @@ -144,9 +313,13 @@ def _boom(*args: object, **kwargs: object) -> object: assert rows[failing_label].ok is False assert "kaboom" in rows[failing_label].detail + for label in dependent: + assert rows[label].ok is False + assert rows[label].detail.startswith("not attempted;") + assert "kaboom" not in rows[label].detail + expected_failures = {failing_label, *dependent} for label, row in rows.items(): - if label != failing_label: - assert row.ok is True + assert row.ok is (label not in expected_failures), label @pytest.mark.parametrize( @@ -254,7 +427,7 @@ def test_doctor_exit_code_reflects_health( monkeypatch.setattr( doctor, "_run_diagnostics", - lambda settings, profile: [_DiagnosticRow(*row) for row in rows], + lambda settings, profile, *, warm=False: [_DiagnosticRow(*row) for row in rows], ) result = CliRunner().invoke(cli, ["doctor"]) @@ -324,3 +497,168 @@ def test_an_invalid_sandbox_setting_names_the_setting( assert "snippet_timeout" in row.detail assert "less than or equal to 300" in row.detail assert "\n" not in row.detail # one table cell, one line + + +class TestWarm: + """``--warm`` is the old ``warm`` command, folded in as one more row.""" + + def test_the_default_report_never_generates( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """`doctor` is free to run: no row spends a token unless asked. + + The old `check`/`warm` split existed to keep a diagnostic cheap, and + folding both into `doctor` has to keep that property or every run bills + a generation against a paid provider. + """ + _patch_all_checks_passing(monkeypatch) + generated: list[object] = [] + monkeypatch.setattr("aiida_agents.cli.agent._probe_model", generated.append) + + result = CliRunner().invoke(cli, ["doctor"]) + + assert result.exit_code == 0 + assert generated == [] + assert "Model generates" not in result.output + + def test_warm_generates_and_reports_the_duration( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """`--warm` runs the generation probe and times it.""" + _patch_all_checks_passing(monkeypatch) + generated: list[object] = [] + monkeypatch.setattr("aiida_agents.cli.agent._probe_model", generated.append) + + result = CliRunner().invoke(cli, ["doctor", "--warm"]) + + assert result.exit_code == 0 + assert len(generated) == 1 + assert "Model generates" in result.output + assert "warmed in" in result.output + + def test_a_failed_generation_is_a_failed_row_not_a_traceback( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """An endpoint that advertises a model it cannot serve fails one row. + + Reachability said yes and generation said no, which is the whole reason + `--warm` exists; it must still report, and still exit 1. + + A serving failure rather than a connection one on purpose: a connection + error would route through `_probe_failure_hint`, which + `test_probe_failure_hint_routes_message` already covers exhaustively. + This is the only test that takes the `_short_reason` fallback all the way + to a rendered row. + """ + _patch_all_checks_passing(monkeypatch) + + def _boom(settings: ModelSettings) -> None: + raise RuntimeError("model runner has unexpectedly stopped") + + monkeypatch.setattr("aiida_agents.cli.agent._probe_model", _boom) + + result = CliRunner().invoke(cli, ["doctor", "--warm"]) + + assert result.exit_code == 1 + assert "model runner has unexpectedly stopped" in result.output + # Not redundant with the exit code: an escaping RuntimeError also exits + # 1, and this is what tells the two apart. + assert result.exception is None or isinstance(result.exception, SystemExit) + + def test_an_unreachable_model_is_not_warmed( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """One problem prints as one red row, not two. + + Warming a model the reachability row already failed on would fail again + for the same reason, and a report that says the same thing twice reads + as two separate faults. + """ + _patch_all_checks_passing(monkeypatch) + + def _unreachable(settings: ModelSettings) -> None: + raise RuntimeError("connection refused") + + generated: list[object] = [] + monkeypatch.setattr("aiida_agents.cli.agent._probe_reachable", _unreachable) + monkeypatch.setattr("aiida_agents.cli.agent._probe_model", generated.append) + + rows = _rows_by_label(warm=True) + + assert generated == [] + assert rows["Model generates"].ok is False + assert rows["Model generates"].detail == ( + "not attempted; the model is unreachable" + ) + + +def test_the_daemon_is_not_probed_when_the_profile_did_not_load( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A profile that will not load makes the daemon row unanswerable, not red. + + The daemon is per profile: ``get_daemon_client()`` resolves through the + *loaded* one. With no profile loaded it raises "Could not determine the + current profile. Consider loading a profile using `aiida.load_profile()`", + which told a researcher to call a Python API and made one fault print as two + red rows, the second of them about something they never asked about. + """ + from aiida_agents.cli.doctor import _DAEMON_LABEL + + _patch_all_checks_passing(monkeypatch) + probed: list[object] = [] + + def _no_such_profile(profile: str | None) -> object: + msg = "profile `nope` does not exist" + raise ValueError(msg) + + def _record(profile_name: str | None = None) -> _DaemonClient: + probed.append(profile_name) + return _DaemonClient() + + monkeypatch.setattr("aiida.load_profile", _no_such_profile) + monkeypatch.setattr("aiida.engine.daemon.client.get_daemon_client", _record) + + rows = _rows_by_label() + + assert probed == [] + assert rows[_DAEMON_LABEL].ok is False + assert rows[_DAEMON_LABEL].detail == "not attempted; the profile did not load" + + +@pytest.mark.parametrize( + "message, expected", + [ + pytest.param( + "model 'm' not found (status 404)", + "model not pulled (ollama pull m)", + id="not-pulled", + ), + pytest.param( + "401 Unauthorized", + "authentication failed; check the provider's API key", + id="bad-key", + ), + pytest.param("some novel failure", "some novel failure", id="falls-back"), + ], +) +def test_a_failed_model_probe_says_what_to_do( + monkeypatch: pytest.MonkeyPatch, message: str, expected: str +) -> None: + """The model row names the fix, not just the provider SDK's wording. + + This is what the removed `check` command used to print through + `_diagnose_probe_failure`; folding it into the row keeps the advice and drops + the interactive prompt, which has no place in a report. + """ + from aiida_agents.cli.doctor import _check_model + + def _boom(settings: ModelSettings) -> None: + raise RuntimeError(message) + + monkeypatch.setattr("aiida_agents.cli.agent._probe_reachable", _boom) + + row = _check_model(ModelSettings(provider="ollama", model="m")) + + assert row.ok is False + assert row.detail == expected diff --git a/tests/cli/test_guards.py b/tests/cli/test_guards.py index 4409591..8c3979d 100644 --- a/tests/cli/test_guards.py +++ b/tests/cli/test_guards.py @@ -29,7 +29,7 @@ def test_action_command_fails_fast_on_mistyped_setting( @pytest.mark.parametrize( "args", [ - pytest.param(["check", "--help"], id="command-help"), + pytest.param(["doctor", "--help"], id="command-help"), pytest.param(["rag", "--help"], id="group-help"), pytest.param(["rag", "status", "-h"], id="subcommand-h"), ],