feat(fit-params): --fit-print-plan emits per-device byte plan as JSON (#66 step 2 prep)

common/arg.cpp

@@ -2470,6 +2470,15 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             }
         }
     ).set_examples({LLAMA_EXAMPLE_FIT_PARAMS}).set_env("LLAMA_ARG_FIT_ESTIMATE"));
+    add_opt(common_arg(
+        { "--fit-print-plan" },
+        "print the resolved per-device byte plan from common_fit_params as JSON on stdout, "
+        "then exit. Designed for the router's admit-side dry-run (see #66); see also out_bytes_per_device "
+        "in common/fit.h. Mutually exclusive with --fit-print; if both are set, --fit-print-plan wins.",
+        [](common_params & params) {
+            params.fit_params_print_plan = true;
+        }
+    ).set_examples({LLAMA_EXAMPLE_FIT_PARAMS}).set_env("LLAMA_ARG_FIT_PRINT_PLAN"));
     add_opt(common_arg(
         { "-fitt", "--fit-target" }, "MiB0,MiB1,MiB2,...",
         string_format("target margin per device for --fit, comma-separated list of values, "

common/common.h

@@ -453,9 +453,10 @@ struct common_params {
     int32_t n_gpu_layers       = -1;    // number of layers to store in VRAM, -1 is auto, <= -2 is all
     int32_t main_gpu           = 0;     // the GPU that is used for scratch and small tensors
     float   tensor_split[128]  = {0};   // how split tensors should be distributed across GPUs
-    bool    fit_params         = true;  // whether to fit unset model/context parameters to free device memory
-    bool    fit_params_print   = false; // print the estimated required memory to run the model
-    int32_t fit_params_min_ctx = 4096;  // minimum context size to set when trying to reduce memory use
+    bool    fit_params           = true;  // whether to fit unset model/context parameters to free device memory
+    bool    fit_params_print     = false; // print the estimated required memory to run the model
+    bool    fit_params_print_plan = false; // print resolved per-device byte plan as JSON (router admit-side input; see #66)
+    int32_t fit_params_min_ctx   = 4096;  // minimum context size to set when trying to reduce memory use
 
     // margin per device in bytes for fitting parameters to free memory:
     std::vector<size_t> fit_params_target = std::vector<size_t>(llama_max_devices(), 1024 * 1024*1024);

common/fit.cpp

@@ -246,6 +246,9 @@ static void common_params_fit_impl(
             if (projected_free_per_device[0] >= margins[0]) {
                 LOG_TRC("%s: will leave %" PRId64 " >= %" PRId64 " MiB of free device memory, no changes needed\n",
                     __func__, projected_free_per_device[0]/MiB, margins[0]/MiB);
+                if (out_bytes_per_device) {
+                    out_bytes_per_device->assign({(int64_t) dmds_full[0].mb.total()});
+                }
                 return;
             }
         } else {
@@ -258,6 +261,13 @@ static void common_params_fit_impl(
             }
             if (!changes_needed) {
                 LOG_TRC("%s: targets for free memory can be met on all devices, no changes needed\n", __func__);
+                if (out_bytes_per_device) {
+                    out_bytes_per_device->clear();
+                    out_bytes_per_device->reserve(nd);
+                    for (size_t id = 0; id < nd; id++) {
+                        out_bytes_per_device->push_back((int64_t) dmds_full[id].mb.total());
+                    }
+                }
                 return;
             }
         }

common/fit.h

@@ -18,11 +18,14 @@ enum common_params_fit_status {
 //   - this function is NOT thread safe because it modifies the global llama logger state
 //   - only parameters that have the same value as in llama_default_model_params are modified
 //     with the exception of the context size which is modified if and only if equal to 0
-//   - if `out_bytes_per_device` is non-null, it is resized to the device-count and populated
-//     with the projected per-device byte demand for the resolved plan. Index 0 is the CPU
-//     device, indices 1..N are GPU/accel devices in the same order as `tensor_split`. The
-//     router uses this to admit candidates against per-device free-after-reserved memory
-//     instead of total-pool memory (see ht-llama.cpp issue #66). Populated on SUCCESS only.
+//   - if `out_bytes_per_device` is non-null, it is resized to the number of GPU/accel
+//     devices and populated with the projected per-device byte demand for the resolved
+//     plan. plan[i] is the i-th GPU/accel device in the same order as `tensor_split`;
+//     CPU/host memory is NOT included (the router only needs per-GPU demand for admit).
+//     For CPU-only builds (no GPU devices), the plan is empty — treat as trivially
+//     admittable. Populated on SUCCESS only — undefined on FAILURE/ERROR.
+//     The router uses this to admit candidates against per-device free-after-reserved
+//     memory instead of total-pool memory (see ht-llama.cpp issue #66).
 enum common_params_fit_status common_fit_params(
                                const char   * path_model,
                 struct llama_model_params   * mparams,

tools/fit-params/fit-params.cpp

@@ -30,6 +30,38 @@ int llama_fit_params(int argc, char ** argv) {
     auto mparams = common_model_params_to_llama(params);
     auto cparams = common_context_params_to_llama(params);
 
+    if (params.fit_params_print_plan) {
+        // Router admit-side dry-run (#66). Run common_fit_params with the per-device
+        // byte plan output, then emit single-line JSON on stdout:
+        //   {"per_device_bytes":[N0,N1,...],"n_devices":K,"total_bytes":T}
+        // Index 0 is the CPU device; indices 1..N-1 are GPU/accel devices in the
+        // same order as tensor_split. Caller subprocess can parse this without
+        // depending on llama internals.
+        std::vector<int64_t> plan;
+        const common_params_fit_status status = common_fit_params(params.model.path.c_str(), &mparams, &cparams,
+                params.tensor_split, params.tensor_buft_overrides.data(), params.fit_params_target.data(), params.fit_params_min_ctx,
+                params.verbosity >= LOG_LEVEL_DEBUG ? GGML_LOG_LEVEL_DEBUG : GGML_LOG_LEVEL_ERROR,
+                &plan);
+        if (status != COMMON_PARAMS_FIT_STATUS_SUCCESS) {
+            LOG_ERR("%s: failed to fit CLI arguments to free memory, exiting...\n", __func__);
+            // Emit a clean JSON failure marker on stdout so subprocess callers can
+            // distinguish fit-failure from parse-failure.
+            printf("{\"error\":\"fit_failed\",\"status\":%d}\n", int(status));
+            return 1;
+        }
+        common_log_flush(common_log_main());
+
+        int64_t total = 0;
+        for (int64_t b : plan) total += b;
+
+        printf("{\"per_device_bytes\":[");
+        for (size_t i = 0; i < plan.size(); ++i) {
+            printf("%s%" PRId64, i == 0 ? "" : ",", plan[i]);
+        }
+        printf("],\"n_devices\":%zu,\"total_bytes\":%" PRId64 "}\n", plan.size(), total);
+        return 0;
+    }
+
     if (!params.fit_params_print) {
         const common_params_fit_status status = common_fit_params(params.model.path.c_str(), &mparams, &cparams,
                 params.tensor_split, params.tensor_buft_overrides.data(), params.fit_params_target.data(), params.fit_params_min_ctx,

tools/fit-params/tests.sh

@@ -0,0 +1,124 @@
+#!/usr/bin/env bash
+#
+# Smoke tests for llama-fit-params, in particular the --fit-print-plan
+# JSON output mode (ht/#66 step 2 prep, PR #72). Verifies:
+#   1) success path emits a single-line JSON with the documented schema
+#   2) plan vector size matches n_devices
+#   3) total_bytes equals sum of per_device_bytes
+#   4) plan is empty on a CPU-only build (no GPU/accel devices)
+#
+# Pattern lifted from tools/gguf-split/tests.sh — invoke from CI with
+# the build's bin/ dir as $1.
+
+set -eu
+
+if [ $# -lt 1 ]
+then
+    echo "usage:   $0 path_to_build_binary [path_to_temp_folder]"
+    echo "example: $0 ../../build/bin ../../tmp"
+    exit 1
+fi
+
+if [ $# -gt 1 ]
+then
+    TMP_DIR=$2
+else
+    TMP_DIR=/tmp
+fi
+
+set -x
+
+FIT_PARAMS=$1/llama-fit-params
+WORK_PATH=$TMP_DIR/fit-params-test
+ROOT_DIR=$(realpath "$(dirname "$0")"/../../)
+
+mkdir -p "$WORK_PATH"
+
+# 1. Get a small model (same one gguf-split tests use, already in CI cache after that runs)
+(
+cd "$WORK_PATH"
+"$ROOT_DIR"/scripts/hf.sh --repo ggml-org/Qwen3-0.6B-GGUF --file Qwen3-0.6B-Q8_0.gguf
+)
+MODEL="$WORK_PATH/Qwen3-0.6B-Q8_0.gguf"
+echo PASS
+
+# 2. --fit-print-plan emits valid single-line JSON on success.
+JSON=$("$FIT_PARAMS" --fit-print-plan -m "$MODEL" 2>/dev/null | tail -1)
+echo "got: $JSON"
+echo "$JSON" | python3 -c "
+import json, sys
+o = json.loads(sys.stdin.read())
+assert 'per_device_bytes' in o, 'missing per_device_bytes'
+assert 'n_devices' in o, 'missing n_devices'
+assert 'total_bytes' in o, 'missing total_bytes'
+assert isinstance(o['per_device_bytes'], list), 'per_device_bytes not a list'
+assert isinstance(o['n_devices'], int), 'n_devices not an int'
+assert isinstance(o['total_bytes'], int), 'total_bytes not an int'
+print('schema OK')
+"
+echo PASS
+
+# 3. plan vector size matches n_devices.
+echo "$JSON" | python3 -c "
+import json, sys
+o = json.loads(sys.stdin.read())
+assert len(o['per_device_bytes']) == o['n_devices'], \
+    f'len(per_device_bytes)={len(o[\"per_device_bytes\"])} != n_devices={o[\"n_devices\"]}'
+print('size invariant OK')
+"
+echo PASS
+
+# 4. total_bytes equals sum of per_device_bytes.
+echo "$JSON" | python3 -c "
+import json, sys
+o = json.loads(sys.stdin.read())
+expected = sum(o['per_device_bytes'])
+assert o['total_bytes'] == expected, f'total_bytes={o[\"total_bytes\"]} != sum={expected}'
+print('total invariant OK')
+"
+echo PASS
+
+# 5. On a CPU-only build, plan is empty (no GPU/accel devices).
+#    Skip the assertion when running under a CUDA/Metal/Vulkan build —
+#    only assert the invariant on CPU-only builds (detected by n_devices==0).
+echo "$JSON" | python3 -c "
+import json, sys
+o = json.loads(sys.stdin.read())
+if o['n_devices'] == 0:
+    assert o['per_device_bytes'] == [], 'plan must be empty when n_devices==0'
+    assert o['total_bytes'] == 0, 'total must be 0 when n_devices==0'
+    print('CPU-only convention OK')
+else:
+    print(f'GPU build detected ({o[\"n_devices\"]} devices) — CPU-empty assertion skipped')
+"
+echo PASS
+
+# 6. Fit-failure path: invoking against a nonexistent model surfaces a clean
+#    JSON error marker on stdout (not garbage) so subprocess callers can
+#    distinguish fit-failure from parse-failure.
+FAIL_JSON=$("$FIT_PARAMS" --fit-print-plan -m "$WORK_PATH/does-not-exist.gguf" 2>/dev/null || true)
+FAIL_JSON_LAST=$(echo "$FAIL_JSON" | tail -1)
+echo "got fail: $FAIL_JSON_LAST"
+if echo "$FAIL_JSON_LAST" | grep -q '^{'; then
+    echo "$FAIL_JSON_LAST" | python3 -c "
+import json, sys
+try:
+    o = json.loads(sys.stdin.read())
+    if 'error' in o:
+        print('failure JSON marker OK')
+    else:
+        print('WARNING: success-shaped JSON for missing model (got per_device_bytes)')
+        sys.exit(1)
+except json.JSONDecodeError:
+    print('WARNING: failure path emitted non-JSON on stdout')
+    sys.exit(1)
+"
+else
+    # Missing-model may be caught upstream of common_fit_params and not emit
+    # JSON. That's acceptable — non-zero exit suffices to signal failure.
+    echo "fail path emitted no JSON (caught upstream of fit) — acceptable"
+fi
+echo PASS
+
+echo
+echo "ALL fit-params --fit-print-plan smoke tests PASSED"