From 0c739d901b230eb020503ac251266273a98313a6 Mon Sep 17 00:00:00 2001 From: Pascal Date: Sat, 16 May 2026 22:01:32 +0200 Subject: [PATCH 01/39] server: SSE replay buffer, survives client disconnect Opt in on POST /v1/chat/completions when the client sends X-Stream-Resume: 1 and a non empty X-Conversation-Id. The conv id is the session identity end to end, no extra opaque token. The drain runs detached server side and buffers SSE bytes, the generation survives HTTP disconnect, F5, or lets users switch from iOS Safari to another app without losing the actively generated response. Routes: GET /v1/stream/?from=N replay GET /v1/streams[?conversation_id=X] list, drives sidebar spinners DELETE /v1/stream/ Stop, idempotent Router parent fans out to children for list and delete, probes on GET to route to the owner, fans out DELETE on POST so "one session per conv" holds across model swaps. WebUI: the layout snapshots /v1/streams at mount and on visibilitychange, the sidebar reflects live inferences across all convs. The chat page reattaches on mount, append vs fresh is detected from existing content so continue mid stream keeps its prefix. update_slots: on llama_memory_seq_rm refusal at a deep position, full clear of the seq and reprefill from zero instead of GGML_ABORT. OAI strict path unchanged when the opt in headers are absent. --- tools/server/CMakeLists.txt | 2 + tools/server/server-context.cpp | 281 +++++++++++++++- tools/server/server-context.h | 6 + tools/server/server-http.cpp | 17 + tools/server/server-http.h | 1 + tools/server/server-models.cpp | 224 +++++++++++++ tools/server/server-models.h | 3 + tools/server/server-stream.cpp | 284 ++++++++++++++++ tools/server/server-stream.h | 119 +++++++ tools/server/server.cpp | 12 + .../app/chat/ChatScreen/ChatScreen.svelte | 5 + .../ChatScreenStreamResumeStatus.svelte | 17 + tools/ui/src/lib/components/app/chat/index.ts | 2 + tools/ui/src/lib/services/chat.service.ts | 252 ++++++++++---- tools/ui/src/lib/services/mcp.service.ts | 25 +- .../lib/services/stream-discovery.service.ts | 28 ++ .../src/lib/services/stream-resume.service.ts | 76 +++++ tools/ui/src/lib/stores/agentic.svelte.ts | 2 +- tools/ui/src/lib/stores/chat.svelte.ts | 317 +++++++++++++++++- tools/ui/src/lib/types/api.d.ts | 14 + tools/ui/src/lib/types/index.ts | 4 +- tools/ui/src/lib/types/settings.d.ts | 7 + tools/ui/src/lib/utils/abort.ts | 15 +- .../src/routes/(chat)/chat/[id]/+page.svelte | 46 ++- tools/ui/src/routes/+layout.svelte | 13 + tools/ui/tests/unit/abort.test.ts | 56 ++++ tools/ui/tests/unit/stream-discovery.test.ts | 72 ++++ tools/ui/tests/unit/stream-resume.test.ts | 78 +++++ 28 files changed, 1868 insertions(+), 110 deletions(-) create mode 100644 tools/server/server-stream.cpp create mode 100644 tools/server/server-stream.h create mode 100644 tools/ui/src/lib/components/app/chat/ChatScreen/ChatScreenStreamResumeStatus.svelte create mode 100644 tools/ui/src/lib/services/stream-discovery.service.ts create mode 100644 tools/ui/src/lib/services/stream-resume.service.ts create mode 100644 tools/ui/tests/unit/abort.test.ts create mode 100644 tools/ui/tests/unit/stream-discovery.test.ts create mode 100644 tools/ui/tests/unit/stream-resume.test.ts diff --git a/tools/server/CMakeLists.txt b/tools/server/CMakeLists.txt index 47bb582c3081..b5c40884fd6e 100644 --- a/tools/server/CMakeLists.txt +++ b/tools/server/CMakeLists.txt @@ -15,6 +15,8 @@ add_library(${TARGET} STATIC server-common.h server-context.cpp server-context.h + server-stream.cpp + server-stream.h server-tools.cpp server-tools.h server-schema.cpp diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 39b7eb218e69..07441f1b4145 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -5,6 +5,7 @@ #include "server-task.h" #include "server-queue.h" #include "server-schema.h" +#include "server-stream.h" #include "build-info.h" #include "common.h" @@ -864,6 +865,7 @@ struct server_context_impl { server_queue queue_tasks; server_response queue_results; + mutable stream_session_manager stream_sessions; // note: chat_params must not be refreshed upon existing sleeping state server_chat_params chat_params; @@ -872,6 +874,7 @@ struct server_context_impl { server_context_impl() { mtmd_helper_log_set(common_log_default_callback, nullptr); + stream_sessions.start_gc(); } ~server_context_impl() { @@ -3375,9 +3378,26 @@ struct server_context_impl { SLT_TRC(slot, "cached n_tokens = %d, memory_seq_rm [%d, end)\n", slot.prompt.n_tokens(), p0); - common_context_seq_rm(ctx_tgt, slot.id, p0, -1); + // the startup probe in common_context_can_seq_rm only tests a 2 token tail removal + // on seq 0, it cannot guarantee that every partial eviction will succeed at any + // position on any live seq. on refusal by the memory backend we clear the whole + // seq on both contexts and let update_slots reprefill from zero on this iteration + auto * mem_tgt = llama_get_memory(ctx_tgt); + bool partial_ok_tgt = llama_memory_seq_rm(mem_tgt, slot.id, p0, -1); + bool partial_ok_dft = true; if (ctx_dft) { - common_context_seq_rm(ctx_dft.get(), slot.id, p0, -1); + partial_ok_dft = llama_memory_seq_rm(llama_get_memory(ctx_dft.get()), slot.id, p0, -1); + } + if (!partial_ok_tgt || !partial_ok_dft) { + SLT_WRN(slot, "partial KV eviction refused at p0=%d (tgt=%d, dft=%d), full clear of seq %d, reprefilling from zero\n", + p0, partial_ok_tgt ? 1 : 0, partial_ok_dft ? 1 : 0, slot.id); + llama_memory_seq_rm(mem_tgt, slot.id, -1, -1); + if (ctx_dft) { + llama_memory_seq_rm(llama_get_memory(ctx_dft.get()), slot.id, -1, -1); + } + slot.prompt.tokens.keep_first(0); + slot.n_prompt_tokens_cache = 0; + slot.n_prompt_tokens_processed = 0; } // If using an alora, there may be uncached tokens that come @@ -4043,6 +4063,72 @@ void server_context::set_state_callback(server_state_callback_t callback) { } // +// runs in a detached thread, owns the reader and posts the tasks on it +// pulls results, formats them as SSE bytes, appends to the session +// reacts to server shutdown via the shared atomic from the manager +static void spawn_stream_drain( + std::unique_ptr reader, + stream_session_ptr session, + task_response_type res_type, + std::shared_ptr> shutdown) { + std::thread([reader = std::move(reader), + session, + res_type, + shutdown]() mutable { + SRV_INF("stream drain thread started for conv=%s\n", session->conversation_id.c_str()); + // wire the user Stop hook, evict_and_cancel will call this and the reader cancels its queue tasks + session->set_stop_producer([raw = reader.get()] { + raw->stop(); + }); + auto should_stop = [shutdown] { + return shutdown->load(std::memory_order_relaxed); + }; + auto fmt_ok = [res_type](const json & j) -> std::string { + if (res_type == TASK_RESPONSE_TYPE_ANTHROPIC) return format_anthropic_sse(j); + if (res_type == TASK_RESPONSE_TYPE_OAI_RESP) return format_oai_resp_sse(j); + return format_oai_sse(j); + }; + auto fmt_err = [res_type](const json & err) -> std::string { + if (res_type == TASK_RESPONSE_TYPE_ANTHROPIC) { + return format_anthropic_sse({{"event", "error"}, {"data", err}}); + } + return format_oai_sse(json{{"error", err}}); + }; + try { + while (reader->has_next()) { + auto r = reader->next(should_stop); + if (!r) { + break; + } + json j = r->to_json(); + if (r->is_error()) { + auto sse = fmt_err(j); + session->append(sse.data(), sse.size()); + break; + } + auto sse = fmt_ok(j); + if (!session->append(sse.data(), sse.size())) { + break; + } + } + // emit the OAI terminator for the formats that use it + if (res_type != TASK_RESPONSE_TYPE_NONE + && res_type != TASK_RESPONSE_TYPE_OAI_RESP + && res_type != TASK_RESPONSE_TYPE_ANTHROPIC) { + static constexpr char done_str[] = "data: [DONE]\n\n"; + session->append(done_str, sizeof(done_str) - 1); + } + } catch (const std::exception & e) { + auto sse = fmt_err(format_error_response(e.what(), ERROR_TYPE_SERVER)); + session->append(sse.data(), sse.size()); + } + // unwire the stop hook before reader goes out of scope, no dangling captured raw ptr + session->set_stop_producer(nullptr); + session->finalize(); + SRV_INF("stream drain thread finished for conv=%s bytes=%zu\n", session->conversation_id.c_str(), session->total_size()); + }).detach(); +} + // server_routes // @@ -4059,6 +4145,58 @@ std::unique_ptr server_routes::handle_completions_impl( auto & rd = res->rd; auto & params = this->params; + // detect background streaming opt-in via X-Stream-Resume: 1 header. + // when set together with a non empty X-Conversation-Id, the generation survives HTTP disconnect + // and can be resumed via GET /v1/stream/. only meaningful for streaming requests, + // non stream OAI calls keep the standard flow + bool stream = json_value(data, "stream", false); + bool resumable_hdr = false; + std::string conversation_id; + if (stream) { + // request headers preserve the wire casing, the scan is case insensitive + // we capture two headers in a single pass: X-Stream-Resume and X-Conversation-Id + for (const auto & [hk, hv] : req.headers) { + if (hk.size() == 15) { + bool match = true; + static const char target[] = "x-stream-resume"; + for (size_t i = 0; i < 15; ++i) { + char c = hk[i]; + if (c >= 'A' && c <= 'Z') c = char(c + 32); + if (c != target[i]) { match = false; break; } + } + if (match && hv == "1") { + resumable_hdr = true; + } + } else if (hk.size() == 17) { + bool match = true; + static const char target[] = "x-conversation-id"; + for (size_t i = 0; i < 17; ++i) { + char c = hk[i]; + if (c >= 'A' && c <= 'Z') c = char(c + 32); + if (c != target[i]) { match = false; break; } + } + if (match) { + conversation_id = hv; + } + } + } + } + // resumable mode requires both the opt in header and a conversation id, the conv id is the + // session identity end to end (client localStorage, server map, /v1/stream/ routes). + // an opt in without conv id falls back silently to the regular non resumable streaming path + const bool resumable = resumable_hdr && !conversation_id.empty(); + std::unique_ptr drain_reader; + stream_session_ptr session; + server_response_reader * post_target = &rd; + if (resumable) { + drain_reader = std::make_unique( + queue_tasks, queue_results, HTTP_POLLING_SECONDS); + // create_or_replace evicts and cancels any prior session on this conv, + // guaranteeing the invariant that at most one live session exists per conv + session = ctx_server.stream_sessions.create_or_replace(conversation_id); + post_target = drain_reader.get(); + } + try { std::vector tasks; @@ -4126,14 +4264,12 @@ std::unique_ptr server_routes::handle_completions_impl( tasks.push_back(std::move(task)); } - rd.post_tasks(std::move(tasks)); + post_target->post_tasks(std::move(tasks)); } catch (const std::exception & e) { res->error(format_error_response(e.what(), ERROR_TYPE_INVALID_REQUEST)); return res; } - bool stream = json_value(data, "stream", false); - if (!stream) { // non-stream, wait for the results auto all_results = rd.wait_for_all(req.should_stop); @@ -4164,6 +4300,30 @@ std::unique_ptr server_routes::handle_completions_impl( res->ok(arr); } } + } else if (resumable) { + // spawn the detached drain that pumps the response into the session buffer + spawn_stream_drain( + std::move(drain_reader), + session, + res_type, + ctx_server.stream_sessions.shutdown_flag()); + // HTTP response reads from the session, decoupled from the producer + res->status = 200; + res->content_type = "text/event-stream"; + auto offset_ptr = std::make_shared(0); + auto session_capture = session; + res->next = [session_capture, offset_ptr, &req](std::string & output) -> bool { + bool got_any = false; + session_capture->read_from(*offset_ptr, + [&](const char * d, size_t n) { + output.append(d, n); + *offset_ptr += n; + got_any = true; + return false; // exit read_from after the current available bytes + }, + req.should_stop); + return got_any; + }; } else { // in streaming mode, the first error must be treated as non-stream response // this is to match the OAI API behavior @@ -5067,6 +5227,18 @@ void server_routes::init_routes() { res->ok(result->to_json()); return res; }; + + this->get_stream = [this](const server_http_req & req) { + return handle_stream_get_impl(req); + }; + + this->get_streams = [this](const server_http_req & req) { + return handle_streams_list_impl(req); + }; + + this->delete_stream = [this](const server_http_req & req) { + return handle_stream_delete_impl(req); + }; } json server_routes::get_model_info() const { @@ -5337,3 +5509,102 @@ std::unique_ptr server_routes::handle_count_tokens(const l res->ok(response); return res; } + +std::unique_ptr server_routes::handle_stream_get_impl(const server_http_req & req) { + auto res = create_response(); + + // GET /v1/stream/?from=N replays the SSE bytes for that conversation, + // blocks for more bytes when the session is still running, ends on finalize + std::string conv_id = req.get_param("conv_id"); + if (conv_id.empty()) { + res->error(format_error_response("Missing conversation id in path", ERROR_TYPE_INVALID_REQUEST)); + return res; + } + auto session = ctx_server.stream_sessions.get(conv_id); + if (!session) { + res->error(format_error_response("Stream not found or expired", ERROR_TYPE_NOT_FOUND)); + return res; + } + size_t from = 0; + { + std::string from_str = req.get_param("from"); + if (!from_str.empty()) { + try { + from = static_cast(std::stoull(from_str)); + } catch (const std::exception &) { + res->error(format_error_response("Invalid 'from' offset", ERROR_TYPE_INVALID_REQUEST)); + return res; + } + } + } + if (from < session->dropped_prefix()) { + res->error(format_error_response("Stream offset lost, please restart", ERROR_TYPE_INVALID_REQUEST)); + return res; + } + res->status = 200; + res->content_type = "text/event-stream"; + + auto offset_ptr = std::make_shared(from); + auto session_capture = session; + res->next = [session_capture, offset_ptr, &req](std::string & output) -> bool { + bool got_any = false; + session_capture->read_from(*offset_ptr, + [&](const char * d, size_t n) { + output.append(d, n); + *offset_ptr += n; + got_any = true; + return false; + }, + req.should_stop); + return got_any; + }; + return res; +} + +std::unique_ptr server_routes::handle_streams_list_impl(const server_http_req & req) { + auto res = create_response(); + + // GET /v1/streams returns sessions as a JSON array. + // with conversation_id set: at most one entry for that conv (running or finalized). + // without conversation_id: every live or recently completed session known to this server, + // used by the WebUI at mount and on visibilitychange to populate the sidebar spinners + std::string conversation_id = req.get_param("conversation_id"); + std::vector sessions; + if (conversation_id.empty()) { + sessions = ctx_server.stream_sessions.list_all(); + } else { + auto s = ctx_server.stream_sessions.get(conversation_id); + if (s) { + sessions.push_back(s); + } + } + json arr = json::array(); + for (auto & s : sessions) { + arr.push_back({ + {"conversation_id", s->conversation_id}, + {"is_done", s->is_done()}, + {"total_bytes", s->total_size()}, + {"started_at", s->started_ts}, + {"completed_at", s->completed_at()}, + }); + } + res->ok(arr); + return res; +} + +std::unique_ptr server_routes::handle_stream_delete_impl(const server_http_req & req) { + auto res = create_response(); + + // DELETE /v1/stream/ cancels the producer side then evicts the buffer. + // idempotent: a session that already finalized or was never created simply returns 204 + std::string conv_id = req.get_param("conv_id"); + if (conv_id.empty()) { + res->error(format_error_response("Missing conversation id in path", ERROR_TYPE_INVALID_REQUEST)); + return res; + } + SRV_INF("DELETE /v1/stream/%s -> evict_and_cancel\n", conv_id.c_str()); + ctx_server.stream_sessions.evict_and_cancel(conv_id); + res->status = 204; + res->content_type = "application/json"; + return res; +} diff --git a/tools/server/server-context.h b/tools/server/server-context.h index 952f825f7245..8bd07db73c95 100644 --- a/tools/server/server-context.h +++ b/tools/server/server-context.h @@ -152,6 +152,9 @@ struct server_routes { server_http_context::handler_t post_rerank; server_http_context::handler_t get_lora_adapters; server_http_context::handler_t post_lora_adapters; + server_http_context::handler_t get_stream; + server_http_context::handler_t get_streams; + server_http_context::handler_t delete_stream; // to be used in router mode json get_model_info() const; @@ -168,6 +171,9 @@ struct server_routes { std::unique_ptr handle_slots_erase(const server_http_req &, int id_slot); std::unique_ptr handle_embeddings_impl(const server_http_req & req, task_response_type res_type); std::unique_ptr handle_count_tokens(const llama_vocab * vocab, mtmd_context * mctx, const server_http_req & req, task_response_type res_type); + std::unique_ptr handle_stream_get_impl(const server_http_req & req); + std::unique_ptr handle_streams_list_impl(const server_http_req & req); + std::unique_ptr handle_stream_delete_impl(const server_http_req & req); // using unique_ptr to allow late initialization of const std::unique_ptr meta; diff --git a/tools/server/server-http.cpp b/tools/server/server-http.cpp index 4f2abab00cee..ec9c8080631f 100644 --- a/tools/server/server-http.cpp +++ b/tools/server/server-http.cpp @@ -543,6 +543,23 @@ void server_http_context::get(const std::string & path, const server_http_contex }); } +void server_http_context::del_(const std::string & path, const server_http_context::handler_t & handler) const { + handlers.emplace(path, handler); + pimpl->srv->Delete(path_prefix + path, [handler](const httplib::Request & req, httplib::Response & res) { + server_http_req_ptr request = std::make_unique(server_http_req{ + get_params(req), + get_headers(req), + req.path, + build_query_string(req), + req.body, + {}, + req.is_connection_closed + }); + server_http_res_ptr response = handler(*request); + process_handler_response(std::move(request), response, res); + }); +} + void server_http_context::post(const std::string & path, const server_http_context::handler_t & handler) const { handlers.emplace(path, handler); pimpl->srv->Post(path_prefix + path, [handler](const httplib::Request & req, httplib::Response & res) { diff --git a/tools/server/server-http.h b/tools/server/server-http.h index 6b4a4b87a631..c31dd9109de2 100644 --- a/tools/server/server-http.h +++ b/tools/server/server-http.h @@ -87,6 +87,7 @@ struct server_http_context { void get(const std::string & path, const handler_t & handler) const; void post(const std::string & path, const handler_t & handler) const; void del(const std::string & path, const handler_t & handler) const; + void del_(const std::string & path, const handler_t & handler) const; // Register the Google Cloud Platform (Vertex AI) compat (AIP_PREDICT_ROUTE env var, or /predict) // Must be called AFTER all other API routes are registered diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index a4df3ef108f6..d4516f3dc2fd 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -1263,6 +1263,11 @@ bool server_models::ensure_model_ready(const std::string & name) { return true; } +// forward declarations for the file scope helpers used below, the bodies live further down +// next to the other routes helpers to keep the proxy methods compact +static void fan_out_delete_others_for_conv( + server_models & models, const std::string & conversation_id, const std::string & target_child); + server_http_res_ptr server_models::proxy_request(const server_http_req & req, const std::string & method, const std::string & name, bool update_last_used) { auto meta = get_meta(name); if (!meta.has_value()) { @@ -1275,6 +1280,34 @@ server_http_res_ptr server_models::proxy_request(const server_http_req & req, co std::unique_lock lk(mutex); mapping[name].meta.last_used = ggml_time_ms(); } + // when the client opts in to resumable streaming (X-Stream-Resume: 1 + non empty + // X-Conversation-Id), fan out a DELETE on every other ready child to evict any prior + // session for this conv. enforces the cross child invariant 'one session per convId', + // safe to call unconditionally and cheap on loopback. ignored for any other request shape + { + std::string conv_id; + bool resume_opt_in = false; + for (const auto & [hk, hv] : req.headers) { + if (hk.size() == 17) { + std::string lower(hk); + std::transform(lower.begin(), lower.end(), lower.begin(), + [](unsigned char c) { return (c >= 'A' && c <= 'Z') ? char(c + 32) : char(c); }); + if (lower == "x-conversation-id") { + conv_id = hv; + } + } else if (hk.size() == 15) { + std::string lower(hk); + std::transform(lower.begin(), lower.end(), lower.begin(), + [](unsigned char c) { return (c >= 'A' && c <= 'Z') ? char(c + 32) : char(c); }); + if (lower == "x-stream-resume" && hv == "1") { + resume_opt_in = true; + } + } + } + if (resume_opt_in && !conv_id.empty()) { + fan_out_delete_others_for_conv(*this, conv_id, name); + } + } SRV_INF("proxying request to model %s on port %d\n", name.c_str(), meta->port); std::string proxy_path = req.path; if (!req.query_string.empty()) { @@ -1293,6 +1326,9 @@ server_http_res_ptr server_models::proxy_request(const server_http_req & req, co base_params.timeout_read, base_params.timeout_write ); + // session identity end to end is the X-Conversation-Id sent by the client, no extra opaque + // token to mangle here. the parent can later route GET /v1/stream/ back to the right + // child by probing /v1/streams across childs return proxy; } @@ -1539,6 +1575,80 @@ struct server_models_sse_client { } }; +// percent encode a single query string value, covers reserved chars without dragging in +// httplib::detail. used by the stream routes to forward conversation_id to children +static std::string encode_qs(const std::string & in) { + std::string out; + out.reserve(in.size() * 3); + for (unsigned char c : in) { + bool safe = (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') + || c == '-' || c == '_' || c == '.' || c == '~'; + if (safe) { + out.push_back(char(c)); + } else { + char buf[4]; + std::snprintf(buf, sizeof(buf), "%%%02X", c); + out.append(buf, 3); + } + } + return out; +} + +// scan every ready child for an active session on this conversation_id by fanning out a +// short list query on the loopback, returns the meta of the first child whose array is +// non empty. with the invariant 'one session per convId across all children' enforced by +// the POST path, at most one child can match +static std::optional find_child_for_conv( + server_models & models, const std::string & conversation_id) { + if (conversation_id.empty()) { + return std::nullopt; + } + std::string child_path = "/v1/streams?conversation_id=" + encode_qs(conversation_id); + for (auto & meta : models.get_all_meta()) { + if (!meta.is_ready()) { + continue; + } + httplib::Client cli(CHILD_ADDR, meta.port); + cli.set_connection_timeout(0, 250 * 1000); + cli.set_read_timeout(0, 250 * 1000); + cli.set_write_timeout(0, 250 * 1000); + auto resp = cli.Get(child_path.c_str()); + if (!resp || resp->status != 200) { + continue; + } + try { + json arr = json::parse(resp->body); + if (arr.is_array() && !arr.empty()) { + return meta; + } + } catch (const std::exception &) { + continue; + } + } + return std::nullopt; +} + +// fan out a DELETE on every ready child EXCEPT the one we are about to route the POST to, +// so a model swap on the same conversation_id evicts the previous session cleanly. safe to +// call unconditionally: a child without the session returns 204 and does nothing +static void fan_out_delete_others_for_conv( + server_models & models, const std::string & conversation_id, const std::string & target_child) { + if (conversation_id.empty()) { + return; + } + std::string child_path = "/v1/stream/" + encode_qs(conversation_id); + for (auto & meta : models.get_all_meta()) { + if (!meta.is_ready() || meta.name == target_child) { + continue; + } + httplib::Client cli(CHILD_ADDR, meta.port); + cli.set_connection_timeout(0, 250 * 1000); + cli.set_read_timeout(0, 250 * 1000); + cli.set_write_timeout(0, 250 * 1000); + cli.Delete(child_path.c_str()); + } +} + static void res_ok(std::unique_ptr & res, const json & response_data) { res->status = 200; res->data = safe_json_to_str(response_data); @@ -1620,6 +1730,120 @@ void server_models_routes::init_routes() { return models.proxy_request(req, method, name, false); }; + + this->proxy_get_stream = [this](const server_http_req & req) { + auto res = std::make_unique(); + + // GET /v1/stream/?from=N. find the child that owns the session for this conv + // via the loopback probe in find_child_for_conv, then forward the SSE GET to it. + // returns 404 if no child currently has an alive or recently completed session + std::string conv_id = req.get_param("conv_id"); + if (conv_id.empty()) { + res_err(res, format_error_response("Missing conversation id in path", ERROR_TYPE_INVALID_REQUEST)); + return res; + } + auto owner = find_child_for_conv(models, conv_id); + if (!owner.has_value()) { + res_err(res, format_error_response("Stream not found or expired", ERROR_TYPE_NOT_FOUND)); + return res; + } + + std::string from = req.get_param("from"); + std::string child_path = "/v1/stream/" + encode_qs(conv_id); + if (!from.empty()) { + child_path += "?from=" + from; + } + SRV_INF("proxying stream resume to model %s on port %d, path=%s\n", + owner->name.c_str(), owner->port, child_path.c_str()); + + auto proxy = std::make_unique( + "GET", + "http", + CHILD_ADDR, + owner->port, + child_path, + req.headers, + req.body, + req.files, + req.should_stop, + params.timeout_read, + params.timeout_write); + return std::unique_ptr(std::move(proxy)); + }; + + this->proxy_get_streams = [this](const server_http_req & req) { + auto res = std::make_unique(); + + // GET /v1/streams returns sessions as a JSON array. with conversation_id set the filter + // is forwarded to childs and at most one entry comes back, without it the WebUI uses the + // result at mount and on visibilitychange to populate the sidebar spinners across convs. + // sequential fan out on every ready child, fail soft on per child error, aggregate + std::string conversation_id = req.get_param("conversation_id"); + std::string child_path = "/v1/streams"; + if (!conversation_id.empty()) { + child_path += "?conversation_id=" + encode_qs(conversation_id); + } + + json aggregated = json::array(); + for (auto & meta : models.get_all_meta()) { + if (!meta.is_ready()) { + continue; + } + httplib::Client cli(CHILD_ADDR, meta.port); + cli.set_connection_timeout(0, 250 * 1000); + cli.set_read_timeout(0, 250 * 1000); + cli.set_write_timeout(0, 250 * 1000); + auto resp = cli.Get(child_path.c_str()); + if (!resp || resp->status != 200) { + continue; + } + try { + json child_arr = json::parse(resp->body); + if (!child_arr.is_array()) { + continue; + } + for (auto & entry : child_arr) { + if (entry.is_object()) { + aggregated.push_back(entry); + } + } + } catch (const std::exception &) { + continue; + } + } + res_ok(res, aggregated); + return res; + }; + + this->proxy_delete_stream = [this](const server_http_req & req) { + auto res = std::make_unique(); + + // DELETE /v1/stream/ fans out to every ready child. each child runs an idempotent + // evict_and_cancel, returning 204 whether or not it actually owned a session for this conv. + // a Stop must feel instantaneous so timeouts are short, the child route is in memory + std::string conv_id = req.get_param("conv_id"); + if (conv_id.empty()) { + res_err(res, format_error_response("Missing conversation id in path", ERROR_TYPE_INVALID_REQUEST)); + return res; + } + + std::string child_path = "/v1/stream/" + encode_qs(conv_id); + for (auto & meta : models.get_all_meta()) { + if (!meta.is_ready()) { + continue; + } + httplib::Client cli(CHILD_ADDR, meta.port); + cli.set_connection_timeout(0, 250 * 1000); + cli.set_read_timeout(0, 500 * 1000); + cli.set_write_timeout(0, 250 * 1000); + auto resp = cli.Delete(child_path.c_str()); + (void) resp; // best effort, 404 and network errors are equivalent to no op + } + res->status = 204; + res->content_type = "application/json"; + return res; + }; + this->proxy_post = [this](const server_http_req & req) { std::string method = "POST"; json body = json::parse(req.body); diff --git a/tools/server/server-models.h b/tools/server/server-models.h index 9ed4aeead0dd..070a4468e28f 100644 --- a/tools/server/server-models.h +++ b/tools/server/server-models.h @@ -261,6 +261,9 @@ struct server_models_routes { server_http_context::handler_t get_router_props; server_http_context::handler_t proxy_get; server_http_context::handler_t proxy_post; + server_http_context::handler_t proxy_get_stream; + server_http_context::handler_t proxy_get_streams; + server_http_context::handler_t proxy_delete_stream; server_http_context::handler_t get_router_models; server_http_context::handler_t post_router_models_load; server_http_context::handler_t post_router_models_unload; diff --git a/tools/server/server-stream.cpp b/tools/server/server-stream.cpp new file mode 100644 index 000000000000..54ae052e5fde --- /dev/null +++ b/tools/server/server-stream.cpp @@ -0,0 +1,284 @@ +#include "server-stream.h" + +#include +#include + +namespace { +constexpr int64_t STREAM_SESSION_TTL_SECONDS = 300; +constexpr size_t STREAM_SESSION_MAX_BYTES = 4 * 1024 * 1024; +constexpr int64_t STREAM_SESSION_GC_INTERVAL_SECONDS = 60; +constexpr int64_t STREAM_READ_WAKE_INTERVAL_MS = 200; + +// returns unix time in seconds +int64_t now_seconds() { + return std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch() + ).count(); +} +} + +stream_session::stream_session(std::string conversation_id_, size_t max_bytes_) + : conversation_id(std::move(conversation_id_)) + , started_ts(now_seconds()) + , prefix_dropped(0) + , cap_bytes(max_bytes_) + , done(false) + , completed_ts(0) { + buffer.reserve(64 * 1024); +} + +bool stream_session::append(const char * data, size_t len) { + if (len == 0) { + return true; + } + { + std::lock_guard lock(mu); + if (done.load(std::memory_order_relaxed)) { + return false; + } + if (len >= cap_bytes) { + // single chunk bigger than the cap, keep only the tail that fits + size_t skip = len - cap_bytes; + prefix_dropped += buffer.size() + skip; + buffer.clear(); + buffer.insert(buffer.end(), data + skip, data + len); + } else { + size_t needed = buffer.size() + len; + if (needed > cap_bytes) { + size_t to_drop = needed - cap_bytes; + buffer.erase(buffer.begin(), buffer.begin() + to_drop); + prefix_dropped += to_drop; + } + buffer.insert(buffer.end(), data, data + len); + } + } + cv.notify_all(); + return true; +} + +void stream_session::finalize() { + bool was_done = done.exchange(true, std::memory_order_acq_rel); + if (was_done) { + return; + } + completed_ts.store(now_seconds(), std::memory_order_release); + cv.notify_all(); +} + +stream_read_status stream_session::read_from(size_t offset, + const std::function & sink, + const std::function & should_stop) { + std::unique_lock lock(mu); + while (true) { + if (should_stop && should_stop()) { + return stream_read_status::OK; + } + if (offset < prefix_dropped) { + return stream_read_status::OFFSET_LOST; + } + size_t logical_end = prefix_dropped + buffer.size(); + if (offset < logical_end) { + size_t local_off = offset - prefix_dropped; + size_t n = buffer.size() - local_off; + // copy the available chunk under the lock, release before calling the sink + std::vector chunk(buffer.begin() + local_off, buffer.begin() + local_off + n); + offset += n; + lock.unlock(); + bool keep_going = sink(chunk.data(), chunk.size()); + if (!keep_going) { + return stream_read_status::OK; + } + lock.lock(); + continue; + } + if (done.load(std::memory_order_acquire)) { + return stream_read_status::OK; + } + // wait for new bytes, finalize, or a periodic wake to re check should_stop + cv.wait_for(lock, std::chrono::milliseconds(STREAM_READ_WAKE_INTERVAL_MS)); + } +} + +bool stream_session::is_done() const { + return done.load(std::memory_order_acquire); +} + +size_t stream_session::total_size() const { + std::lock_guard lock(mu); + return prefix_dropped + buffer.size(); +} + +size_t stream_session::dropped_prefix() const { + std::lock_guard lock(mu); + return prefix_dropped; +} + +int64_t stream_session::completed_at() const { + return completed_ts.load(std::memory_order_acquire); +} + +void stream_session::set_stop_producer(std::function fn) { + std::lock_guard lock(mu); + stop_producer = std::move(fn); +} + +void stream_session::cancel() { + // copy the hook under the lock then invoke outside, the producer side may grab queue locks + // and we do not want to hold our mu across that path + std::function fn; + { + std::lock_guard lock(mu); + fn = stop_producer; + } + if (fn) { + fn(); + } +} + +stream_session_manager::stream_session_manager() + : running(false) + , drain_shutdown(std::make_shared>(false)) { +} + +stream_session_manager::~stream_session_manager() { + stop_gc(); +} + +stream_session_ptr stream_session_manager::create_or_replace(const std::string & conversation_id) { + // evict any previous session on the same conv, this guarantees the invariant + // "one conv = at most one live session" and propagates cancel to its producer + stream_session_ptr previous; + auto fresh = std::make_shared(conversation_id, STREAM_SESSION_MAX_BYTES); + { + std::unique_lock lock(map_mu); + auto it = sessions.find(conversation_id); + if (it != sessions.end()) { + previous = it->second; + it->second = fresh; + } else { + sessions.emplace(conversation_id, fresh); + } + } + if (previous) { + previous->cancel(); + previous->finalize(); + } + return fresh; +} + +stream_session_ptr stream_session_manager::get(const std::string & conversation_id) { + std::shared_lock lock(map_mu); + auto it = sessions.find(conversation_id); + if (it == sessions.end()) { + return nullptr; + } + return it->second; +} + +std::vector stream_session_manager::list_all() const { + std::vector out; + std::shared_lock lock(map_mu); + out.reserve(sessions.size()); + for (auto & kv : sessions) { + out.push_back(kv.second); + } + return out; +} + +void stream_session_manager::evict(const std::string & conversation_id) { + stream_session_ptr s; + { + std::unique_lock lock(map_mu); + auto it = sessions.find(conversation_id); + if (it == sessions.end()) { + return; + } + s = it->second; + sessions.erase(it); + } + // finalize outside the map lock so any pending readers wake up and exit + s->finalize(); +} + +void stream_session_manager::evict_and_cancel(const std::string & conversation_id) { + stream_session_ptr s; + { + std::unique_lock lock(map_mu); + auto it = sessions.find(conversation_id); + if (it == sessions.end()) { + return; + } + s = it->second; + sessions.erase(it); + } + // signal the producer side first so the inference is cancelled at the queue level, + // then finalize, which wakes any pending HTTP reader and lets the drain exit naturally + s->cancel(); + s->finalize(); +} + +void stream_session_manager::start_gc() { + if (running.exchange(true)) { + return; + } + gc_thread = std::thread([this] { gc_loop(); }); +} + +void stream_session_manager::stop_gc() { + drain_shutdown->store(true, std::memory_order_release); + bool was_running = running.exchange(false); + if (was_running) { + { + std::lock_guard lock(gc_wake_mu); + } + gc_wake_cv.notify_all(); + if (gc_thread.joinable()) { + gc_thread.join(); + } + } + // finalize all live sessions so no reader ever hangs + std::vector snapshot; + { + std::unique_lock lock(map_mu); + snapshot.reserve(sessions.size()); + for (auto & kv : sessions) { + snapshot.push_back(kv.second); + } + sessions.clear(); + } + for (auto & s : snapshot) { + s->finalize(); + } +} + +void stream_session_manager::gc_loop() { + while (running.load(std::memory_order_acquire)) { + { + std::unique_lock lock(gc_wake_mu); + gc_wake_cv.wait_for(lock, + std::chrono::seconds(STREAM_SESSION_GC_INTERVAL_SECONDS), + [this] { return !running.load(std::memory_order_acquire); }); + } + if (!running.load(std::memory_order_acquire)) { + return; + } + int64_t cutoff = now_seconds() - STREAM_SESSION_TTL_SECONDS; + std::vector to_drop; + { + std::unique_lock lock(map_mu); + for (auto it = sessions.begin(); it != sessions.end(); ) { + int64_t completed = it->second->completed_at(); + if (completed != 0 && completed <= cutoff) { + to_drop.push_back(it->second); + it = sessions.erase(it); + } else { + ++it; + } + } + } + // finalize outside the map lock, idempotent if the session was already done + for (auto & s : to_drop) { + s->finalize(); + } + } +} diff --git a/tools/server/server-stream.h b/tools/server/server-stream.h new file mode 100644 index 000000000000..f4ee9c7893f6 --- /dev/null +++ b/tools/server/server-stream.h @@ -0,0 +1,119 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +enum class stream_read_status { + OK, + OFFSET_LOST, +}; + +// streaming buffer for one generation, survives HTTP disconnect. +// the producer side pushes raw SSE bytes via append. HTTP readers drain from +// any offset via read_from. read_from blocks until new bytes arrive or the +// session is finalized. identity of the session is the conversation_id, no +// extra opaque token: one conv = at most one live session at a time +struct stream_session { + std::string conversation_id; + int64_t started_ts; // unix seconds at construction, used by /v1/streams listing + + stream_session(std::string conversation_id_, size_t max_bytes_); + stream_session(const stream_session &) = delete; + stream_session & operator=(const stream_session &) = delete; + + // append raw bytes, drops from the front if the cap is reached. + // returns false if the session is already finalized + bool append(const char * data, size_t len); + + // mark the session as complete, wakes all pending readers + void finalize(); + + // drain bytes from offset, calling sink for each chunk. blocks until more + // bytes arrive or finalize is called. returns OK on clean exit, OFFSET_LOST + // if offset falls below the dropped prefix + stream_read_status read_from(size_t offset, + const std::function & sink, + const std::function & should_stop); + + bool is_done() const; + size_t total_size() const; // bytes that ever entered the session + size_t dropped_prefix() const; // bytes evicted from the front due to cap + int64_t completed_at() const; // 0 while alive, unix seconds after finalize + + // attach a producer side stop hook, the drain sets this on startup so we can cancel its + // underlying reader. pass an empty function to detach (drain must clear before destroying + // its reader) + void set_stop_producer(std::function fn); + + // invoke the stop hook if attached, signals the producer to abort its inference asap, + // idempotent + void cancel(); + +private: + mutable std::mutex mu; + std::condition_variable cv; + std::vector buffer; + size_t prefix_dropped; + size_t cap_bytes; + std::atomic done; + std::atomic completed_ts; + std::function stop_producer; // protected by mu +}; + +using stream_session_ptr = std::shared_ptr; + +// owns all live sessions, runs a periodic GC to evict expired ones. +// the map is keyed by conversation_id, so the invariant "one conv = at most one +// live session" is enforced at the type level +class stream_session_manager { +public: + stream_session_manager(); + ~stream_session_manager(); + + stream_session_manager(const stream_session_manager &) = delete; + stream_session_manager & operator=(const stream_session_manager &) = delete; + + // install a new session for this conversation, evicting and cancelling any previous one. + // the conversation_id must be non empty, the caller is responsible for that check. + // returns the new session + stream_session_ptr create_or_replace(const std::string & conversation_id); + + // lookup, returns null if unknown or already evicted + stream_session_ptr get(const std::string & conversation_id); + + // list every live or recently completed session, used by GET /v1/streams without filter + std::vector list_all() const; + + // remove from the map and finalize, wakes any pending readers + void evict(const std::string & conversation_id); + + // signal the producer to cancel asap then evict, used by the explicit user Stop path + void evict_and_cancel(const std::string & conversation_id); + + void start_gc(); + void stop_gc(); + + // shared atomic flipped to true on stop_gc, drain threads poll it to exit cleanly + std::shared_ptr> shutdown_flag() const { return drain_shutdown; } + +private: + void gc_loop(); + + mutable std::shared_mutex map_mu; + std::unordered_map sessions; // key: conversation_id + std::thread gc_thread; + std::atomic running; + std::mutex gc_wake_mu; + std::condition_variable gc_wake_cv; + std::shared_ptr> drain_shutdown; +}; diff --git a/tools/server/server.cpp b/tools/server/server.cpp index 4165c1015e8a..ee98aa7caa71 100644 --- a/tools/server/server.cpp +++ b/tools/server/server.cpp @@ -184,6 +184,9 @@ int llama_server(int argc, char ** argv) { routes.post_lora_adapters = models_routes->proxy_post; routes.get_slots = models_routes->proxy_get; routes.post_slots = models_routes->proxy_post; + routes.get_stream = models_routes->proxy_get_stream; + routes.get_streams = models_routes->proxy_get_streams; + routes.delete_stream = models_routes->proxy_delete_stream; // custom routes for router routes.get_props = models_routes->get_router_props; @@ -238,6 +241,15 @@ int llama_server(int argc, char ** argv) { ctx_http.get ("/slots", ex_wrapper(routes.get_slots)); ctx_http.post("/slots/:id_slot", ex_wrapper(routes.post_slots)); + // resumable streaming, the conversation_id is the session identity end to end: + // GET /v1/stream/?from=N replays SSE bytes for a session in progress or recently completed + ctx_http.get ("/v1/stream/:conv_id", ex_wrapper(routes.get_stream)); + // GET /v1/streams lists sessions, with optional conversation_id query to filter to one conv, + // without filter the WebUI uses it at mount and on visibilitychange to populate sidebar spinners + ctx_http.get ("/v1/streams", ex_wrapper(routes.get_streams)); + // DELETE /v1/stream/ is the explicit user Stop, cancels the producer and evicts, idempotent + ctx_http.del_("/v1/stream/:conv_id", ex_wrapper(routes.delete_stream)); + // Google Cloud Platform (Vertex AI) compat ctx_http.register_gcp_compat(); diff --git a/tools/ui/src/lib/components/app/chat/ChatScreen/ChatScreen.svelte b/tools/ui/src/lib/components/app/chat/ChatScreen/ChatScreen.svelte index 18635ba392ce..d2fba5a932e0 100644 --- a/tools/ui/src/lib/components/app/chat/ChatScreen/ChatScreen.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatScreen/ChatScreen.svelte @@ -5,6 +5,7 @@ ChatMessages, ChatScreenDragOverlay, ChatScreenProcessingInfo, + ChatScreenStreamResumeStatus, ServerLoadingSplash, ChatScreenServerError } from '$lib/components/app'; @@ -281,6 +282,10 @@ + {#if page.params.id} + + {/if} +
{#if (isMobile.current ? mobileScrollDownHint || isMobileUserScrolledUp : autoScroll.userScrolledUp) && page.url.hash.includes(ROUTES.CHAT) && page.params.id} + import { chatStore } from '$lib/stores/chat.svelte'; + import { Loader2 } from '@lucide/svelte'; + + let state = $derived(chatStore.streamConnectionState); + + +{#if state === 'resuming'} +
+ + Reconnecting to the stream... +
+{/if} diff --git a/tools/ui/src/lib/components/app/chat/index.ts b/tools/ui/src/lib/components/app/chat/index.ts index 517f24d7407b..4d039d056b5e 100644 --- a/tools/ui/src/lib/components/app/chat/index.ts +++ b/tools/ui/src/lib/components/app/chat/index.ts @@ -683,3 +683,5 @@ export { default as ChatScreenProcessingInfo } from './ChatScreen/ChatScreenProc * Rendered inside ChatScreen when `serverError` store has a value. */ export { default as ChatScreenServerError } from './ChatScreen/ChatScreenServerError.svelte'; + +export { default as ChatScreenStreamResumeStatus } from './ChatScreen/ChatScreenStreamResumeStatus.svelte'; diff --git a/tools/ui/src/lib/services/chat.service.ts b/tools/ui/src/lib/services/chat.service.ts index 9001c9572fea..b0d72476143c 100644 --- a/tools/ui/src/lib/services/chat.service.ts +++ b/tools/ui/src/lib/services/chat.service.ts @@ -1,6 +1,11 @@ import { getJsonHeaders } from '$lib/utils/api-headers'; import { formatAttachmentText } from '$lib/utils/formatters'; import { isAbortError } from '$lib/utils/abort'; +import { + saveStreamState, + clearStreamState, + resumeStream +} from '$lib/services/stream-resume.service'; import { ATTACHMENT_LABEL_PDF_FILE, ATTACHMENT_LABEL_MCP_PROMPT, @@ -31,7 +36,8 @@ import type { import type { AudioInputFormat, DatabaseMessageExtraMcpPrompt, - DatabaseMessageExtraMcpResource + DatabaseMessageExtraMcpResource, + StreamConnectionState } from '$lib/types'; import { modelsStore } from '$lib/stores/models.svelte'; import { settingsStore } from '../stores/settings.svelte'; @@ -128,6 +134,7 @@ export class ChatService { onChunk, onComplete, onError, + onConnectionState, onReasoningChunk, onToolCallChunk, onModel, @@ -312,9 +319,18 @@ export class ChatService { } try { + const headers: Record = { ...getJsonHeaders() }; + if (stream) { + headers['X-Stream-Resume'] = '1'; + } + // tag the request with the conversation id so the server can later list live or recently completed + // sessions for that conversation, this is what powers discoverActiveStream on tab reopen + if (conversationId) { + headers['X-Conversation-Id'] = conversationId; + } const response = await fetch(API_CHAT.COMPLETIONS, { method: 'POST', - headers: getJsonHeaders(), + headers, body: JSON.stringify(requestBody), signal }); @@ -341,7 +357,8 @@ export class ChatService { onCompletionId, onTimings, conversationId, - signal + signal, + onConnectionState ); return; @@ -473,6 +490,15 @@ export class ChatService { * @param excludeReasoning - Whether to strip reasoning content (should match excludeReasoningFromContext setting) * @param signal - Optional AbortSignal to cancel the pre-encode request */ + static async cancelServerStream(conversationId: string): Promise { + if (!conversationId) return; + try { + await fetch(`./v1/stream/${encodeURIComponent(conversationId)}`, { method: 'DELETE' }); + } catch (e) { + console.warn('cancelServerStream failed:', e); + } + } + static async preEncode( messages: ApiChatMessageData[] | (DatabaseMessage & { extra?: DatabaseMessageExtra[] })[], model?: string | null, @@ -557,7 +583,7 @@ export class ChatService { * @returns {Promise} Promise that resolves when streaming is complete * @throws {Error} if the stream cannot be read or parsed */ - private static async handleStreamResponse( + static async handleStreamResponse( response: Response, onChunk?: (chunk: string) => void, onComplete?: ( @@ -573,15 +599,33 @@ export class ChatService { onCompletionId?: (id: string) => void, onTimings?: (timings?: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => void, conversationId?: string, - abortSignal?: AbortSignal + abortSignal?: AbortSignal, + onConnectionState?: (state: StreamConnectionState) => void ): Promise { - const reader = response.body?.getReader(); + let reader = response.body?.getReader(); if (!reader) { throw new Error('No response body'); } - const decoder = new TextDecoder(); + // bytesParsed is the absolute server side buffer offset of the next byte to parse + // segmentStartOffset is the absolute offset where the current reader started, reset on resume + // segmentBytesRead is wire bytes read by the current reader + let bytesParsed = 0; + let segmentStartOffset = 0; + let segmentBytesRead = 0; + let lastByteAt = Date.now(); + // each resume must produce at least one byte to be retried again + // if a resume returns 200 but yields nothing, we abandon + // since the session has a bounded size, the total number of retries is bounded by construction + let madeProgress = true; + const encoder = new TextEncoder(); + if (conversationId) { + saveStreamState(conversationId, 0); + } + onConnectionState?.('streaming'); + + let decoder = new TextDecoder(); let aggregatedContent = ''; let fullReasoningContent = ''; let aggregatedToolCalls: ApiChatCompletionToolCall[] = []; @@ -633,84 +677,157 @@ export class ChatService { } }; + const onVisibilityChange = () => { + if (typeof document === 'undefined') return; + if (document.visibilityState !== 'visible') return; + if (streamFinished) return; + if (!conversationId) return; + // the bytes have been quiet for too long, the OS likely killed the socket + // kicking the reader unblocks reader.read with done=true so the outer loop can resume + if (Date.now() - lastByteAt > 300) { + reader!.cancel().catch(() => {}); + } + }; + if (typeof document !== 'undefined') { + document.addEventListener('visibilitychange', onVisibilityChange); + } + try { let chunk = ''; + // outer loop drives the resume cycle, swaps reader on premature end of stream while (true) { - if (abortSignal?.aborted) break; + while (true) { + if (abortSignal?.aborted) break; - const { done, value } = await reader.read(); - if (done) break; + const { done, value } = await reader.read(); + if (done) break; - if (abortSignal?.aborted) break; + if (abortSignal?.aborted) break; - chunk += decoder.decode(value, { stream: true }); - const lines = chunk.split(SSE_LINE_SEPARATOR); - chunk = lines.pop() || ''; + if (value && value.byteLength > 0) { + segmentBytesRead += value.byteLength; + lastByteAt = Date.now(); + if (!madeProgress) { + madeProgress = true; + onConnectionState?.('streaming'); + } + } - for (const line of lines) { - if (abortSignal?.aborted) break; + chunk += decoder.decode(value, { stream: true }); + const lines = chunk.split(SSE_LINE_SEPARATOR); + chunk = lines.pop() || ''; - if (line.startsWith(SSE_DATA_PREFIX)) { - const data = line.slice(SSE_DATA_PREFIX.length).trim(); - if (data === SSE_DONE_MARKER) { - streamFinished = true; + // the persisted offset must point right after the last fully parsed line, + // the trailing `chunk` is partial bytes still waiting for a newline + if (conversationId) { + const tailBytes = encoder.encode(chunk).byteLength; + bytesParsed = segmentStartOffset + segmentBytesRead - tailBytes; + saveStreamState(conversationId, bytesParsed); + } - continue; - } + for (const line of lines) { + if (abortSignal?.aborted) break; - try { - const parsed: ApiChatCompletionStreamChunk = JSON.parse(data); - const choice = parsed.choices?.[0]; - const content = choice?.delta?.content; - const reasoningContent = choice?.delta?.reasoning_content; - const toolCalls = choice?.delta?.tool_calls; - const timings = parsed.timings; - const promptProgress = parsed.prompt_progress; - - const chunkModel = ChatService.extractModelName(parsed); - if (chunkModel && !modelEmitted) { - modelEmitted = true; - onModel?.(chunkModel); - } + if (line.startsWith(SSE_DATA_PREFIX)) { + const data = line.slice(SSE_DATA_PREFIX.length).trim(); + if (data === SSE_DONE_MARKER) { + streamFinished = true; - if (parsed.id && !idEmitted) { - idEmitted = true; - onCompletionId?.(parsed.id); + continue; } - if (promptProgress) { - ChatService.notifyTimings(undefined, promptProgress, onTimings); - } + try { + const parsed: ApiChatCompletionStreamChunk = JSON.parse(data); + const choice = parsed.choices?.[0]; + const content = choice?.delta?.content; + const reasoningContent = choice?.delta?.reasoning_content; + const toolCalls = choice?.delta?.tool_calls; + const timings = parsed.timings; + const promptProgress = parsed.prompt_progress; + + const chunkModel = ChatService.extractModelName(parsed); + if (chunkModel && !modelEmitted) { + modelEmitted = true; + onModel?.(chunkModel); + } - if (timings) { - ChatService.notifyTimings(timings, promptProgress, onTimings); - lastTimings = timings; - } + if (parsed.id && !idEmitted) { + idEmitted = true; + onCompletionId?.(parsed.id); + } - if (content) { - finalizeOpenToolCallBatch(); - aggregatedContent += content; - if (!abortSignal?.aborted) { - onChunk?.(content); + if (promptProgress) { + ChatService.notifyTimings(undefined, promptProgress, onTimings); } - } - if (reasoningContent) { - finalizeOpenToolCallBatch(); - fullReasoningContent += reasoningContent; - if (!abortSignal?.aborted) { - onReasoningChunk?.(reasoningContent); + if (timings) { + ChatService.notifyTimings(timings, promptProgress, onTimings); + lastTimings = timings; + } + + if (content) { + finalizeOpenToolCallBatch(); + aggregatedContent += content; + if (!abortSignal?.aborted) { + onChunk?.(content); + } } - } - processToolCallDelta(toolCalls); - } catch (e) { - console.error('Error parsing JSON chunk:', e); + if (reasoningContent) { + finalizeOpenToolCallBatch(); + fullReasoningContent += reasoningContent; + if (!abortSignal?.aborted) { + onReasoningChunk?.(reasoningContent); + } + } + + processToolCallDelta(toolCalls); + } catch (e) { + console.error('Error parsing JSON chunk:', e); + } } } + + if (abortSignal?.aborted) break; + if (streamFinished) break; } + // inner reader done, decide whether to try a resume if (abortSignal?.aborted) break; + if (streamFinished) break; + if (!conversationId) break; + + if (!madeProgress) { + onConnectionState?.('lost'); + onError?.(new Error('Stream resume produced no new bytes, giving up')); + break; + } + + onConnectionState?.('resuming'); + madeProgress = false; + + // the server resends starting at bytesParsed, discard any partial line we held + // it will be retransmitted from a clean line boundary + const resumeResp = await resumeStream(conversationId, abortSignal).catch(() => null); + if (!resumeResp || resumeResp.status !== 200) { + onConnectionState?.('lost'); + onError?.(new Error('Stream connection lost and could not be resumed')); + break; + } + const newReader = resumeResp.body?.getReader(); + if (!newReader) break; + + try { + reader.releaseLock(); + } catch { + /* ignore */ + } + reader = newReader; + decoder = new TextDecoder(); + chunk = ''; + segmentStartOffset = bytesParsed; + segmentBytesRead = 0; + lastByteAt = Date.now(); } if (abortSignal?.aborted) return; @@ -718,6 +835,10 @@ export class ChatService { if (streamFinished) { finalizeOpenToolCallBatch(); + if (conversationId) { + clearStreamState(conversationId); + } + const finalToolCalls = aggregatedToolCalls.length > 0 ? JSON.stringify(aggregatedToolCalls) : undefined; @@ -735,7 +856,14 @@ export class ChatService { throw err; } finally { - reader.releaseLock(); + if (typeof document !== 'undefined') { + document.removeEventListener('visibilitychange', onVisibilityChange); + } + try { + reader.releaseLock(); + } catch { + /* ignore */ + } } } diff --git a/tools/ui/src/lib/services/mcp.service.ts b/tools/ui/src/lib/services/mcp.service.ts index 90de0d5d88ae..4432989171d1 100644 --- a/tools/ui/src/lib/services/mcp.service.ts +++ b/tools/ui/src/lib/services/mcp.service.ts @@ -628,19 +628,20 @@ export class MCPService { ); const runtimeErrorHandler = (error: Error) => { - // Ignore errors that are expected when the SDK's transport is closed, - // or when connecting to servers that don't support SSE (stateless-only - // endpoints returning 405). The SDK wraps the original AbortError in - // a new Error with the message "SSE stream disconnected: AbortError", - // and also produces "Cannot cancel a stream locked by a reader". - // DOMException is thrown by the browser when aborting fetch requests. - const msg = error.message || String(error); + // the SDK reports any post initialize error here, including the abort we trigger + // ourselves on the next health check cycle, on tab unload, or on server teardown. + // these are lifecycle aborts, not actionable errors, so we keep them out of the red console. + // the SDK wraps the original AbortError in a generic Error like + // "SSE stream disconnected: AbortError: The operation was aborted." + // which isAbortError cannot recognize by name alone, so we also pattern match on the message + if (isAbortError(error)) { + return; + } + const msg = error?.message ?? ''; if ( - error.name === 'AbortError' || - error instanceof DOMException || - msg.includes('SSE stream disconnected') || - msg.includes('stream locked by a reader') || - msg.includes('The operation was aborted') + /SSE stream disconnected:.*AbortError/i.test(msg) || + /AbortError: .*aborted/i.test(msg) || + /stream locked by a reader/i.test(msg) ) { return; } diff --git a/tools/ui/src/lib/services/stream-discovery.service.ts b/tools/ui/src/lib/services/stream-discovery.service.ts new file mode 100644 index 000000000000..3792e4ad0122 --- /dev/null +++ b/tools/ui/src/lib/services/stream-discovery.service.ts @@ -0,0 +1,28 @@ +import type { ApiStreamSession } from '$lib/types'; + +/** + * Pick the running session to splice into when discoverActiveStream lists candidates for + * a conversation. Finalized sessions are not candidates: their final content was already + * written to the DB by the original onComplete handler, so attaching to them would replay + * a buffer that may not match what the DB holds. In particular a continue session's buffer + * holds only the appended deltas, not the pre continue prefix, so replaying it as a fresh + * generation would erase the original assistant content. + * + * Among running sessions we tie break on the most recent started_at, which covers the + * pathological case of multiple inferences left running on the same conversation (eg user + * spawned two tabs). + * + * Returns null when no running session exists or the input is empty. + */ +export function selectActiveStream( + sessions: ApiStreamSession[] | null | undefined +): ApiStreamSession | null { + if (!Array.isArray(sessions) || sessions.length === 0) { + return null; + } + const running = sessions.filter((s) => !s.is_done); + if (running.length === 0) { + return null; + } + return running.reduce((best, cur) => (cur.started_at > best.started_at ? cur : best)); +} diff --git a/tools/ui/src/lib/services/stream-resume.service.ts b/tools/ui/src/lib/services/stream-resume.service.ts new file mode 100644 index 000000000000..5c3664fb3078 --- /dev/null +++ b/tools/ui/src/lib/services/stream-resume.service.ts @@ -0,0 +1,76 @@ +/** + * Stream resume persistence and reconnection helper. + * + * Tracks the running byte count for an in flight streaming generation per + * conversation_id, so a later visit can resume the SSE replay at the right + * offset. The conversation_id is the session identity end to end (server map, + * client localStorage, /v1/stream/ routes), no extra opaque token. + */ + +interface ResumableStreamState { + bytesReceived: number; + updatedAt: number; +} + +const STORAGE_PREFIX = 'llamacpp.stream.resume.'; + +function storageKey(conversationId: string): string { + return STORAGE_PREFIX + conversationId; +} + +export function saveStreamState(conversationId: string, bytesReceived: number): void { + if (!conversationId) return; + try { + const state: ResumableStreamState = { + bytesReceived, + updatedAt: Date.now() + }; + localStorage.setItem(storageKey(conversationId), JSON.stringify(state)); + } catch { + // localStorage may be full or disabled, silently ignore + } +} + +export function getStreamState(conversationId: string): ResumableStreamState | null { + if (!conversationId) return null; + try { + const raw = localStorage.getItem(storageKey(conversationId)); + if (!raw) return null; + const parsed = JSON.parse(raw) as ResumableStreamState; + if (!parsed || typeof parsed.bytesReceived !== 'number') return null; + return parsed; + } catch { + return null; + } +} + +export function clearStreamState(conversationId: string): void { + if (!conversationId) return; + try { + localStorage.removeItem(storageKey(conversationId)); + } catch { + // nothing to do + } +} + +/** + * Reconnect to an interrupted stream for this conversation. Returns the fetch + * Response so the existing SSE parser can drain it just like a fresh stream. + * The caller is expected to feed the running byte count back via + * saveStreamState as more data flows. + * + * The server returns 200 with text/event-stream on success, 404 if no session + * exists for the conv_id (already evicted or never created), and 400 if the + * requested offset is below the dropped prefix (buffer cap was hit and head + * bytes were lost). + */ +export async function resumeStream( + conversationId: string, + signal?: AbortSignal +): Promise { + if (!conversationId) return null; + const state = getStreamState(conversationId); + const from = state?.bytesReceived ?? 0; + const url = `./v1/stream/${encodeURIComponent(conversationId)}?from=${from}`; + return await fetch(url, { method: 'GET', signal }); +} diff --git a/tools/ui/src/lib/stores/agentic.svelte.ts b/tools/ui/src/lib/stores/agentic.svelte.ts index 5579cc1e5a64..27491257ac72 100644 --- a/tools/ui/src/lib/stores/agentic.svelte.ts +++ b/tools/ui/src/lib/stores/agentic.svelte.ts @@ -614,7 +614,7 @@ class AgenticStore { throw error; } }, - undefined, + conversationId, signal ); diff --git a/tools/ui/src/lib/stores/chat.svelte.ts b/tools/ui/src/lib/stores/chat.svelte.ts index b899130e50a2..dd4995efc0ae 100644 --- a/tools/ui/src/lib/stores/chat.svelte.ts +++ b/tools/ui/src/lib/stores/chat.svelte.ts @@ -11,9 +11,11 @@ * @see ChatService in services/chat.service.ts for API operations */ -import { SvelteMap } from 'svelte/reactivity'; +import { SvelteMap, SvelteSet } from 'svelte/reactivity'; import { DatabaseService } from '$lib/services/database.service'; import { ChatService } from '$lib/services/chat.service'; +import { selectActiveStream } from '$lib/services/stream-discovery.service'; +import { getStreamState, clearStreamState } from '$lib/services/stream-resume.service'; import { conversationsStore } from '$lib/stores/conversations.svelte'; import { config } from '$lib/stores/settings.svelte'; import { agenticStore } from '$lib/stores/agentic.svelte'; @@ -49,8 +51,10 @@ import type { import type { ApiChatMessageData, ApiProcessingState, + ApiStreamSession, DatabaseMessage, - DatabaseMessageExtra + DatabaseMessageExtra, + StreamConnectionState } from '$lib/types'; import { ContinueIntentKind, ErrorDialogType, MessageRole, MessageType } from '$lib/enums'; @@ -65,9 +69,16 @@ class ChatStore { isLoading = $state(false); // true while the active conversation streams reasoning content but no visible content yet isReasoning = $state(false); + // resumable stream connection state for the active conversation + // streaming -> bytes flowing normally, resuming -> waiting on /v1/stream/:id reconnect, lost -> unrecoverable + streamConnectionState = $state('streaming'); chatLoadingStates = new SvelteMap(); chatReasoningStates = new SvelteMap(); chatStreamingStates = new SvelteMap(); + // convs that the backend reports as having a running session, populated by the global sync + // at app mount and on visibilitychange. it does not overlap with chatLoadingStates which + // tracks inferences driven by this browser, both are unioned to feed the sidebar spinners + private remoteRunningConvs = new SvelteSet(); private abortControllers = new SvelteMap(); private preEncodeAbortController: AbortController | null = null; private processingStates = new SvelteMap(); @@ -98,6 +109,11 @@ class ChatStore { this.chatLoadingStates.delete(convId); if (convId === conversationsStore.activeConversation?.id) this.isLoading = false; this.setChatReasoning(convId, false); + // the local pipe is the authoritative observer of session end: when it finishes (clean + // onComplete or explicit Stop), the backend session is finalized too, so we drop the + // sidebar hint for this conv right away instead of waiting for the next visibilitychange + // snapshot. without this the spinner ghosts until the user toggles the tab + this.remoteRunningConvs.delete(convId); } } @@ -137,6 +153,248 @@ class ChatStore { } } } + /** + * Server side stream discovery, split in three pieces: + * + * probeServerStream(convId) -> hits GET /v1/streams?conversation_id, returns the session to attach + * to or null. Pure read, no side effect, no UI lock. Safe to fire in parallel with anything. + * + * attachServerStream(convId) -> flips the spinner immediately, fetches the replay stream + * from byte 0, finds the assistant slot to splice into (creates a placeholder if the conv has + * no assistant message yet, for cross device or fresh local DB cases), and pipes the SSE bytes + * into the message via handleStreamResponse. + * + * discoverActiveStream(convId) -> probe + attach in one call. Used by callers that do not need + * to overlap the probe with other async work. + * + * The mount of the chat page in +page.svelte calls probeServerStream in parallel with + * loadConversation, then attachServerStream once both have settled. This gives the earliest + * possible time to spinner and avoids racing against an empty activeMessages array. + */ + async probeServerStream(convId: string): Promise { + if (!convId) return null; + let listResp: Response; + try { + listResp = await fetch(`./v1/streams?conversation_id=${encodeURIComponent(convId)}`); + } catch (e) { + console.warn('probeServerStream fetch failed:', e); + return null; + } + if (!listResp.ok) { + console.warn(`probeServerStream got HTTP ${listResp.status} for conv ${convId}`); + return null; + } + let sessions: ApiStreamSession[]; + try { + sessions = (await listResp.json()) as ApiStreamSession[]; + } catch (e) { + console.warn('probeServerStream JSON parse failed:', e); + return null; + } + return selectActiveStream(sessions); + } + + async attachServerStream(convId: string): Promise { + if (!convId) return; + if (this.chatStreamingStates.has(convId)) return; + + // flip the spinner immediately, the user sees activity as soon as the conv becomes active + this.setChatLoading(convId, true); + this.setStreamingActive(true); + this.setActiveProcessingConversation(convId); + + const unlock = () => { + this.setStreamingActive(false); + this.setChatLoading(convId, false); + this.clearChatStreaming(convId); + }; + + // fetch the replay stream from byte 0, rebuild the assistant message from scratch. + // the conv id is the only identifier we need, end to end + let response: Response; + try { + response = await fetch(`./v1/stream/${encodeURIComponent(convId)}?from=0`); + } catch (e) { + console.error('attachServerStream replay fetch failed:', e); + unlock(); + return; + } + if (!response.ok) { + console.warn(`attachServerStream replay got HTTP ${response.status} for conv ${convId}`); + unlock(); + return; + } + + // locate the slot to splice into, create a placeholder assistant message if there is none + let messages = conversationsStore.activeMessages as DatabaseMessage[]; + let targetIdx = this.findLastAssistantIdx(messages); + if (targetIdx === -1) { + const lastUserIdx = this.findLastUserIdx(messages); + if (lastUserIdx === -1) { + console.warn( + `attachServerStream: conv ${convId} has no user or assistant message, cannot splice` + ); + unlock(); + return; + } + try { + const placeholder = await DatabaseService.createMessageBranch( + { + convId, + role: MessageRole.ASSISTANT, + content: '', + type: MessageType.TEXT, + timestamp: Date.now(), + parent: messages[lastUserIdx].id, + children: [], + toolCalls: '' + } as Omit, + messages[lastUserIdx].id + ); + conversationsStore.addMessageToActive(placeholder); + messages = conversationsStore.activeMessages as DatabaseMessage[]; + targetIdx = this.findLastAssistantIdx(messages); + } catch (e) { + console.error('attachServerStream placeholder creation failed:', e); + unlock(); + return; + } + } + if (targetIdx === -1) { + unlock(); + return; + } + const targetMessage = messages[targetIdx]; + const targetMessageId = targetMessage.id; + // when the assistant slot already has content, the running session is a continue or + // another append flow and its buffer holds only the appended deltas. preserve the prefix + // and let the replay add to it. when the slot is empty the session buffer holds the whole + // message so we wipe and rebuild from byte 0 + const existingContent = targetMessage.content ?? ''; + const existingReasoning = targetMessage.reasoningContent ?? ''; + const isAppendMode = existingContent.length > 0; + if (!isAppendMode) { + conversationsStore.updateMessageAtIndex(targetIdx, { + content: '', + reasoningContent: undefined + }); + } + + this.setChatStreaming(convId, existingContent, targetMessageId); + const abortController = this.getOrCreateAbortController(convId); + + let streamedContent = ''; + let streamedReasoningContent = ''; + + const cleanup = () => { + unlock(); + this.setProcessingState(convId, null); + }; + + try { + await ChatService.handleStreamResponse( + response, + (chunk: string) => { + streamedContent += chunk; + const displayed = isAppendMode ? existingContent + streamedContent : streamedContent; + conversationsStore.updateMessageAtIndex(targetIdx, { content: displayed }); + this.setChatStreaming(convId, displayed, targetMessageId); + }, + async ( + finalContent?: string, + reasoningContent?: string, + timings?: ChatMessageTimings, + toolCalls?: string + ) => { + const streamed = streamedContent || finalContent || ''; + const streamedR = streamedReasoningContent || reasoningContent || ''; + const content = isAppendMode ? existingContent + streamed : streamed; + const reasoning = isAppendMode ? existingReasoning + streamedR : streamedR; + await DatabaseService.updateMessage(targetMessageId, { + content, + reasoningContent: reasoning || undefined, + toolCalls: toolCalls || '', + timings + }); + conversationsStore.updateMessageAtIndex(targetIdx, { + content, + reasoningContent: reasoning || undefined, + timings + }); + cleanup(); + }, + (err: Error) => { + console.error('attachServerStream pipe error:', err); + cleanup(); + }, + (chunk: string) => { + streamedReasoningContent += chunk; + const displayed = isAppendMode + ? existingReasoning + streamedReasoningContent + : streamedReasoningContent; + conversationsStore.updateMessageAtIndex(targetIdx, { + reasoningContent: displayed + }); + }, + undefined, + undefined, + undefined, + convId, + abortController.signal, + (connState: StreamConnectionState) => { + if (convId === conversationsStore.activeConversation?.id) { + this.streamConnectionState = connState; + } + } + ); + } catch (e) { + console.error('attachServerStream pipe crashed:', e); + cleanup(); + } + } + + async discoverActiveStream(convId: string): Promise { + if (!convId) return; + if (this.chatStreamingStates.has(convId)) return; + if (this.chatLoadingStates.get(convId)) return; + // the sidebar spinner hint is consumed the moment we run the authoritative probe, so a + // finalized session no longer ghosts in the sidebar after navigation + this.remoteRunningConvs.delete(convId); + + // primary path: ask the server which sessions exist for this conversation + const serverTarget = await this.probeServerStream(convId); + if (serverTarget) { + await this.attachServerStream(convId); + return; + } + + // fallback: local state remembers an interrupted byte offset for this conv, the server may + // still have a live session matching that conv id (we just lost the bytes mid stream). try to + // attach with conv id only, the server probe inside attachServerStream tells us if it exists + const localState = getStreamState(convId); + if (!localState) { + return; + } + await this.attachServerStream(convId); + // if attachServerStream failed (session gone, TTL expired), clear the local state to avoid retrying forever + if (!this.chatStreamingStates.has(convId) && !this.chatLoadingStates.get(convId)) { + clearStreamState(convId); + } + } + + private findLastAssistantIdx(messages: DatabaseMessage[]): number { + for (let i = messages.length - 1; i >= 0; i--) { + if (messages[i].role === MessageRole.ASSISTANT) return i; + } + return -1; + } + + private findLastUserIdx(messages: DatabaseMessage[]): number { + for (let i = messages.length - 1; i >= 0; i--) { + if (messages[i].role === MessageRole.USER) return i; + } + return -1; + } clearUIState(): void { this.isLoading = false; @@ -265,13 +523,52 @@ class ChatStore { } getAllLoadingChats(): string[] { - return Array.from(this.chatLoadingStates.keys()); + // union of local (this browser is piping) and remote (backend reports a running session + // for this conv but no local pipe yet) sources. the sidebar shows one spinner per entry + const out = new SvelteSet(this.chatLoadingStates.keys()); + for (const id of this.remoteRunningConvs) { + out.add(id); + } + return Array.from(out); } getAllStreamingChats(): string[] { return Array.from(this.chatStreamingStates.keys()); } + /** + * Resync the remote running convs set from the backend. Called by the layout at mount and on + * visibilitychange, no polling. A snapshot semantic: the set is replaced wholesale, stale entries + * for sessions that finalized while the browser was elsewhere are dropped naturally. + */ + async syncRemoteRunningStreams(): Promise { + let sessions: ApiStreamSession[]; + try { + const resp = await fetch('./v1/streams'); + if (!resp.ok) return; + const body = (await resp.json()) as unknown; + if (!Array.isArray(body)) return; + sessions = body as ApiStreamSession[]; + } catch (e) { + console.warn('syncRemoteRunningStreams fetch failed:', e); + return; + } + const running = new SvelteSet(); + for (const s of sessions) { + if (s && !s.is_done && typeof s.conversation_id === 'string' && s.conversation_id) { + running.add(s.conversation_id); + } + } + for (const id of Array.from(this.remoteRunningConvs)) { + if (!running.has(id)) { + this.remoteRunningConvs.delete(id); + } + } + for (const id of running) { + this.remoteRunningConvs.add(id); + } + } + getChatStreamingPublic(convId: string): { response: string; messageId: string } | undefined { return this.getChatStreaming(convId); } @@ -922,6 +1219,11 @@ class ChatStore { onModel: streamCallbacks.onModel, onCompletionId: streamCallbacks.onCompletionId, onTimings: streamCallbacks.onTimings, + onConnectionState: (state: StreamConnectionState) => { + if (convId === conversationsStore.activeConversation?.id) { + this.streamConnectionState = state; + } + }, onComplete: async ( finalContent?: string, reasoningContent?: string, @@ -979,6 +1281,10 @@ class ChatStore { async stopGenerationForChat(convId: string): Promise { await this.savePartialResponseIfNeeded(convId); this.setStreamingActive(false); + // tell the server to stop the generation, not just to drop the HTTP socket. without this + // the detached drain keeps producing tokens until eos or max_tokens. the conv id is the + // session identity so the DELETE call is straight + void ChatService.cancelServerStream(convId); this.abortRequest(convId); this.setChatLoading(convId, false); this.clearChatStreaming(convId); @@ -1403,6 +1709,11 @@ class ChatStore { { ...this.getApiOptions(), continueFinalMessage: true, + onConnectionState: (state: StreamConnectionState) => { + if (msg.convId === conversationsStore.activeConversation?.id) { + this.streamConnectionState = state; + } + }, onChunk: (chunk: string) => { appendedContent += chunk; hasReceivedContent = true; diff --git a/tools/ui/src/lib/types/api.d.ts b/tools/ui/src/lib/types/api.d.ts index 2a2524d00237..68fde6e60392 100644 --- a/tools/ui/src/lib/types/api.d.ts +++ b/tools/ui/src/lib/types/api.d.ts @@ -512,3 +512,17 @@ export interface ApiRouterModelsUnloadResponse { success: boolean; error?: string; } + +/** + * Entry returned by GET /v1/streams (optional conversation_id query filter). One entry per + * live or recently completed background streaming session, keyed by its conversation_id. + * The WebUI uses this at mount and on visibilitychange to populate sidebar spinners and to + * reattach to an ongoing inference for the active conversation. + */ +export interface ApiStreamSession { + conversation_id: string; + is_done: boolean; + total_bytes: number; + started_at: number; + completed_at: number; +} diff --git a/tools/ui/src/lib/types/index.ts b/tools/ui/src/lib/types/index.ts index 9b0b118045a4..26b5f6c98fb0 100644 --- a/tools/ui/src/lib/types/index.ts +++ b/tools/ui/src/lib/types/index.ts @@ -34,7 +34,8 @@ export type { ApiRouterModelsListResponse, ApiRouterModelsUnloadRequest, ApiRouterModelsUnloadResponse, - AudioInputFormat + AudioInputFormat, + ApiStreamSession } from './api'; // Chat types @@ -86,6 +87,7 @@ export type { SettingsConfigValue, SettingsFieldConfig, SettingsChatServiceOptions, + StreamConnectionState, SettingsConfigType, SettingsExportType, ParameterValue, diff --git a/tools/ui/src/lib/types/settings.d.ts b/tools/ui/src/lib/types/settings.d.ts index d1cdca957cff..b181c6825418 100644 --- a/tools/ui/src/lib/types/settings.d.ts +++ b/tools/ui/src/lib/types/settings.d.ts @@ -119,8 +119,15 @@ export interface SettingsChatServiceOptions { toolCalls?: string ) => void; onError?: (error: Error) => void; + onConnectionState?: (state: StreamConnectionState) => void; } +// Connection lifecycle for resumable streams +// streaming: data is flowing from the initial POST or a successful resume GET +// resuming : the server connection was lost and the client is attempting a resume +// lost : the stream is unrecoverable, the user must restart +export type StreamConnectionState = 'streaming' | 'resuming' | 'lost'; + export type SettingsConfigType = typeof SETTING_CONFIG_DEFAULT & { [key: string]: SettingsConfigValue; }; diff --git a/tools/ui/src/lib/utils/abort.ts b/tools/ui/src/lib/utils/abort.ts index fc4f31ec6941..e1771d92c948 100644 --- a/tools/ui/src/lib/utils/abort.ts +++ b/tools/ui/src/lib/utils/abort.ts @@ -51,8 +51,19 @@ export function isAbortError(error: unknown): boolean { if (error instanceof DOMException && error.name === 'AbortError') { return true; } - if (error instanceof Error && error.name === 'AbortError') { - return true; + if (error instanceof Error) { + if (error.name === 'AbortError') { + return true; + } + // browser specific patterns emitted when a fetch reader is interrupted by page + // unload, navigation, or transient network drop. these are functionally aborts, + // not actionable application errors, so they should not surface as red console logs + if (error instanceof TypeError) { + const msg = error.message ?? ''; + if (/input stream/i.test(msg)) return true; // Firefox: stream cut at unload + if (/network connection was lost/i.test(msg)) return true; // Safari: transient network drop + if (/load failed/i.test(msg)) return true; // Safari: page navigation during fetch + } } return false; } diff --git a/tools/ui/src/routes/(chat)/chat/[id]/+page.svelte b/tools/ui/src/routes/(chat)/chat/[id]/+page.svelte index e31d4443ef39..f14553c90a90 100644 --- a/tools/ui/src/routes/(chat)/chat/[id]/+page.svelte +++ b/tools/ui/src/routes/(chat)/chat/[id]/+page.svelte @@ -4,7 +4,7 @@ import { afterNavigate } from '$app/navigation'; import { DialogModelNotAvailable } from '$lib/components/app'; import { APP_NAME, ROUTES } from '$lib/constants'; - import { chatStore, isLoading } from '$lib/stores/chat.svelte'; + import { chatStore } from '$lib/stores/chat.svelte'; import { conversationsStore, activeConversation } from '$lib/stores/conversations.svelte'; import { modelsStore, modelOptions } from '$lib/stores/models.svelte'; @@ -83,7 +83,7 @@ // Skip loading if this conversation is already active (e.g., just created) if (activeConversation()?.id === chatId) { - // Still handle URL params even if conversation is active + void chatStore.discoverActiveStream(chatId); if ((qParam !== null || modelParam !== null) && !urlParamsProcessed) { handleUrlParams(); } @@ -92,35 +92,33 @@ (async () => { const success = await conversationsStore.loadConversation(chatId); - if (success) { - chatStore.syncLoadingStateForChat(chatId); - - // Handle URL params after conversation is loaded - if ((qParam !== null || modelParam !== null) && !urlParamsProcessed) { - await handleUrlParams(); - } - } else { + if (!success) { await goto(ROUTES.START); + return; + } + chatStore.syncLoadingStateForChat(chatId); + // server probe (with localStorage fallback) and attach + await chatStore.discoverActiveStream(chatId); + + if ((qParam !== null || modelParam !== null) && !urlParamsProcessed) { + await handleUrlParams(); } })(); } }); $effect(() => { - if (typeof window !== 'undefined') { - const handleBeforeUnload = () => { - if (isLoading()) { - console.log('Page unload detected while streaming - aborting stream'); - chatStore.stopGeneration(); - } - }; - - window.addEventListener('beforeunload', handleBeforeUnload); - - return () => { - window.removeEventListener('beforeunload', handleBeforeUnload); - }; - } + if (typeof window === 'undefined' || typeof document === 'undefined') return; + + // when the tab comes back to the foreground, re-run discovery to catch any race + // where the initial mount probe missed an active session + const onVisibility = () => { + if (document.visibilityState !== 'visible') return; + if (!chatId) return; + void chatStore.discoverActiveStream(chatId); + }; + document.addEventListener('visibilitychange', onVisibility); + return () => document.removeEventListener('visibilitychange', onVisibility); }); diff --git a/tools/ui/src/routes/+layout.svelte b/tools/ui/src/routes/+layout.svelte index 339b067dd2c1..0bedc725db0c 100644 --- a/tools/ui/src/routes/+layout.svelte +++ b/tools/ui/src/routes/+layout.svelte @@ -11,6 +11,7 @@ import { PwaMetaTags, PwaRefreshAlert } from '$lib/components/pwa'; import { pwaAssetsHead } from 'virtual:pwa-assets/head'; + import { chatStore } from '$lib/stores/chat.svelte'; import { conversationsStore } from '$lib/stores/conversations.svelte'; import * as Tooltip from '$lib/components/ui/tooltip'; import { isRouterMode, serverStore } from '$lib/stores/server.svelte'; @@ -154,6 +155,18 @@ onMount(() => { updateFavicon(); + // global snapshot of backend running streams. populates the sidebar spinners so the user + // sees at a glance every conv that has a live inference, even ones not yet opened. snapshot + // only, no polling: refresh happens on mount and on visibilitychange via the effect below + void chatStore.syncRemoteRunningStreams(); + + if (typeof document === 'undefined') return; + const onVisibility = () => { + if (document.visibilityState !== 'visible') return; + void chatStore.syncRemoteRunningStreams(); + }; + document.addEventListener('visibilitychange', onVisibility); + return () => document.removeEventListener('visibilitychange', onVisibility); }); $effect(() => { diff --git a/tools/ui/tests/unit/abort.test.ts b/tools/ui/tests/unit/abort.test.ts new file mode 100644 index 000000000000..306e71ca15d7 --- /dev/null +++ b/tools/ui/tests/unit/abort.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from 'vitest'; +import { isAbortError } from '$lib/utils/abort'; + +describe('isAbortError', () => { + it('returns false for null, undefined and non-error values', () => { + expect(isAbortError(null)).toBe(false); + expect(isAbortError(undefined)).toBe(false); + expect(isAbortError('string error')).toBe(false); + expect(isAbortError({ name: 'AbortError' })).toBe(false); + expect(isAbortError(42)).toBe(false); + }); + + it('returns true for DOMException with AbortError name', () => { + const err = new DOMException('Operation was aborted', 'AbortError'); + expect(isAbortError(err)).toBe(true); + }); + + it('returns true for plain Error with AbortError name', () => { + const err = new Error('aborted'); + err.name = 'AbortError'; + expect(isAbortError(err)).toBe(true); + }); + + it('returns false for unrelated Error instances', () => { + expect(isAbortError(new Error('something failed'))).toBe(false); + expect(isAbortError(new TypeError('not related'))).toBe(false); + expect(isAbortError(new RangeError('out of range'))).toBe(false); + }); + + it('recognizes Firefox TypeError "Error in input stream" emitted at page unload', () => { + expect(isAbortError(new TypeError('Error in input stream'))).toBe(true); + expect(isAbortError(new TypeError('TypeError: Error in input stream'))).toBe(true); + }); + + it('recognizes Safari "The network connection was lost" during transient drop', () => { + expect(isAbortError(new TypeError('The network connection was lost.'))).toBe(true); + }); + + it('recognizes Safari "Load failed" during page navigation', () => { + expect(isAbortError(new TypeError('Load failed'))).toBe(true); + }); + + it('does NOT recognize generic TypeError messages as aborts', () => { + // matching too broadly would hide real bugs, the predicate must stay conservative + expect(isAbortError(new TypeError('Failed to fetch'))).toBe(false); + expect(isAbortError(new TypeError('Cannot read property of undefined'))).toBe(false); + expect(isAbortError(new TypeError('NetworkError when attempting to fetch resource'))).toBe( + false + ); + }); + + it('is case insensitive on the matched substrings', () => { + expect(isAbortError(new TypeError('error in INPUT STREAM'))).toBe(true); + expect(isAbortError(new TypeError('the network connection WAS LOST'))).toBe(true); + }); +}); diff --git a/tools/ui/tests/unit/stream-discovery.test.ts b/tools/ui/tests/unit/stream-discovery.test.ts new file mode 100644 index 000000000000..33027afdf08b --- /dev/null +++ b/tools/ui/tests/unit/stream-discovery.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, it } from 'vitest'; +import { selectActiveStream } from '$lib/services/stream-discovery.service'; +import type { ApiStreamSession } from '$lib/types'; + +function makeSession(overrides: Partial): ApiStreamSession { + return { + conversation_id: 'conv', + is_done: true, + total_bytes: 0, + started_at: 0, + completed_at: 0, + ...overrides + }; +} + +describe('selectActiveStream', () => { + it('returns null on empty input', () => { + expect(selectActiveStream([])).toBeNull(); + }); + + it('returns null on null or undefined input', () => { + expect(selectActiveStream(null)).toBeNull(); + expect(selectActiveStream(undefined)).toBeNull(); + }); + + it('returns the single session when it is running', () => { + const s = makeSession({ conversation_id: 'only', is_done: false, started_at: 42 }); + expect(selectActiveStream([s])).toBe(s); + }); + + it('returns null when the single session is finalized', () => { + const s = makeSession({ conversation_id: 'only', is_done: true, started_at: 42 }); + expect(selectActiveStream([s])).toBeNull(); + }); + + it('prefers a still running session over a finalized one regardless of started_at', () => { + const finalized = makeSession({ conversation_id: 'old', is_done: true, started_at: 1000 }); + const running = makeSession({ conversation_id: 'new', is_done: false, started_at: 10 }); + expect(selectActiveStream([finalized, running])?.conversation_id).toBe('new'); + expect(selectActiveStream([running, finalized])?.conversation_id).toBe('new'); + }); + + it('among running sessions, picks the most recently started one', () => { + const a = makeSession({ conversation_id: 'a', is_done: false, started_at: 100 }); + const b = makeSession({ conversation_id: 'b', is_done: false, started_at: 200 }); + const c = makeSession({ conversation_id: 'c', is_done: false, started_at: 150 }); + expect(selectActiveStream([a, b, c])?.conversation_id).toBe('b'); + expect(selectActiveStream([c, a, b])?.conversation_id).toBe('b'); + }); + + it('returns null when all sessions are finalized, the DB already holds the content', () => { + const a = makeSession({ conversation_id: 'a', is_done: true, started_at: 10 }); + const b = makeSession({ conversation_id: 'b', is_done: true, started_at: 30 }); + const c = makeSession({ conversation_id: 'c', is_done: true, started_at: 20 }); + expect(selectActiveStream([a, b, c])).toBeNull(); + }); + + it('keeps the first match on ties when both are running with identical started_at', () => { + // reduce visits left to right, the initial accumulator stays unless a strictly greater value appears + const a = makeSession({ conversation_id: 'first', is_done: false, started_at: 50 }); + const b = makeSession({ conversation_id: 'second', is_done: false, started_at: 50 }); + expect(selectActiveStream([a, b])?.conversation_id).toBe('first'); + }); + + it('handles a typical realistic mix: two finalized old, one freshly running, one freshly finalized', () => { + const old1 = makeSession({ conversation_id: 'old1', is_done: true, started_at: 100 }); + const old2 = makeSession({ conversation_id: 'old2', is_done: true, started_at: 200 }); + const freshFin = makeSession({ conversation_id: 'freshFin', is_done: true, started_at: 500 }); + const running = makeSession({ conversation_id: 'running', is_done: false, started_at: 400 }); + expect(selectActiveStream([old1, old2, freshFin, running])?.conversation_id).toBe('running'); + }); +}); diff --git a/tools/ui/tests/unit/stream-resume.test.ts b/tools/ui/tests/unit/stream-resume.test.ts new file mode 100644 index 000000000000..715d5d6c77d0 --- /dev/null +++ b/tools/ui/tests/unit/stream-resume.test.ts @@ -0,0 +1,78 @@ +import { afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest'; + +// node env unit project has no DOM, install a minimal localStorage backed by a Map +beforeAll(() => { + const store = new Map(); + const polyfill: Storage = { + get length() { + return store.size; + }, + clear: () => store.clear(), + getItem: (k) => (store.has(k) ? store.get(k)! : null), + key: (i) => Array.from(store.keys())[i] ?? null, + removeItem: (k) => { + store.delete(k); + }, + setItem: (k, v) => { + store.set(k, String(v)); + } + }; + (globalThis as unknown as { localStorage: Storage }).localStorage = polyfill; +}); + +import { + saveStreamState, + getStreamState, + clearStreamState +} from '$lib/services/stream-resume.service'; + +describe('stream-resume.service', () => { + beforeEach(() => { + localStorage.clear(); + }); + afterEach(() => { + localStorage.clear(); + }); + + it('returns null when no state exists for the conversation', () => { + expect(getStreamState('conv-a')).toBeNull(); + }); + + it('saves and reads back the byte count', () => { + saveStreamState('conv-a', 4242); + const got = getStreamState('conv-a'); + expect(got).not.toBeNull(); + expect(got!.bytesReceived).toBe(4242); + expect(typeof got!.updatedAt).toBe('number'); + }); + + it('overwrites the previous byte count on a new save for the same conversation', () => { + saveStreamState('conv-a', 100); + saveStreamState('conv-a', 200); + const got = getStreamState('conv-a'); + expect(got!.bytesReceived).toBe(200); + }); + + it('keeps states for distinct conversations isolated', () => { + saveStreamState('conv-a', 10); + saveStreamState('conv-b', 20); + expect(getStreamState('conv-a')!.bytesReceived).toBe(10); + expect(getStreamState('conv-b')!.bytesReceived).toBe(20); + }); + + it('clears the state for a given conversation', () => { + saveStreamState('conv-a', 10); + clearStreamState('conv-a'); + expect(getStreamState('conv-a')).toBeNull(); + }); + + it('ignores empty conversation id on save', () => { + saveStreamState('', 1); + expect(getStreamState('')).toBeNull(); + }); + + it('returns null on corrupted storage payload', () => { + localStorage.setItem('llamacpp.stream.resume.conv-a', '{not-json'); + expect(getStreamState('conv-a')).toBeNull(); + }); +}); From adff9a580ebdcc8f5b465c7999ca12b76407ffd9 Mon Sep 17 00:00:00 2001 From: Pascal Date: Sun, 17 May 2026 10:19:02 +0200 Subject: [PATCH 02/39] server: create stream session only after post_tasks succeeds --- tools/server/server-context.cpp | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 07441f1b4145..a792b084827e 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -4186,14 +4186,10 @@ std::unique_ptr server_routes::handle_completions_impl( // an opt in without conv id falls back silently to the regular non resumable streaming path const bool resumable = resumable_hdr && !conversation_id.empty(); std::unique_ptr drain_reader; - stream_session_ptr session; server_response_reader * post_target = &rd; if (resumable) { drain_reader = std::make_unique( queue_tasks, queue_results, HTTP_POLLING_SECONDS); - // create_or_replace evicts and cancels any prior session on this conv, - // guaranteeing the invariant that at most one live session exists per conv - session = ctx_server.stream_sessions.create_or_replace(conversation_id); post_target = drain_reader.get(); } @@ -4270,6 +4266,15 @@ std::unique_ptr server_routes::handle_completions_impl( return res; } + // session creation comes after post_tasks succeeds, so a throw during parsing or task + // construction returns early without leaving an orphan in the manager map. + // create_or_replace evicts and cancels any prior session on this conv, the invariant + // 'one live session per conv' holds + stream_session_ptr session; + if (resumable) { + session = ctx_server.stream_sessions.create_or_replace(conversation_id); + } + if (!stream) { // non-stream, wait for the results auto all_results = rd.wait_for_all(req.should_stop); From 026b654c5bf364710aa2c2340d29ab74ed3c1f5f Mon Sep 17 00:00:00 2001 From: Pascal Date: Sun, 17 May 2026 21:02:46 +0200 Subject: [PATCH 03/39] server, ui: drop X-Stream-Resume, X-Conversation-Id alone enables the replay buffer --- tools/server/server-context.cpp | 30 ++++++----------------- tools/server/server-models.cpp | 19 +++++--------- tools/ui/src/lib/services/chat.service.ts | 9 +++---- 3 files changed, 16 insertions(+), 42 deletions(-) diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index a792b084827e..b9697cd9e642 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -4145,29 +4145,15 @@ std::unique_ptr server_routes::handle_completions_impl( auto & rd = res->rd; auto & params = this->params; - // detect background streaming opt-in via X-Stream-Resume: 1 header. - // when set together with a non empty X-Conversation-Id, the generation survives HTTP disconnect - // and can be resumed via GET /v1/stream/. only meaningful for streaming requests, - // non stream OAI calls keep the standard flow - bool stream = json_value(data, "stream", false); - bool resumable_hdr = false; + // resumable mode opt in: a non empty X-Conversation-Id on a streaming request enables it. + // the conv id is the session identity end to end (client localStorage, server map, + // /v1/stream/ routes). non stream OAI calls keep the standard flow regardless + bool stream = json_value(data, "stream", false); std::string conversation_id; if (stream) { // request headers preserve the wire casing, the scan is case insensitive - // we capture two headers in a single pass: X-Stream-Resume and X-Conversation-Id for (const auto & [hk, hv] : req.headers) { - if (hk.size() == 15) { - bool match = true; - static const char target[] = "x-stream-resume"; - for (size_t i = 0; i < 15; ++i) { - char c = hk[i]; - if (c >= 'A' && c <= 'Z') c = char(c + 32); - if (c != target[i]) { match = false; break; } - } - if (match && hv == "1") { - resumable_hdr = true; - } - } else if (hk.size() == 17) { + if (hk.size() == 17) { bool match = true; static const char target[] = "x-conversation-id"; for (size_t i = 0; i < 17; ++i) { @@ -4177,14 +4163,12 @@ std::unique_ptr server_routes::handle_completions_impl( } if (match) { conversation_id = hv; + break; } } } } - // resumable mode requires both the opt in header and a conversation id, the conv id is the - // session identity end to end (client localStorage, server map, /v1/stream/ routes). - // an opt in without conv id falls back silently to the regular non resumable streaming path - const bool resumable = resumable_hdr && !conversation_id.empty(); + const bool resumable = !conversation_id.empty(); std::unique_ptr drain_reader; server_response_reader * post_target = &rd; if (resumable) { diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index d4516f3dc2fd..16ea5f304b55 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -1280,13 +1280,12 @@ server_http_res_ptr server_models::proxy_request(const server_http_req & req, co std::unique_lock lk(mutex); mapping[name].meta.last_used = ggml_time_ms(); } - // when the client opts in to resumable streaming (X-Stream-Resume: 1 + non empty - // X-Conversation-Id), fan out a DELETE on every other ready child to evict any prior - // session for this conv. enforces the cross child invariant 'one session per convId', - // safe to call unconditionally and cheap on loopback. ignored for any other request shape + // when the client opts in to resumable streaming via a non empty X-Conversation-Id, + // fan out a DELETE on every other ready child to evict any prior session for this conv. + // enforces the cross child invariant 'one session per convId', safe to call unconditionally + // and cheap on loopback. ignored for any request without that header { std::string conv_id; - bool resume_opt_in = false; for (const auto & [hk, hv] : req.headers) { if (hk.size() == 17) { std::string lower(hk); @@ -1294,17 +1293,11 @@ server_http_res_ptr server_models::proxy_request(const server_http_req & req, co [](unsigned char c) { return (c >= 'A' && c <= 'Z') ? char(c + 32) : char(c); }); if (lower == "x-conversation-id") { conv_id = hv; - } - } else if (hk.size() == 15) { - std::string lower(hk); - std::transform(lower.begin(), lower.end(), lower.begin(), - [](unsigned char c) { return (c >= 'A' && c <= 'Z') ? char(c + 32) : char(c); }); - if (lower == "x-stream-resume" && hv == "1") { - resume_opt_in = true; + break; } } } - if (resume_opt_in && !conv_id.empty()) { + if (!conv_id.empty()) { fan_out_delete_others_for_conv(*this, conv_id, name); } } diff --git a/tools/ui/src/lib/services/chat.service.ts b/tools/ui/src/lib/services/chat.service.ts index b0d72476143c..1cc28e7f07b7 100644 --- a/tools/ui/src/lib/services/chat.service.ts +++ b/tools/ui/src/lib/services/chat.service.ts @@ -320,12 +320,9 @@ export class ChatService { try { const headers: Record = { ...getJsonHeaders() }; - if (stream) { - headers['X-Stream-Resume'] = '1'; - } - // tag the request with the conversation id so the server can later list live or recently completed - // sessions for that conversation, this is what powers discoverActiveStream on tab reopen - if (conversationId) { + // tag streaming requests with the conversation id, this single header is the opt in for the + // server side replay buffer and powers discoverActiveStream on tab reopen + if (stream && conversationId) { headers['X-Conversation-Id'] = conversationId; } const response = await fetch(API_CHAT.COMPLETIONS, { From 2b1b3b70ced7554bf2448ee6531f9fca407325e0 Mon Sep 17 00:00:00 2001 From: Pascal Date: Sun, 17 May 2026 21:08:26 +0200 Subject: [PATCH 04/39] server: drop magic 17, derive the X-Conversation-Id header length from sizeof at build time --- tools/server/server-context.cpp | 24 ++++++++++++------------ tools/server/server-models.cpp | 17 +++++++++-------- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index b9697cd9e642..f7f63059d36f 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -4152,19 +4152,19 @@ std::unique_ptr server_routes::handle_completions_impl( std::string conversation_id; if (stream) { // request headers preserve the wire casing, the scan is case insensitive + static constexpr char target[] = "x-conversation-id"; + static constexpr size_t target_len = sizeof(target) - 1; for (const auto & [hk, hv] : req.headers) { - if (hk.size() == 17) { - bool match = true; - static const char target[] = "x-conversation-id"; - for (size_t i = 0; i < 17; ++i) { - char c = hk[i]; - if (c >= 'A' && c <= 'Z') c = char(c + 32); - if (c != target[i]) { match = false; break; } - } - if (match) { - conversation_id = hv; - break; - } + if (hk.size() != target_len) continue; + bool match = true; + for (size_t i = 0; i < target_len; ++i) { + char c = hk[i]; + if (c >= 'A' && c <= 'Z') c = char(c + 32); + if (c != target[i]) { match = false; break; } + } + if (match) { + conversation_id = hv; + break; } } } diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index 16ea5f304b55..b6ddc369ff63 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -1285,16 +1285,17 @@ server_http_res_ptr server_models::proxy_request(const server_http_req & req, co // enforces the cross child invariant 'one session per convId', safe to call unconditionally // and cheap on loopback. ignored for any request without that header { + static constexpr char target[] = "x-conversation-id"; + static constexpr size_t target_len = sizeof(target) - 1; std::string conv_id; for (const auto & [hk, hv] : req.headers) { - if (hk.size() == 17) { - std::string lower(hk); - std::transform(lower.begin(), lower.end(), lower.begin(), - [](unsigned char c) { return (c >= 'A' && c <= 'Z') ? char(c + 32) : char(c); }); - if (lower == "x-conversation-id") { - conv_id = hv; - break; - } + if (hk.size() != target_len) continue; + std::string lower(hk); + std::transform(lower.begin(), lower.end(), lower.begin(), + [](unsigned char c) { return (c >= 'A' && c <= 'Z') ? char(c + 32) : char(c); }); + if (lower == target) { + conv_id = hv; + break; } } if (!conv_id.empty()) { From 00dd0816129a5210351c78bb90e0828f8eb69f99 Mon Sep 17 00:00:00 2001 From: Pascal Date: Mon, 18 May 2026 07:52:12 +0200 Subject: [PATCH 05/39] refactor: address review feedback from ngxson --- tools/server/server-context.cpp | 326 +++----------- tools/server/server-context.h | 6 - tools/server/server-http.cpp | 46 +- tools/server/server-http.h | 11 + tools/server/server-models.cpp | 407 +++++++++--------- tools/server/server-models.h | 11 +- tools/server/server-stream.cpp | 124 ++++++ tools/server/server-stream.h | 17 + tools/server/server.cpp | 41 +- tools/ui/src/lib/services/chat.service.ts | 22 +- .../src/lib/services/stream-resume.service.ts | 8 +- tools/ui/src/lib/stores/chat.svelte.ts | 15 +- tools/ui/src/lib/utils/stream-identity.ts | 15 + 13 files changed, 533 insertions(+), 516 deletions(-) create mode 100644 tools/ui/src/lib/utils/stream-identity.ts diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index f7f63059d36f..3349de0fe96e 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -865,7 +865,6 @@ struct server_context_impl { server_queue queue_tasks; server_response queue_results; - mutable stream_session_manager stream_sessions; // note: chat_params must not be refreshed upon existing sleeping state server_chat_params chat_params; @@ -874,7 +873,6 @@ struct server_context_impl { server_context_impl() { mtmd_helper_log_set(common_log_default_callback, nullptr); - stream_sessions.start_gc(); } ~server_context_impl() { @@ -3378,26 +3376,9 @@ struct server_context_impl { SLT_TRC(slot, "cached n_tokens = %d, memory_seq_rm [%d, end)\n", slot.prompt.n_tokens(), p0); - // the startup probe in common_context_can_seq_rm only tests a 2 token tail removal - // on seq 0, it cannot guarantee that every partial eviction will succeed at any - // position on any live seq. on refusal by the memory backend we clear the whole - // seq on both contexts and let update_slots reprefill from zero on this iteration - auto * mem_tgt = llama_get_memory(ctx_tgt); - bool partial_ok_tgt = llama_memory_seq_rm(mem_tgt, slot.id, p0, -1); - bool partial_ok_dft = true; + common_context_seq_rm(ctx_tgt, slot.id, p0, -1); if (ctx_dft) { - partial_ok_dft = llama_memory_seq_rm(llama_get_memory(ctx_dft.get()), slot.id, p0, -1); - } - if (!partial_ok_tgt || !partial_ok_dft) { - SLT_WRN(slot, "partial KV eviction refused at p0=%d (tgt=%d, dft=%d), full clear of seq %d, reprefilling from zero\n", - p0, partial_ok_tgt ? 1 : 0, partial_ok_dft ? 1 : 0, slot.id); - llama_memory_seq_rm(mem_tgt, slot.id, -1, -1); - if (ctx_dft) { - llama_memory_seq_rm(llama_get_memory(ctx_dft.get()), slot.id, -1, -1); - } - slot.prompt.tokens.keep_first(0); - slot.n_prompt_tokens_cache = 0; - slot.n_prompt_tokens_processed = 0; + common_context_seq_rm(ctx_dft.get(), slot.id, p0, -1); } // If using an alora, there may be uncached tokens that come @@ -4063,72 +4044,6 @@ void server_context::set_state_callback(server_state_callback_t callback) { } // -// runs in a detached thread, owns the reader and posts the tasks on it -// pulls results, formats them as SSE bytes, appends to the session -// reacts to server shutdown via the shared atomic from the manager -static void spawn_stream_drain( - std::unique_ptr reader, - stream_session_ptr session, - task_response_type res_type, - std::shared_ptr> shutdown) { - std::thread([reader = std::move(reader), - session, - res_type, - shutdown]() mutable { - SRV_INF("stream drain thread started for conv=%s\n", session->conversation_id.c_str()); - // wire the user Stop hook, evict_and_cancel will call this and the reader cancels its queue tasks - session->set_stop_producer([raw = reader.get()] { - raw->stop(); - }); - auto should_stop = [shutdown] { - return shutdown->load(std::memory_order_relaxed); - }; - auto fmt_ok = [res_type](const json & j) -> std::string { - if (res_type == TASK_RESPONSE_TYPE_ANTHROPIC) return format_anthropic_sse(j); - if (res_type == TASK_RESPONSE_TYPE_OAI_RESP) return format_oai_resp_sse(j); - return format_oai_sse(j); - }; - auto fmt_err = [res_type](const json & err) -> std::string { - if (res_type == TASK_RESPONSE_TYPE_ANTHROPIC) { - return format_anthropic_sse({{"event", "error"}, {"data", err}}); - } - return format_oai_sse(json{{"error", err}}); - }; - try { - while (reader->has_next()) { - auto r = reader->next(should_stop); - if (!r) { - break; - } - json j = r->to_json(); - if (r->is_error()) { - auto sse = fmt_err(j); - session->append(sse.data(), sse.size()); - break; - } - auto sse = fmt_ok(j); - if (!session->append(sse.data(), sse.size())) { - break; - } - } - // emit the OAI terminator for the formats that use it - if (res_type != TASK_RESPONSE_TYPE_NONE - && res_type != TASK_RESPONSE_TYPE_OAI_RESP - && res_type != TASK_RESPONSE_TYPE_ANTHROPIC) { - static constexpr char done_str[] = "data: [DONE]\n\n"; - session->append(done_str, sizeof(done_str) - 1); - } - } catch (const std::exception & e) { - auto sse = fmt_err(format_error_response(e.what(), ERROR_TYPE_SERVER)); - session->append(sse.data(), sse.size()); - } - // unwire the stop hook before reader goes out of scope, no dangling captured raw ptr - session->set_stop_producer(nullptr); - session->finalize(); - SRV_INF("stream drain thread finished for conv=%s bytes=%zu\n", session->conversation_id.c_str(), session->total_size()); - }).detach(); -} - // server_routes // @@ -4145,38 +4060,6 @@ std::unique_ptr server_routes::handle_completions_impl( auto & rd = res->rd; auto & params = this->params; - // resumable mode opt in: a non empty X-Conversation-Id on a streaming request enables it. - // the conv id is the session identity end to end (client localStorage, server map, - // /v1/stream/ routes). non stream OAI calls keep the standard flow regardless - bool stream = json_value(data, "stream", false); - std::string conversation_id; - if (stream) { - // request headers preserve the wire casing, the scan is case insensitive - static constexpr char target[] = "x-conversation-id"; - static constexpr size_t target_len = sizeof(target) - 1; - for (const auto & [hk, hv] : req.headers) { - if (hk.size() != target_len) continue; - bool match = true; - for (size_t i = 0; i < target_len; ++i) { - char c = hk[i]; - if (c >= 'A' && c <= 'Z') c = char(c + 32); - if (c != target[i]) { match = false; break; } - } - if (match) { - conversation_id = hv; - break; - } - } - } - const bool resumable = !conversation_id.empty(); - std::unique_ptr drain_reader; - server_response_reader * post_target = &rd; - if (resumable) { - drain_reader = std::make_unique( - queue_tasks, queue_results, HTTP_POLLING_SECONDS); - post_target = drain_reader.get(); - } - try { std::vector tasks; @@ -4244,19 +4127,34 @@ std::unique_ptr server_routes::handle_completions_impl( tasks.push_back(std::move(task)); } - post_target->post_tasks(std::move(tasks)); + rd.post_tasks(std::move(tasks)); } catch (const std::exception & e) { res->error(format_error_response(e.what(), ERROR_TYPE_INVALID_REQUEST)); return res; } - // session creation comes after post_tasks succeeds, so a throw during parsing or task - // construction returns early without leaving an orphan in the manager map. - // create_or_replace evicts and cancels any prior session on this conv, the invariant - // 'one live session per conv' holds - stream_session_ptr session; - if (resumable) { - session = ctx_server.stream_sessions.create_or_replace(conversation_id); + bool stream = json_value(data, "stream", false); + + // resumable streaming opt in, a non empty X-Conversation-Id on a streaming request enables it. + // the conv id is the session identity end to end, client localStorage and server map share + // the same key. non stream and non opted in calls keep the standard flow unchanged + std::string conversation_id; + if (stream) { + static constexpr char target[] = "x-conversation-id"; + static constexpr size_t target_len = sizeof(target) - 1; + for (const auto & [hk, hv] : req.headers) { + if (hk.size() != target_len) continue; + bool match = true; + for (size_t i = 0; i < target_len; ++i) { + char c = hk[i]; + if (c >= 'A' && c <= 'Z') c = char(c + 32); + if (c != target[i]) { match = false; break; } + } + if (match) { + conversation_id = hv; + break; + } + } } if (!stream) { @@ -4289,30 +4187,6 @@ std::unique_ptr server_routes::handle_completions_impl( res->ok(arr); } } - } else if (resumable) { - // spawn the detached drain that pumps the response into the session buffer - spawn_stream_drain( - std::move(drain_reader), - session, - res_type, - ctx_server.stream_sessions.shutdown_flag()); - // HTTP response reads from the session, decoupled from the producer - res->status = 200; - res->content_type = "text/event-stream"; - auto offset_ptr = std::make_shared(0); - auto session_capture = session; - res->next = [session_capture, offset_ptr, &req](std::string & output) -> bool { - bool got_any = false; - session_capture->read_from(*offset_ptr, - [&](const char * d, size_t n) { - output.append(d, n); - *offset_ptr += n; - got_any = true; - return false; // exit read_from after the current available bytes - }, - req.should_stop); - return got_any; - }; } else { // in streaming mode, the first error must be treated as non-stream response // this is to match the OAI API behavior @@ -4359,8 +4233,19 @@ std::unique_ptr server_routes::handle_completions_impl( } }; + // when a tee is attached the session must outlive the http socket. ignore the + // connection closed signal in that case, only an explicit DELETE through the + // session stop_producer hook (which calls rd.stop()) is allowed to abort the + // producer. without a tee the legacy flow stays bit identical + auto effective_should_stop = [res_this, &req]() -> bool { + if (res_this->tee) { + return false; + } + return req.should_stop(); + }; + try { - if (req.should_stop()) { + if (effective_should_stop()) { SRV_DBG("%s", "stopping streaming due to should_stop condition\n"); return false; // should_stop condition met } @@ -4394,8 +4279,8 @@ std::unique_ptr server_routes::handle_completions_impl( // receive subsequent results bool timeout = false; int64_t start_time = ggml_time_ms(); - auto result = rd.next([&timeout, &req, &start_time, ¶ms]() { - if (req.should_stop()) { + auto result = rd.next([&timeout, &start_time, ¶ms, &effective_should_stop]() { + if (effective_should_stop()) { return true; // should_stop condition met } else if (params.sse_ping_interval > 0 && ggml_time_ms() - start_time > (int64_t)params.sse_ping_interval * 1000) { timeout = true; @@ -4413,7 +4298,7 @@ std::unique_ptr server_routes::handle_completions_impl( if (result == nullptr) { SRV_DBG("%s", "stopping streaming due to should_stop condition\n"); - GGML_ASSERT(req.should_stop()); + GGML_ASSERT(effective_should_stop()); return false; // should_stop condition met } @@ -4451,6 +4336,26 @@ std::unique_ptr server_routes::handle_completions_impl( }; } + // attach the resumable session, tee mirrors each SSE chunk into the ring buffer, on_stream_end + // finalizes it on either drain path (wire or detached), and the stop_producer hook lets the + // DELETE /v1/stream/ route abort the underlying reader from anywhere. the http layer + // owns the keep alive after a client disconnect via its detached drain, server-context only + // declares the wiring here and never spawns a thread itself + if (!conversation_id.empty()) { + auto session = g_stream_sessions.create_or_replace(conversation_id); + session->set_stop_producer([res_this = res.get()] { + res_this->rd.stop(); + }); + res->tee = [session](const char * d, size_t n) { + session->append(d, n); + }; + res->on_stream_end = [session] { + // detach the stop hook before the response goes out of scope, no dangling captured ptr + session->set_stop_producer(nullptr); + session->finalize(); + }; + } + return res; } @@ -5216,18 +5121,6 @@ void server_routes::init_routes() { res->ok(result->to_json()); return res; }; - - this->get_stream = [this](const server_http_req & req) { - return handle_stream_get_impl(req); - }; - - this->get_streams = [this](const server_http_req & req) { - return handle_streams_list_impl(req); - }; - - this->delete_stream = [this](const server_http_req & req) { - return handle_stream_delete_impl(req); - }; } json server_routes::get_model_info() const { @@ -5498,102 +5391,3 @@ std::unique_ptr server_routes::handle_count_tokens(const l res->ok(response); return res; } - -std::unique_ptr server_routes::handle_stream_get_impl(const server_http_req & req) { - auto res = create_response(); - - // GET /v1/stream/?from=N replays the SSE bytes for that conversation, - // blocks for more bytes when the session is still running, ends on finalize - std::string conv_id = req.get_param("conv_id"); - if (conv_id.empty()) { - res->error(format_error_response("Missing conversation id in path", ERROR_TYPE_INVALID_REQUEST)); - return res; - } - auto session = ctx_server.stream_sessions.get(conv_id); - if (!session) { - res->error(format_error_response("Stream not found or expired", ERROR_TYPE_NOT_FOUND)); - return res; - } - size_t from = 0; - { - std::string from_str = req.get_param("from"); - if (!from_str.empty()) { - try { - from = static_cast(std::stoull(from_str)); - } catch (const std::exception &) { - res->error(format_error_response("Invalid 'from' offset", ERROR_TYPE_INVALID_REQUEST)); - return res; - } - } - } - if (from < session->dropped_prefix()) { - res->error(format_error_response("Stream offset lost, please restart", ERROR_TYPE_INVALID_REQUEST)); - return res; - } - res->status = 200; - res->content_type = "text/event-stream"; - - auto offset_ptr = std::make_shared(from); - auto session_capture = session; - res->next = [session_capture, offset_ptr, &req](std::string & output) -> bool { - bool got_any = false; - session_capture->read_from(*offset_ptr, - [&](const char * d, size_t n) { - output.append(d, n); - *offset_ptr += n; - got_any = true; - return false; - }, - req.should_stop); - return got_any; - }; - return res; -} - -std::unique_ptr server_routes::handle_streams_list_impl(const server_http_req & req) { - auto res = create_response(); - - // GET /v1/streams returns sessions as a JSON array. - // with conversation_id set: at most one entry for that conv (running or finalized). - // without conversation_id: every live or recently completed session known to this server, - // used by the WebUI at mount and on visibilitychange to populate the sidebar spinners - std::string conversation_id = req.get_param("conversation_id"); - std::vector sessions; - if (conversation_id.empty()) { - sessions = ctx_server.stream_sessions.list_all(); - } else { - auto s = ctx_server.stream_sessions.get(conversation_id); - if (s) { - sessions.push_back(s); - } - } - json arr = json::array(); - for (auto & s : sessions) { - arr.push_back({ - {"conversation_id", s->conversation_id}, - {"is_done", s->is_done()}, - {"total_bytes", s->total_size()}, - {"started_at", s->started_ts}, - {"completed_at", s->completed_at()}, - }); - } - res->ok(arr); - return res; -} - -std::unique_ptr server_routes::handle_stream_delete_impl(const server_http_req & req) { - auto res = create_response(); - - // DELETE /v1/stream/ cancels the producer side then evicts the buffer. - // idempotent: a session that already finalized or was never created simply returns 204 - std::string conv_id = req.get_param("conv_id"); - if (conv_id.empty()) { - res->error(format_error_response("Missing conversation id in path", ERROR_TYPE_INVALID_REQUEST)); - return res; - } - SRV_INF("DELETE /v1/stream/%s -> evict_and_cancel\n", conv_id.c_str()); - ctx_server.stream_sessions.evict_and_cancel(conv_id); - res->status = 204; - res->content_type = "application/json"; - return res; -} diff --git a/tools/server/server-context.h b/tools/server/server-context.h index 8bd07db73c95..952f825f7245 100644 --- a/tools/server/server-context.h +++ b/tools/server/server-context.h @@ -152,9 +152,6 @@ struct server_routes { server_http_context::handler_t post_rerank; server_http_context::handler_t get_lora_adapters; server_http_context::handler_t post_lora_adapters; - server_http_context::handler_t get_stream; - server_http_context::handler_t get_streams; - server_http_context::handler_t delete_stream; // to be used in router mode json get_model_info() const; @@ -171,9 +168,6 @@ struct server_routes { std::unique_ptr handle_slots_erase(const server_http_req &, int id_slot); std::unique_ptr handle_embeddings_impl(const server_http_req & req, task_response_type res_type); std::unique_ptr handle_count_tokens(const llama_vocab * vocab, mtmd_context * mctx, const server_http_req & req, task_response_type res_type); - std::unique_ptr handle_stream_get_impl(const server_http_req & req); - std::unique_ptr handle_streams_list_impl(const server_http_req & req); - std::unique_ptr handle_stream_delete_impl(const server_http_req & req); // using unique_ptr to allow late initialization of const std::unique_ptr meta; diff --git a/tools/server/server-http.cpp b/tools/server/server-http.cpp index ec9c8080631f..c823aef9dbf5 100644 --- a/tools/server/server-http.cpp +++ b/tools/server/server-http.cpp @@ -499,22 +499,64 @@ static void process_handler_response(server_http_req_ptr && request, server_http // convert to shared_ptr as both chunked_content_provider() and on_complete() need to use it std::shared_ptr q_ptr = std::move(request); std::shared_ptr r_ptr = std::move(response); - const auto chunked_content_provider = [response = r_ptr](size_t, const httplib::DataSink & sink) -> bool { + // shared flag, flipped to true the moment the producer signals next() == false on the + // normal wire path. on_complete uses it to decide whether to spawn the detached drain. + // covers every disconnect timing httplib can observe: between two chunks (peer dead + // detected before content_provider is called), during a chunk (sink.write fails), or + // never (the producer drained cleanly). without this httplib bails the provider when + // the peer is gone, on_complete fires, the shared_ptr resets, the underlying reader + // is destroyed, and the backend stops mid generation + auto stream_done = std::make_shared>(false); + + const auto chunked_content_provider = [response = r_ptr, stream_done](size_t, const httplib::DataSink & sink) -> bool { std::string chunk; const bool has_next = response->next(chunk); if (!chunk.empty()) { + // mirror to the tee first, the session must reflect the SSE stream regardless + // of whether the wire write succeeds for this chunk + if (response->tee) { + response->tee(chunk.data(), chunk.size()); + } if (!sink.write(chunk.data(), chunk.size())) { + // peer is gone mid chunk, on_complete will pick up the detached drain return false; } SRV_DBG("http: streamed chunk: %s\n", chunk.c_str()); } if (!has_next) { + stream_done->store(true, std::memory_order_release); + if (response->on_stream_end) { + response->on_stream_end(); + } sink.done(); SRV_DBG("%s", "http: stream ended\n"); } return has_next; }; - const auto on_complete = [request = q_ptr, response = r_ptr](bool) mutable { + const auto on_complete = [request = q_ptr, response = r_ptr, stream_done](bool) mutable { + // if the producer is still running when httplib hands the response back, the peer + // is gone but the generation must keep going. detach a thread that pumps next() + // into the tee until done, then runs on_stream_end. capture by value keeps the + // response alive past the reset below, the detached thread holds its own reference + if (!stream_done->load(std::memory_order_acquire) && response->tee) { + auto resp_keep = response; + std::thread([resp_keep]() { + std::string c; + while (true) { + c.clear(); + bool more = resp_keep->next(c); + if (!c.empty() && resp_keep->tee) { + resp_keep->tee(c.data(), c.size()); + } + if (!more) { + break; + } + } + if (resp_keep->on_stream_end) { + resp_keep->on_stream_end(); + } + }).detach(); + } response.reset(); // trigger the destruction of the response object request.reset(); // trigger the destruction of the request object }; diff --git a/tools/server/server-http.h b/tools/server/server-http.h index c31dd9109de2..d5743962d86d 100644 --- a/tools/server/server-http.h +++ b/tools/server/server-http.h @@ -29,6 +29,17 @@ struct server_http_res { return next != nullptr; } + // optional, each chunk produced by next() is forwarded here before being written to the + // wire. on wire failure (peer gone), the http layer detaches a background drain that keeps + // invoking next() and forwarding to the tee until next() returns false. lets streams + // survive a client disconnect, the producer keeps writing into the tee even when nobody + // is listening on the original HTTP socket + std::function tee = nullptr; + + // optional, called when the stream ends on either path (wire drained to false, or detached + // drain reached false). used by the tee owner to finalize its sink, idempotent expected + std::function on_stream_end = nullptr; + virtual ~server_http_res() = default; }; diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index b6ddc369ff63..831aebe78bbf 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -7,6 +7,7 @@ #include "download.h" #include // TODO: remove this once we use HTTP client from download.h +#include #include #include @@ -1263,11 +1264,6 @@ bool server_models::ensure_model_ready(const std::string & name) { return true; } -// forward declarations for the file scope helpers used below, the bodies live further down -// next to the other routes helpers to keep the proxy methods compact -static void fan_out_delete_others_for_conv( - server_models & models, const std::string & conversation_id, const std::string & target_child); - server_http_res_ptr server_models::proxy_request(const server_http_req & req, const std::string & method, const std::string & name, bool update_last_used) { auto meta = get_meta(name); if (!meta.has_value()) { @@ -1280,28 +1276,6 @@ server_http_res_ptr server_models::proxy_request(const server_http_req & req, co std::unique_lock lk(mutex); mapping[name].meta.last_used = ggml_time_ms(); } - // when the client opts in to resumable streaming via a non empty X-Conversation-Id, - // fan out a DELETE on every other ready child to evict any prior session for this conv. - // enforces the cross child invariant 'one session per convId', safe to call unconditionally - // and cheap on loopback. ignored for any request without that header - { - static constexpr char target[] = "x-conversation-id"; - static constexpr size_t target_len = sizeof(target) - 1; - std::string conv_id; - for (const auto & [hk, hv] : req.headers) { - if (hk.size() != target_len) continue; - std::string lower(hk); - std::transform(lower.begin(), lower.end(), lower.begin(), - [](unsigned char c) { return (c >= 'A' && c <= 'Z') ? char(c + 32) : char(c); }); - if (lower == target) { - conv_id = hv; - break; - } - } - if (!conv_id.empty()) { - fan_out_delete_others_for_conv(*this, conv_id, name); - } - } SRV_INF("proxying request to model %s on port %d\n", name.c_str(), meta->port); std::string proxy_path = req.path; if (!req.query_string.empty()) { @@ -1320,9 +1294,6 @@ server_http_res_ptr server_models::proxy_request(const server_http_req & req, co base_params.timeout_read, base_params.timeout_write ); - // session identity end to end is the X-Conversation-Id sent by the client, no extra opaque - // token to mangle here. the parent can later route GET /v1/stream/ back to the right - // child by probing /v1/streams across childs return proxy; } @@ -1569,8 +1540,51 @@ struct server_models_sse_client { } }; -// percent encode a single query string value, covers reserved chars without dragging in -// httplib::detail. used by the stream routes to forward conversation_id to children +static void res_ok(std::unique_ptr & res, const json & response_data) { + res->status = 200; + res->data = safe_json_to_str(response_data); +} + +static void res_err(std::unique_ptr & res, const json & error_data) { + res->status = json_value(error_data, "code", 500); + res->data = safe_json_to_str({{ "error", error_data }}); +} + +static bool router_validate_model(std::string & name, server_models & models, bool models_autoload, std::unique_ptr & res) { + if (name.empty()) { + res_err(res, format_error_response("model name is missing from the request", ERROR_TYPE_INVALID_REQUEST)); + return false; + } + auto meta = models.get_meta(name); + if (!meta.has_value()) { + res_err(res, format_error_response(string_format("model '%s' not found", name.c_str()), ERROR_TYPE_INVALID_REQUEST)); + return false; + } + // resolve alias to canonical model name + name = meta->name; + if (models_autoload) { + models.ensure_model_ready(name); + } else { + if (!meta->is_running()) { + res_err(res, format_error_response("model is not loaded", ERROR_TYPE_INVALID_REQUEST)); + return false; + } + } + return true; +} + +static bool is_autoload(const common_params & params, const server_http_req & req) { + std::string autoload = req.get_param("autoload"); + if (autoload.empty()) { + return params.models_autoload; + } else { + return autoload == "true" || autoload == "1"; + } +} + +// percent encode a single query string or path component value. covers reserved chars without +// dragging in httplib::detail. used by the resumable stream routes to forward conversation_id +// to children safely static std::string encode_qs(const std::string & in) { std::string out; out.reserve(in.size() * 3); @@ -1588,11 +1602,24 @@ static std::string encode_qs(const std::string & in) { return out; } -// scan every ready child for an active session on this conversation_id by fanning out a -// short list query on the loopback, returns the meta of the first child whose array is -// non empty. with the invariant 'one session per convId across all children' enforced by -// the POST path, at most one child can match -static std::optional find_child_for_conv( +// extract the optional model suffix from a conversation_id. the WebUI encodes the active model +// name after :: when the user has explicitly picked a model, so the router can route stream +// lookups direct to the right child without probing every other one. returns empty when no +// separator is present, in which case the caller falls back to probing or fan out +static std::string extract_model_from_conv(const std::string & conv_id) { + static constexpr char SEP[] = "::"; + static constexpr size_t SEP_LEN = sizeof(SEP) - 1; + auto pos = conv_id.rfind(SEP); + if (pos == std::string::npos) { + return std::string(); + } + return conv_id.substr(pos + SEP_LEN); +} + +// loopback probe across every ready child, returns the meta of the first one that reports a +// live or recently completed session for this conv. only called as a fallback when the conv +// id carries no :: suffix +static std::optional probe_child_for_conv( server_models & models, const std::string & conversation_id) { if (conversation_id.empty()) { return std::nullopt; @@ -1622,69 +1649,6 @@ static std::optional find_child_for_conv( return std::nullopt; } -// fan out a DELETE on every ready child EXCEPT the one we are about to route the POST to, -// so a model swap on the same conversation_id evicts the previous session cleanly. safe to -// call unconditionally: a child without the session returns 204 and does nothing -static void fan_out_delete_others_for_conv( - server_models & models, const std::string & conversation_id, const std::string & target_child) { - if (conversation_id.empty()) { - return; - } - std::string child_path = "/v1/stream/" + encode_qs(conversation_id); - for (auto & meta : models.get_all_meta()) { - if (!meta.is_ready() || meta.name == target_child) { - continue; - } - httplib::Client cli(CHILD_ADDR, meta.port); - cli.set_connection_timeout(0, 250 * 1000); - cli.set_read_timeout(0, 250 * 1000); - cli.set_write_timeout(0, 250 * 1000); - cli.Delete(child_path.c_str()); - } -} - -static void res_ok(std::unique_ptr & res, const json & response_data) { - res->status = 200; - res->data = safe_json_to_str(response_data); -} - -static void res_err(std::unique_ptr & res, const json & error_data) { - res->status = json_value(error_data, "code", 500); - res->data = safe_json_to_str({{ "error", error_data }}); -} - -static bool router_validate_model(std::string & name, server_models & models, bool models_autoload, std::unique_ptr & res) { - if (name.empty()) { - res_err(res, format_error_response("model name is missing from the request", ERROR_TYPE_INVALID_REQUEST)); - return false; - } - auto meta = models.get_meta(name); - if (!meta.has_value()) { - res_err(res, format_error_response(string_format("model '%s' not found", name.c_str()), ERROR_TYPE_INVALID_REQUEST)); - return false; - } - // resolve alias to canonical model name - name = meta->name; - if (models_autoload) { - models.ensure_model_ready(name); - } else { - if (!meta->is_running()) { - res_err(res, format_error_response("model is not loaded", ERROR_TYPE_INVALID_REQUEST)); - return false; - } - } - return true; -} - -static bool is_autoload(const common_params & params, const server_http_req & req) { - std::string autoload = req.get_param("autoload"); - if (autoload.empty()) { - return params.models_autoload; - } else { - return autoload == "true" || autoload == "1"; - } -} - void server_models_routes::init_routes() { this->get_router_props = [this](const server_http_req & req) { std::string name = req.get_param("model"); @@ -1724,120 +1688,6 @@ void server_models_routes::init_routes() { return models.proxy_request(req, method, name, false); }; - - this->proxy_get_stream = [this](const server_http_req & req) { - auto res = std::make_unique(); - - // GET /v1/stream/?from=N. find the child that owns the session for this conv - // via the loopback probe in find_child_for_conv, then forward the SSE GET to it. - // returns 404 if no child currently has an alive or recently completed session - std::string conv_id = req.get_param("conv_id"); - if (conv_id.empty()) { - res_err(res, format_error_response("Missing conversation id in path", ERROR_TYPE_INVALID_REQUEST)); - return res; - } - auto owner = find_child_for_conv(models, conv_id); - if (!owner.has_value()) { - res_err(res, format_error_response("Stream not found or expired", ERROR_TYPE_NOT_FOUND)); - return res; - } - - std::string from = req.get_param("from"); - std::string child_path = "/v1/stream/" + encode_qs(conv_id); - if (!from.empty()) { - child_path += "?from=" + from; - } - SRV_INF("proxying stream resume to model %s on port %d, path=%s\n", - owner->name.c_str(), owner->port, child_path.c_str()); - - auto proxy = std::make_unique( - "GET", - "http", - CHILD_ADDR, - owner->port, - child_path, - req.headers, - req.body, - req.files, - req.should_stop, - params.timeout_read, - params.timeout_write); - return std::unique_ptr(std::move(proxy)); - }; - - this->proxy_get_streams = [this](const server_http_req & req) { - auto res = std::make_unique(); - - // GET /v1/streams returns sessions as a JSON array. with conversation_id set the filter - // is forwarded to childs and at most one entry comes back, without it the WebUI uses the - // result at mount and on visibilitychange to populate the sidebar spinners across convs. - // sequential fan out on every ready child, fail soft on per child error, aggregate - std::string conversation_id = req.get_param("conversation_id"); - std::string child_path = "/v1/streams"; - if (!conversation_id.empty()) { - child_path += "?conversation_id=" + encode_qs(conversation_id); - } - - json aggregated = json::array(); - for (auto & meta : models.get_all_meta()) { - if (!meta.is_ready()) { - continue; - } - httplib::Client cli(CHILD_ADDR, meta.port); - cli.set_connection_timeout(0, 250 * 1000); - cli.set_read_timeout(0, 250 * 1000); - cli.set_write_timeout(0, 250 * 1000); - auto resp = cli.Get(child_path.c_str()); - if (!resp || resp->status != 200) { - continue; - } - try { - json child_arr = json::parse(resp->body); - if (!child_arr.is_array()) { - continue; - } - for (auto & entry : child_arr) { - if (entry.is_object()) { - aggregated.push_back(entry); - } - } - } catch (const std::exception &) { - continue; - } - } - res_ok(res, aggregated); - return res; - }; - - this->proxy_delete_stream = [this](const server_http_req & req) { - auto res = std::make_unique(); - - // DELETE /v1/stream/ fans out to every ready child. each child runs an idempotent - // evict_and_cancel, returning 204 whether or not it actually owned a session for this conv. - // a Stop must feel instantaneous so timeouts are short, the child route is in memory - std::string conv_id = req.get_param("conv_id"); - if (conv_id.empty()) { - res_err(res, format_error_response("Missing conversation id in path", ERROR_TYPE_INVALID_REQUEST)); - return res; - } - - std::string child_path = "/v1/stream/" + encode_qs(conv_id); - for (auto & meta : models.get_all_meta()) { - if (!meta.is_ready()) { - continue; - } - httplib::Client cli(CHILD_ADDR, meta.port); - cli.set_connection_timeout(0, 250 * 1000); - cli.set_read_timeout(0, 500 * 1000); - cli.set_write_timeout(0, 250 * 1000); - auto resp = cli.Delete(child_path.c_str()); - (void) resp; // best effort, 404 and network errors are equivalent to no op - } - res->status = 204; - res->content_type = "application/json"; - return res; - }; - this->proxy_post = [this](const server_http_req & req) { std::string method = "POST"; json body = json::parse(req.body); @@ -2047,6 +1897,135 @@ void server_models_routes::init_routes() { res_ok(res, {{"success", true}}); return res; }; + + this->router_stream_get = [this](const server_http_req & req) { + // GET /v1/stream/?from=N. when the conv carries a ::model suffix, route + // straight to that child, otherwise loopback probe every ready child. returns 404 + // when no child currently owns a session for this conv + auto res = std::make_unique(); + std::string conv_id = req.get_param("conv_id"); + if (conv_id.empty()) { + res_err(res, format_error_response("Missing conversation id in path", ERROR_TYPE_INVALID_REQUEST)); + return res; + } + std::optional owner; + std::string model_hint = extract_model_from_conv(conv_id); + if (!model_hint.empty()) { + auto direct = models.get_meta(model_hint); + if (direct.has_value() && direct->is_ready()) { + owner = direct; + } + } + if (!owner.has_value()) { + owner = probe_child_for_conv(models, conv_id); + } + if (!owner.has_value()) { + res_err(res, format_error_response("Stream not found or expired", ERROR_TYPE_NOT_FOUND)); + return res; + } + std::string from = req.get_param("from"); + std::string child_path = "/v1/stream/" + encode_qs(conv_id); + if (!from.empty()) { + child_path += "?from=" + from; + } + SRV_INF("proxying stream resume to model %s on port %d, path=%s\n", + owner->name.c_str(), owner->port, child_path.c_str()); + auto proxy = std::make_unique( + "GET", + "http", + CHILD_ADDR, + owner->port, + child_path, + req.headers, + req.body, + req.files, + req.should_stop, + params.timeout_read, + params.timeout_write); + return std::unique_ptr(std::move(proxy)); + }; + + this->router_streams_list = [this](const server_http_req & req) { + // GET /v1/streams aggregates sessions from every ready child. the WebUI mounts and + // visibilitychanges use this to drive the sidebar spinners across convs. when a + // conversation_id filter is set we still fan out because the matching session can live + // on any child, the filter just narrows the response set + auto res = std::make_unique(); + std::string conversation_id = req.get_param("conversation_id"); + std::string child_path = "/v1/streams"; + if (!conversation_id.empty()) { + child_path += "?conversation_id=" + encode_qs(conversation_id); + } + json aggregated = json::array(); + for (auto & meta : models.get_all_meta()) { + if (!meta.is_ready()) { + continue; + } + httplib::Client cli(CHILD_ADDR, meta.port); + cli.set_connection_timeout(0, 250 * 1000); + cli.set_read_timeout(0, 250 * 1000); + cli.set_write_timeout(0, 250 * 1000); + auto resp = cli.Get(child_path.c_str()); + if (!resp || resp->status != 200) { + continue; + } + try { + json child_arr = json::parse(resp->body); + if (!child_arr.is_array()) { + continue; + } + for (auto & entry : child_arr) { + if (entry.is_object()) { + aggregated.push_back(entry); + } + } + } catch (const std::exception &) { + continue; + } + } + res_ok(res, aggregated); + return res; + }; + + this->router_stream_delete = [this](const server_http_req & req) { + // DELETE /v1/stream/. with a ::model suffix we forward to that child only, + // otherwise fan out across every ready child. each child runs an idempotent + // evict_and_cancel so a child without the session returns 204 and does nothing + auto res = std::make_unique(); + std::string conv_id = req.get_param("conv_id"); + if (conv_id.empty()) { + res_err(res, format_error_response("Missing conversation id in path", ERROR_TYPE_INVALID_REQUEST)); + return res; + } + std::string child_path = "/v1/stream/" + encode_qs(conv_id); + std::string model_hint = extract_model_from_conv(conv_id); + auto delete_on = [&](int port) { + httplib::Client cli(CHILD_ADDR, port); + cli.set_connection_timeout(0, 250 * 1000); + cli.set_read_timeout(0, 500 * 1000); + cli.set_write_timeout(0, 250 * 1000); + auto resp = cli.Delete(child_path.c_str()); + (void) resp; // best effort, 404 and network errors are equivalent to no op + }; + if (!model_hint.empty()) { + auto direct = models.get_meta(model_hint); + if (direct.has_value() && direct->is_ready()) { + delete_on(direct->port); + res->status = 204; + res->content_type = "application/json"; + return res; + } + } + for (auto & meta : models.get_all_meta()) { + if (!meta.is_ready()) { + continue; + } + delete_on(meta.port); + } + res->status = 204; + res->content_type = "application/json"; + return res; + }; } diff --git a/tools/server/server-models.h b/tools/server/server-models.h index 070a4468e28f..53029f5c17fc 100644 --- a/tools/server/server-models.h +++ b/tools/server/server-models.h @@ -261,9 +261,6 @@ struct server_models_routes { server_http_context::handler_t get_router_props; server_http_context::handler_t proxy_get; server_http_context::handler_t proxy_post; - server_http_context::handler_t proxy_get_stream; - server_http_context::handler_t proxy_get_streams; - server_http_context::handler_t proxy_delete_stream; server_http_context::handler_t get_router_models; server_http_context::handler_t post_router_models_load; server_http_context::handler_t post_router_models_unload; @@ -271,6 +268,14 @@ struct server_models_routes { server_http_context::handler_t get_router_models_sse; server_http_context::handler_t post_router_models; server_http_context::handler_t del_router_models; + + // router side handlers for the resumable streaming routes. each conversation_id may carry + // an optional ::model suffix to enable direct routing without probing every child. when + // the suffix is absent the get/delete paths fall back to a loopback probe and a fan out + // respectively, the list path always fans out and aggregates + server_http_context::handler_t router_stream_get; + server_http_context::handler_t router_streams_list; + server_http_context::handler_t router_stream_delete; }; /** diff --git a/tools/server/server-stream.cpp b/tools/server/server-stream.cpp index 54ae052e5fde..4bcc26bb9c7c 100644 --- a/tools/server/server-stream.cpp +++ b/tools/server/server-stream.cpp @@ -1,6 +1,9 @@ #include "server-stream.h" +#include "server-common.h" +#include "server-http.h" #include +#include #include namespace { @@ -282,3 +285,124 @@ void stream_session_manager::gc_loop() { } } } + +// process wide manager, lifecycle controlled by llama-server main() via start_gc/stop_gc +stream_session_manager g_stream_sessions; + +// helper, builds the standard error response and assigns it to a brand new http_res +static server_http_res_ptr make_error_response(int status, const std::string & message, error_type type) { + auto res = std::make_unique(); + json err = format_error_response(message, type); + res->status = json_value(err, "code", status); + res->content_type = "application/json; charset=utf-8"; + res->data = safe_json_to_str({{"error", err}}); + return res; +} + +server_http_context::handler_t make_stream_get_handler() { + return [](const server_http_req & req) -> server_http_res_ptr { + // GET /v1/stream/?from=N replays the SSE bytes already buffered for the + // session, blocks for more bytes when the session is still running, returns when + // the session is finalized. the body is streamed back as text/event-stream so the + // browser EventSource can attach to it like a fresh request + std::string conv_id = req.get_param("conv_id"); + if (conv_id.empty()) { + return make_error_response(400, "Missing conversation id in path", ERROR_TYPE_INVALID_REQUEST); + } + auto session = g_stream_sessions.get(conv_id); + if (!session) { + return make_error_response(404, "Stream not found or expired", ERROR_TYPE_NOT_FOUND); + } + size_t from = 0; + std::string from_str = req.get_param("from"); + if (!from_str.empty()) { + try { + from = static_cast(std::stoull(from_str)); + } catch (const std::exception &) { + return make_error_response(400, "Invalid 'from' offset", ERROR_TYPE_INVALID_REQUEST); + } + } + if (from < session->dropped_prefix()) { + return make_error_response(400, "Stream offset lost, please restart", ERROR_TYPE_INVALID_REQUEST); + } + auto res = std::make_unique(); + res->status = 200; + res->content_type = "text/event-stream"; + // the next closure reads from the ring buffer at the requested offset, blocks until + // bytes arrive or the session finalizes. exit each call after draining the available + // chunk so set_chunked_content_provider gets a chance to flush to the socket + auto offset_ptr = std::make_shared(from); + auto session_capture = session; + res->next = [session_capture, offset_ptr, &req](std::string & output) -> bool { + bool got_any = false; + session_capture->read_from(*offset_ptr, + [&](const char * d, size_t n) { + output.append(d, n); + *offset_ptr += n; + got_any = true; + return false; + }, + req.should_stop); + return got_any; + }; + return res; + }; +} + +server_http_context::handler_t make_streams_list_handler() { + return [](const server_http_req & req) -> server_http_res_ptr { + // GET /v1/streams returns sessions as a JSON array. with conversation_id set, every + // session whose key is exactly that id or starts with "::" matches, so a single + // call returns every per model variant for a given conv. without conversation_id, + // every live or recently completed session known to this server, used by the WebUI + // at mount and on visibilitychange to populate the sidebar spinners + std::string conversation_id = req.get_param("conversation_id"); + std::vector sessions; + if (conversation_id.empty()) { + sessions = g_stream_sessions.list_all(); + } else { + const std::string with_sep = conversation_id + "::"; + auto all = g_stream_sessions.list_all(); + for (auto & s : all) { + if (s->conversation_id == conversation_id) { + sessions.push_back(s); + } else if (s->conversation_id.compare(0, with_sep.size(), with_sep) == 0) { + sessions.push_back(s); + } + } + } + json arr = json::array(); + for (auto & s : sessions) { + arr.push_back({ + {"conversation_id", s->conversation_id}, + {"is_done", s->is_done()}, + {"total_bytes", s->total_size()}, + {"started_at", s->started_ts}, + {"completed_at", s->completed_at()}, + }); + } + auto res = std::make_unique(); + res->status = 200; + res->content_type = "application/json; charset=utf-8"; + res->data = safe_json_to_str(arr); + return res; + }; +} + +server_http_context::handler_t make_stream_delete_handler() { + return [](const server_http_req & req) -> server_http_res_ptr { + // DELETE /v1/stream/ is the explicit user Stop, cancels the producer hook + // wired by handle_completions_impl and evicts the buffer. idempotent, a session that + // already finalized or was never created returns 204 either way + std::string conv_id = req.get_param("conv_id"); + if (conv_id.empty()) { + return make_error_response(400, "Missing conversation id in path", ERROR_TYPE_INVALID_REQUEST); + } + SRV_INF("DELETE /v1/stream/%s -> evict_and_cancel\n", conv_id.c_str()); + g_stream_sessions.evict_and_cancel(conv_id); + auto res = std::make_unique(); + res->status = 204; + res->content_type = "application/json"; + return res; + }; +} diff --git a/tools/server/server-stream.h b/tools/server/server-stream.h index f4ee9c7893f6..7a4c1761c489 100644 --- a/tools/server/server-stream.h +++ b/tools/server/server-stream.h @@ -1,5 +1,7 @@ #pragma once +#include "server-http.h" + #include #include #include @@ -117,3 +119,18 @@ class stream_session_manager { std::condition_variable gc_wake_cv; std::shared_ptr> drain_shutdown; }; + +// the process wide stream session manager. defined in server-stream.cpp so the symbol +// resolves through the server-context static lib, both llama-server and llama-cli link it. +// start_gc() and stop_gc() are called explicitly from llama-server main(), llama-cli never +// touches it and leaves it idle. the destructor calls stop_gc() unconditionally so the +// process exit path is safe whether or not the GC thread was started +extern stream_session_manager g_stream_sessions; + +// route handler factories. each builds a server_http_context::handler_t that operates +// directly on g_stream_sessions, server.cpp wires them under /v1/stream/* without going +// through server-context's server_routes. keeps the resumable stream surface confined to +// server-stream and server-http +server_http_context::handler_t make_stream_get_handler(); +server_http_context::handler_t make_streams_list_handler(); +server_http_context::handler_t make_stream_delete_handler(); diff --git a/tools/server/server.cpp b/tools/server/server.cpp index ee98aa7caa71..e97429e427b1 100644 --- a/tools/server/server.cpp +++ b/tools/server/server.cpp @@ -2,6 +2,7 @@ #include "server-http.h" #include "server-models.h" #include "server-cors-proxy.h" +#include "server-stream.h" #include "server-tools.h" #include "arg.h" @@ -82,6 +83,10 @@ int llama_server(int argc, char ** argv) { common_init(); + // start the stream session manager GC right after common init, before any HTTP route can + // touch it. lifecycle is symmetric, stop_gc() runs in clean_up() before backend free + g_stream_sessions.start_gc(); + if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_SERVER)) { return 1; } @@ -184,9 +189,6 @@ int llama_server(int argc, char ** argv) { routes.post_lora_adapters = models_routes->proxy_post; routes.get_slots = models_routes->proxy_get; routes.post_slots = models_routes->proxy_post; - routes.get_stream = models_routes->proxy_get_stream; - routes.get_streams = models_routes->proxy_get_streams; - routes.delete_stream = models_routes->proxy_delete_stream; // custom routes for router routes.get_props = models_routes->get_router_props; @@ -241,14 +243,25 @@ int llama_server(int argc, char ** argv) { ctx_http.get ("/slots", ex_wrapper(routes.get_slots)); ctx_http.post("/slots/:id_slot", ex_wrapper(routes.post_slots)); - // resumable streaming, the conversation_id is the session identity end to end: - // GET /v1/stream/?from=N replays SSE bytes for a session in progress or recently completed - ctx_http.get ("/v1/stream/:conv_id", ex_wrapper(routes.get_stream)); - // GET /v1/streams lists sessions, with optional conversation_id query to filter to one conv, - // without filter the WebUI uses it at mount and on visibilitychange to populate sidebar spinners - ctx_http.get ("/v1/streams", ex_wrapper(routes.get_streams)); - // DELETE /v1/stream/ is the explicit user Stop, cancels the producer and evicts, idempotent - ctx_http.del_("/v1/stream/:conv_id", ex_wrapper(routes.delete_stream)); + // resumable streaming, the conversation_id is the session identity end to end. router and + // child wire different handlers under the same paths: a child binds the local g_stream_sessions + // backed factories, the router binds proxies that route via the optional ::model suffix + // (direct) or fall back to loopback probe and fan out (suffixless conv ids) + server_http_context::handler_t stream_get_h; + server_http_context::handler_t streams_list_h; + server_http_context::handler_t stream_delete_h; + if (is_router_server) { + stream_get_h = models_routes->router_stream_get; + streams_list_h = models_routes->router_streams_list; + stream_delete_h = models_routes->router_stream_delete; + } else { + stream_get_h = make_stream_get_handler(); + streams_list_h = make_streams_list_handler(); + stream_delete_h = make_stream_delete_handler(); + } + ctx_http.get ("/v1/stream/:conv_id", ex_wrapper(stream_get_h)); + ctx_http.get ("/v1/streams", ex_wrapper(streams_list_h)); + ctx_http.del_("/v1/stream/:conv_id", ex_wrapper(stream_delete_h)); // Google Cloud Platform (Vertex AI) compat ctx_http.register_gcp_compat(); @@ -300,6 +313,9 @@ int llama_server(int argc, char ** argv) { clean_up = [&models_routes]() { SRV_INF("%s: cleaning up before exit...\n", __func__); + // stop the session GC first, this finalizes every live session and wakes any + // pending HTTP reader, the detached drains can then exit cleanly + g_stream_sessions.stop_gc(); if (models_routes.has_value()) { models_routes->stopping.store(true); // maybe redundant, but just to be safe models_routes->models.unload_all(); @@ -326,6 +342,9 @@ int llama_server(int argc, char ** argv) { // setup clean up function, to be called before exit clean_up = [&ctx_http, &ctx_server]() { SRV_INF("%s: cleaning up before exit...\n", __func__); + // stop the session GC first, this finalizes every live session and wakes any + // pending HTTP reader, the detached drains can then exit cleanly + g_stream_sessions.stop_gc(); ctx_http.stop(); ctx_server.terminate(); llama_backend_free(); diff --git a/tools/ui/src/lib/services/chat.service.ts b/tools/ui/src/lib/services/chat.service.ts index 1cc28e7f07b7..1de49950b347 100644 --- a/tools/ui/src/lib/services/chat.service.ts +++ b/tools/ui/src/lib/services/chat.service.ts @@ -1,6 +1,7 @@ import { getJsonHeaders } from '$lib/utils/api-headers'; import { formatAttachmentText } from '$lib/utils/formatters'; import { isAbortError } from '$lib/utils/abort'; +import { streamIdentity } from '$lib/utils/stream-identity'; import { saveStreamState, clearStreamState, @@ -39,7 +40,7 @@ import type { DatabaseMessageExtraMcpResource, StreamConnectionState } from '$lib/types'; -import { modelsStore } from '$lib/stores/models.svelte'; +import { modelsStore, selectedModelName } from '$lib/stores/models.svelte'; import { settingsStore } from '../stores/settings.svelte'; import { capImageDataURLSize } from '../utils/cap-img-size'; @@ -321,9 +322,10 @@ export class ChatService { try { const headers: Record = { ...getJsonHeaders() }; // tag streaming requests with the conversation id, this single header is the opt in for the - // server side replay buffer and powers discoverActiveStream on tab reopen + // server side replay buffer and powers discoverActiveStream on tab reopen. with an explicit + // model the ::model suffix lets the router skip the loopback probe if (stream && conversationId) { - headers['X-Conversation-Id'] = conversationId; + headers['X-Conversation-Id'] = streamIdentity(conversationId, options.model); } const response = await fetch(API_CHAT.COMPLETIONS, { method: 'POST', @@ -487,10 +489,11 @@ export class ChatService { * @param excludeReasoning - Whether to strip reasoning content (should match excludeReasoningFromContext setting) * @param signal - Optional AbortSignal to cancel the pre-encode request */ - static async cancelServerStream(conversationId: string): Promise { + static async cancelServerStream(conversationId: string, model?: string | null): Promise { if (!conversationId) return; try { - await fetch(`./v1/stream/${encodeURIComponent(conversationId)}`, { method: 'DELETE' }); + const id = streamIdentity(conversationId, model); + await fetch(`./v1/stream/${encodeURIComponent(id)}`, { method: 'DELETE' }); } catch (e) { console.warn('cancelServerStream failed:', e); } @@ -804,8 +807,13 @@ export class ChatService { madeProgress = false; // the server resends starting at bytesParsed, discard any partial line we held - // it will be retransmitted from a clean line boundary - const resumeResp = await resumeStream(conversationId, abortSignal).catch(() => null); + // it will be retransmitted from a clean line boundary. pass the active model name + // so the router routes the GET direct to the owning child, skips the loopback probe + const resumeResp = await resumeStream( + conversationId, + abortSignal, + selectedModelName() + ).catch(() => null); if (!resumeResp || resumeResp.status !== 200) { onConnectionState?.('lost'); onError?.(new Error('Stream connection lost and could not be resumed')); diff --git a/tools/ui/src/lib/services/stream-resume.service.ts b/tools/ui/src/lib/services/stream-resume.service.ts index 5c3664fb3078..879304c043e5 100644 --- a/tools/ui/src/lib/services/stream-resume.service.ts +++ b/tools/ui/src/lib/services/stream-resume.service.ts @@ -7,6 +7,8 @@ * client localStorage, /v1/stream/ routes), no extra opaque token. */ +import { streamIdentity } from '$lib/utils/stream-identity'; + interface ResumableStreamState { bytesReceived: number; updatedAt: number; @@ -66,11 +68,13 @@ export function clearStreamState(conversationId: string): void { */ export async function resumeStream( conversationId: string, - signal?: AbortSignal + signal?: AbortSignal, + model?: string | null ): Promise { if (!conversationId) return null; const state = getStreamState(conversationId); const from = state?.bytesReceived ?? 0; - const url = `./v1/stream/${encodeURIComponent(conversationId)}?from=${from}`; + const id = streamIdentity(conversationId, model); + const url = `./v1/stream/${encodeURIComponent(id)}?from=${from}`; return await fetch(url, { method: 'GET', signal }); } diff --git a/tools/ui/src/lib/stores/chat.svelte.ts b/tools/ui/src/lib/stores/chat.svelte.ts index dd4995efc0ae..4a5303a3cfbf 100644 --- a/tools/ui/src/lib/stores/chat.svelte.ts +++ b/tools/ui/src/lib/stores/chat.svelte.ts @@ -16,6 +16,7 @@ import { DatabaseService } from '$lib/services/database.service'; import { ChatService } from '$lib/services/chat.service'; import { selectActiveStream } from '$lib/services/stream-discovery.service'; import { getStreamState, clearStreamState } from '$lib/services/stream-resume.service'; +import { streamIdentity } from '$lib/utils/stream-identity'; import { conversationsStore } from '$lib/stores/conversations.svelte'; import { config } from '$lib/stores/settings.svelte'; import { agenticStore } from '$lib/stores/agentic.svelte'; @@ -194,7 +195,7 @@ class ChatStore { return selectActiveStream(sessions); } - async attachServerStream(convId: string): Promise { + async attachServerStream(convId: string, streamId?: string): Promise { if (!convId) return; if (this.chatStreamingStates.has(convId)) return; @@ -210,10 +211,12 @@ class ChatStore { }; // fetch the replay stream from byte 0, rebuild the assistant message from scratch. - // the conv id is the only identifier we need, end to end + // resolve the server side identity, fall back to streamIdentity when the caller does not + // pass a streamId. probeServerStream returns the full id (with ::model suffix when present) + const id = streamId || streamIdentity(convId, selectedModelName()); let response: Response; try { - response = await fetch(`./v1/stream/${encodeURIComponent(convId)}?from=0`); + response = await fetch(`./v1/stream/${encodeURIComponent(id)}?from=0`); } catch (e) { console.error('attachServerStream replay fetch failed:', e); unlock(); @@ -364,7 +367,9 @@ class ChatStore { // primary path: ask the server which sessions exist for this conversation const serverTarget = await this.probeServerStream(convId); if (serverTarget) { - await this.attachServerStream(convId); + // pass the full server side identity (may carry a ::model suffix) so the GET routes + // straight to the owning session, no probe or fan out + await this.attachServerStream(convId, serverTarget.conversation_id); return; } @@ -1284,7 +1289,7 @@ class ChatStore { // tell the server to stop the generation, not just to drop the HTTP socket. without this // the detached drain keeps producing tokens until eos or max_tokens. the conv id is the // session identity so the DELETE call is straight - void ChatService.cancelServerStream(convId); + void ChatService.cancelServerStream(convId, selectedModelName()); this.abortRequest(convId); this.setChatLoading(convId, false); this.clearChatStreaming(convId); diff --git a/tools/ui/src/lib/utils/stream-identity.ts b/tools/ui/src/lib/utils/stream-identity.ts new file mode 100644 index 000000000000..8c900a5b11c0 --- /dev/null +++ b/tools/ui/src/lib/utils/stream-identity.ts @@ -0,0 +1,15 @@ +/** + * Build the conversation identity used by the server side replay buffer. + * + * The server identifies a stream session by a conversation id sent in the + * X-Conversation-Id header. When the user has explicitly picked a model the + * client appends ::modelName, which lets the router fan out direct to that + * child for resume and stop without probing every other one. Without the + * suffix the router falls back to a loopback probe and a DELETE fan out, both + * still correct, just slower at the lookup step. + */ +export function streamIdentity(conversationId: string, model?: string | null): string { + if (!conversationId) return ''; + if (!model) return conversationId; + return `${conversationId}::${model}`; +} From ee592f263252f75baebf4c078f6231dacc1363db Mon Sep 17 00:00:00 2001 From: Pascal Date: Mon, 18 May 2026 11:07:18 +0200 Subject: [PATCH 06/39] server-context: cleaning --- tools/server/server-context.cpp | 59 +++++---------------------------- tools/server/server-stream.cpp | 51 ++++++++++++++++++++++++++++ tools/server/server-stream.h | 16 +++++++++ 3 files changed, 75 insertions(+), 51 deletions(-) diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 3349de0fe96e..2309c3455502 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -4135,28 +4135,6 @@ std::unique_ptr server_routes::handle_completions_impl( bool stream = json_value(data, "stream", false); - // resumable streaming opt in, a non empty X-Conversation-Id on a streaming request enables it. - // the conv id is the session identity end to end, client localStorage and server map share - // the same key. non stream and non opted in calls keep the standard flow unchanged - std::string conversation_id; - if (stream) { - static constexpr char target[] = "x-conversation-id"; - static constexpr size_t target_len = sizeof(target) - 1; - for (const auto & [hk, hv] : req.headers) { - if (hk.size() != target_len) continue; - bool match = true; - for (size_t i = 0; i < target_len; ++i) { - char c = hk[i]; - if (c >= 'A' && c <= 'Z') c = char(c + 32); - if (c != target[i]) { match = false; break; } - } - if (match) { - conversation_id = hv; - break; - } - } - } - if (!stream) { // non-stream, wait for the results auto all_results = rd.wait_for_all(req.should_stop); @@ -4233,16 +4211,9 @@ std::unique_ptr server_routes::handle_completions_impl( } }; - // when a tee is attached the session must outlive the http socket. ignore the - // connection closed signal in that case, only an explicit DELETE through the - // session stop_producer hook (which calls rd.stop()) is allowed to abort the - // producer. without a tee the legacy flow stays bit identical - auto effective_should_stop = [res_this, &req]() -> bool { - if (res_this->tee) { - return false; - } - return req.should_stop(); - }; + // delegate to server-stream so the tee-aware rule lives there, not here. without a + // tee attached this is bit identical to req.should_stop, the legacy flow is unchanged + auto effective_should_stop = stream_aware_should_stop(res_this, req.should_stop); try { if (effective_should_stop()) { @@ -4336,25 +4307,11 @@ std::unique_ptr server_routes::handle_completions_impl( }; } - // attach the resumable session, tee mirrors each SSE chunk into the ring buffer, on_stream_end - // finalizes it on either drain path (wire or detached), and the stop_producer hook lets the - // DELETE /v1/stream/ route abort the underlying reader from anywhere. the http layer - // owns the keep alive after a client disconnect via its detached drain, server-context only - // declares the wiring here and never spawns a thread itself - if (!conversation_id.empty()) { - auto session = g_stream_sessions.create_or_replace(conversation_id); - session->set_stop_producer([res_this = res.get()] { - res_this->rd.stop(); - }); - res->tee = [session](const char * d, size_t n) { - session->append(d, n); - }; - res->on_stream_end = [session] { - // detach the stop hook before the response goes out of scope, no dangling captured ptr - session->set_stop_producer(nullptr); - session->finalize(); - }; - } + // delegate the X-Conversation-Id sniff and the three hook wirings (tee, on_stream_end, + // stop_producer) to server-stream. when the header is absent this is a no op, when set + // the session is created or replaced and the response carries the tee that mirrors every + // SSE chunk into the ring buffer, plus the cancel hook for DELETE /v1/stream/ + stream_session_attach_hooks(*res, res->rd, req.headers); return res; } diff --git a/tools/server/server-stream.cpp b/tools/server/server-stream.cpp index 4bcc26bb9c7c..7213f21e7729 100644 --- a/tools/server/server-stream.cpp +++ b/tools/server/server-stream.cpp @@ -1,6 +1,7 @@ #include "server-stream.h" #include "server-common.h" #include "server-http.h" +#include "server-queue.h" #include #include @@ -406,3 +407,53 @@ server_http_context::handler_t make_stream_delete_handler() { return res; }; } + +void stream_session_attach_hooks(server_http_res & res, server_response_reader & rd, const std::map & headers) { + // case insensitive scan for x-conversation-id. headers preserve the wire casing, an ASCII + // tolower comparison is enough here, no locale machinery needed + static constexpr char target[] = "x-conversation-id"; + static constexpr size_t target_len = sizeof(target) - 1; + std::string conversation_id; + for (const auto & [hk, hv] : headers) { + if (hk.size() != target_len) continue; + bool match = true; + for (size_t i = 0; i < target_len; ++i) { + char c = hk[i]; + if (c >= 'A' && c <= 'Z') c = char(c + 32); + if (c != target[i]) { match = false; break; } + } + if (match) { + conversation_id = hv; + break; + } + } + if (conversation_id.empty()) { + return; + } + auto session = g_stream_sessions.create_or_replace(conversation_id); + session->set_stop_producer([rd_ptr = &rd] { + rd_ptr->stop(); + }); + res.tee = [session](const char * d, size_t n) { + session->append(d, n); + }; + res.on_stream_end = [session] { + // detach the stop hook before the response goes out of scope, no dangling captured ptr + session->set_stop_producer(nullptr); + session->finalize(); + }; +} + +std::function stream_aware_should_stop(server_http_res * res, std::function fallback) { + // capture fallback by value, the closure stays valid even after the original local goes + // out of scope, the std::function copy keeps the underlying target alive + return [res, fallback = std::move(fallback)]() -> bool { + if (res->tee) { + // a tee is attached, the session must outlive the http socket. ignore the peer + // disconnect signal, only an explicit DELETE through the session stop_producer + // hook is allowed to abort the producer + return false; + } + return fallback(); + }; +} diff --git a/tools/server/server-stream.h b/tools/server/server-stream.h index 7a4c1761c489..d21297922a08 100644 --- a/tools/server/server-stream.h +++ b/tools/server/server-stream.h @@ -134,3 +134,19 @@ extern stream_session_manager g_stream_sessions; server_http_context::handler_t make_stream_get_handler(); server_http_context::handler_t make_streams_list_handler(); server_http_context::handler_t make_stream_delete_handler(); + +// attach a resumable session to an outgoing streaming response. inspects the request headers +// for X-Conversation-Id, and when present creates or replaces a session on the global manager, +// then wires three closures on the response: tee mirrors each SSE chunk into the ring buffer, +// on_stream_end finalizes the session, and the session's stop_producer hook calls rd.stop() +// so the explicit DELETE /v1/stream/ route can abort the underlying reader. no op +// when the header is absent. server-context just calls this and never touches the manager +struct server_response_reader; // forward declare to avoid pulling server-queue.h into the header +void stream_session_attach_hooks(server_http_res & res, server_response_reader & rd, const std::map & headers); + +// build a should_stop closure that suppresses the peer disconnect signal when a tee is +// attached to the response. used by handler lambdas that pump a server_response_reader so +// the producer keeps running past F5, only an explicit DELETE through the stop_producer +// hook is allowed to abort it. without a tee the returned closure is bit identical to the +// fallback, the legacy non resumable flow is unchanged +std::function stream_aware_should_stop(server_http_res * res, std::function fallback); From c5db9f6acc133ca6a42d2e19072b4359e949565b Mon Sep 17 00:00:00 2001 From: Pascal Date: Mon, 18 May 2026 20:45:32 +0200 Subject: [PATCH 07/39] server-stream: fix use-after-free on rd Guard stop_producer with a shared alive flag, flipped by on_stream_end before rd dies. Prevents a late cancel (session eviction by a later POST on the same conv_id, or a DELETE arriving after the producer ended) from touching a destroyed rd. --- tools/server/server-stream.cpp | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tools/server/server-stream.cpp b/tools/server/server-stream.cpp index 7213f21e7729..48ef5349c678 100644 --- a/tools/server/server-stream.cpp +++ b/tools/server/server-stream.cpp @@ -431,14 +431,22 @@ void stream_session_attach_hooks(server_http_res & res, server_response_reader & return; } auto session = g_stream_sessions.create_or_replace(conversation_id); - session->set_stop_producer([rd_ptr = &rd] { - rd_ptr->stop(); + // stop_producer may outlive the response (eviction by a later POST on the same conv_id, + // late DELETE). guard with a shared alive flag, flipped by on_stream_end before rd dies + auto alive = std::make_shared>(true); + auto * rd_ptr = &rd; + session->set_stop_producer([alive, rd_ptr] { + if (alive->load(std::memory_order_acquire)) { + rd_ptr->stop(); + } }); res.tee = [session](const char * d, size_t n) { session->append(d, n); }; - res.on_stream_end = [session] { - // detach the stop hook before the response goes out of scope, no dangling captured ptr + res.on_stream_end = [session, alive] { + // flip alive first so any concurrent cancel (eg. from a create_or_replace racing with + // our natural end) skips the now invalid rd. then detach the stop hook and finalize + alive->store(false, std::memory_order_release); session->set_stop_producer(nullptr); session->finalize(); }; From 5e4ea0437c3c3f232fdadd3474043ccabbd6fd07 Mon Sep 17 00:00:00 2001 From: Pascal Date: Mon, 18 May 2026 20:45:48 +0200 Subject: [PATCH 08/39] ui: fix cross-conversation contamination Scope streaming flags per conv so one finishing does not unflag the others, guard discoverActiveStream against concurrent runs to avoid duplicate attaches, and stop racing syncRemoteRunningStreams for the sidebar set. --- tools/ui/src/lib/stores/chat.svelte.ts | 139 +++++++++++++++++-------- 1 file changed, 98 insertions(+), 41 deletions(-) diff --git a/tools/ui/src/lib/stores/chat.svelte.ts b/tools/ui/src/lib/stores/chat.svelte.ts index 4a5303a3cfbf..1e3022f412a9 100644 --- a/tools/ui/src/lib/stores/chat.svelte.ts +++ b/tools/ui/src/lib/stores/chat.svelte.ts @@ -80,6 +80,14 @@ class ChatStore { // at app mount and on visibilitychange. it does not overlap with chatLoadingStates which // tracks inferences driven by this browser, both are unioned to feed the sidebar spinners private remoteRunningConvs = new SvelteSet(); + // per conv attach lifecycle, used to derive the global streaming flag without flipping it + // off when one conv finishes while another is still streaming. mirrors chatLoadingStates + // in scope but tracks the attach + tee replay path specifically + private attachingConvs = new SvelteSet(); + // in-flight discoverActiveStream guard, keyed by conv id. prevents a fast remount + visibility + // race from launching two concurrent attaches on the same conv (which would dup chunks into + // the same DB message) + private discoveringConvs = new SvelteSet(); private abortControllers = new SvelteMap(); private preEncodeAbortController: AbortController | null = null; private processingStates = new SvelteMap(); @@ -199,13 +207,24 @@ class ChatStore { if (!convId) return; if (this.chatStreamingStates.has(convId)) return; - // flip the spinner immediately, the user sees activity as soon as the conv becomes active + // flip the spinner immediately, the user sees activity as soon as the conv becomes active. + // the global isStreamingActive flag is derived from attachingConvs.size, so adding here + // turns it on, and removing in unlock only turns it off when this is the last attach this.setChatLoading(convId, true); + this.attachingConvs.add(convId); this.setStreamingActive(true); - this.setActiveProcessingConversation(convId); + // only set the active processing conv if we are looking at it, otherwise a background + // attach would steal the indicator from the conv the user is currently viewing + if (convId === conversationsStore.activeConversation?.id) { + this.setActiveProcessingConversation(convId); + } const unlock = () => { - this.setStreamingActive(false); + this.attachingConvs.delete(convId); + // flip the global flag off only when no other conv is still attaching + if (this.attachingConvs.size === 0) { + this.setStreamingActive(false); + } this.setChatLoading(convId, false); this.clearChatStreaming(convId); }; @@ -228,8 +247,22 @@ class ChatStore { return; } - // locate the slot to splice into, create a placeholder assistant message if there is none - let messages = conversationsStore.activeMessages as DatabaseMessage[]; + // load the target conversation messages by id, not via the active store. when multiple + // attaches run in parallel the active store may reflect another conv and writing through + // its index mixes content across convs (CoT flicker, message bleed). by going through the + // DB we stay isolated, and only mirror into the active store when the attached conv is + // the one currently displayed + let messages: DatabaseMessage[]; + try { + messages = await DatabaseService.getConversationMessages(convId); + } catch (e) { + console.error('attachServerStream load messages failed:', e); + unlock(); + return; + } + + // locate the slot to splice into, create a placeholder assistant message if there is none. + // we use the conv-scoped findLastAssistantIdx helpers, they only depend on the array let targetIdx = this.findLastAssistantIdx(messages); if (targetIdx === -1) { const lastUserIdx = this.findLastUserIdx(messages); @@ -254,9 +287,12 @@ class ChatStore { } as Omit, messages[lastUserIdx].id ); - conversationsStore.addMessageToActive(placeholder); - messages = conversationsStore.activeMessages as DatabaseMessage[]; - targetIdx = this.findLastAssistantIdx(messages); + messages = [...messages, placeholder]; + targetIdx = messages.length - 1; + // only push into the active store when this conv is the one displayed right now + if (convId === conversationsStore.activeConversation?.id) { + conversationsStore.addMessageToActive(placeholder); + } } catch (e) { console.error('attachServerStream placeholder creation failed:', e); unlock(); @@ -276,11 +312,21 @@ class ChatStore { const existingContent = targetMessage.content ?? ''; const existingReasoning = targetMessage.reasoningContent ?? ''; const isAppendMode = existingContent.length > 0; + + // helper: write to the active store only when the attached conv is currently displayed. + // the lookup by message id is robust to reordering of activeMessages, two parallel attaches + // can no longer step on each other's indices + const writeActive = (updates: Partial) => { + if (convId !== conversationsStore.activeConversation?.id) { + return; + } + const liveIdx = conversationsStore.findMessageIndex(targetMessageId); + if (liveIdx === -1) return; + conversationsStore.updateMessageAtIndex(liveIdx, updates); + }; + if (!isAppendMode) { - conversationsStore.updateMessageAtIndex(targetIdx, { - content: '', - reasoningContent: undefined - }); + writeActive({ content: '', reasoningContent: undefined }); } this.setChatStreaming(convId, existingContent, targetMessageId); @@ -300,7 +346,7 @@ class ChatStore { (chunk: string) => { streamedContent += chunk; const displayed = isAppendMode ? existingContent + streamedContent : streamedContent; - conversationsStore.updateMessageAtIndex(targetIdx, { content: displayed }); + writeActive({ content: displayed }); this.setChatStreaming(convId, displayed, targetMessageId); }, async ( @@ -313,13 +359,15 @@ class ChatStore { const streamedR = streamedReasoningContent || reasoningContent || ''; const content = isAppendMode ? existingContent + streamed : streamed; const reasoning = isAppendMode ? existingReasoning + streamedR : streamedR; + // the DB write is the source of truth, mirror to the active store only when + // the conv is currently displayed await DatabaseService.updateMessage(targetMessageId, { content, reasoningContent: reasoning || undefined, toolCalls: toolCalls || '', timings }); - conversationsStore.updateMessageAtIndex(targetIdx, { + writeActive({ content, reasoningContent: reasoning || undefined, timings @@ -335,9 +383,7 @@ class ChatStore { const displayed = isAppendMode ? existingReasoning + streamedReasoningContent : streamedReasoningContent; - conversationsStore.updateMessageAtIndex(targetIdx, { - reasoningContent: displayed - }); + writeActive({ reasoningContent: displayed }); }, undefined, undefined, @@ -360,30 +406,36 @@ class ChatStore { if (!convId) return; if (this.chatStreamingStates.has(convId)) return; if (this.chatLoadingStates.get(convId)) return; - // the sidebar spinner hint is consumed the moment we run the authoritative probe, so a - // finalized session no longer ghosts in the sidebar after navigation - this.remoteRunningConvs.delete(convId); - - // primary path: ask the server which sessions exist for this conversation - const serverTarget = await this.probeServerStream(convId); - if (serverTarget) { - // pass the full server side identity (may carry a ::model suffix) so the GET routes - // straight to the owning session, no probe or fan out - await this.attachServerStream(convId, serverTarget.conversation_id); - return; - } + // concurrency guard: another discover may already be running for this conv (typical race + // between mount and visibilitychange on tab switch). a second concurrent fetch on the same + // /v1/stream/ would duplicate every byte into the DB message, this guard bounces it + if (this.discoveringConvs.has(convId)) return; + this.discoveringConvs.add(convId); - // fallback: local state remembers an interrupted byte offset for this conv, the server may - // still have a live session matching that conv id (we just lost the bytes mid stream). try to - // attach with conv id only, the server probe inside attachServerStream tells us if it exists - const localState = getStreamState(convId); - if (!localState) { - return; - } - await this.attachServerStream(convId); - // if attachServerStream failed (session gone, TTL expired), clear the local state to avoid retrying forever - if (!this.chatStreamingStates.has(convId) && !this.chatLoadingStates.get(convId)) { - clearStreamState(convId); + try { + // primary path: ask the server which sessions exist for this conversation + const serverTarget = await this.probeServerStream(convId); + if (serverTarget) { + // pass the full server side identity (may carry a ::model suffix) so the GET routes + // straight to the owning session, no probe or fan out + await this.attachServerStream(convId, serverTarget.conversation_id); + return; + } + + // fallback: local state remembers an interrupted byte offset for this conv, the server may + // still have a live session matching that conv id (we just lost the bytes mid stream). try to + // attach with conv id only, the server probe inside attachServerStream tells us if it exists + const localState = getStreamState(convId); + if (!localState) { + return; + } + await this.attachServerStream(convId); + // if attachServerStream failed (session gone, TTL expired), clear the local state to avoid retrying forever + if (!this.chatStreamingStates.has(convId) && !this.chatLoadingStates.get(convId)) { + clearStreamState(convId); + } + } finally { + this.discoveringConvs.delete(convId); } } @@ -561,7 +613,12 @@ class ChatStore { const running = new SvelteSet(); for (const s of sessions) { if (s && !s.is_done && typeof s.conversation_id === 'string' && s.conversation_id) { - running.add(s.conversation_id); + // strip the optional ::model suffix, the sidebar lookup is keyed by the bare conv id + // straight from the DB. without this the sidebar spinner never matches and stays off + // when the running session was started with an explicit model + const sepIdx = s.conversation_id.indexOf('::'); + const bareId = sepIdx === -1 ? s.conversation_id : s.conversation_id.slice(0, sepIdx); + running.add(bareId); } } for (const id of Array.from(this.remoteRunningConvs)) { From 14daa7f5b7f4e8346982403b39e251cdf250a963 Mon Sep 17 00:00:00 2001 From: Pascal Date: Mon, 18 May 2026 20:46:04 +0200 Subject: [PATCH 09/39] server-http: keep request alive in detached SSE drain The response next() lambda may reach into *request via &req long after on_complete reset the request shared_ptr. Capture request in the detached thread so it outlives the drain. --- tools/server/server-http.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tools/server/server-http.cpp b/tools/server/server-http.cpp index c823aef9dbf5..d13ec78a1ebb 100644 --- a/tools/server/server-http.cpp +++ b/tools/server/server-http.cpp @@ -537,10 +537,12 @@ static void process_handler_response(server_http_req_ptr && request, server_http // if the producer is still running when httplib hands the response back, the peer // is gone but the generation must keep going. detach a thread that pumps next() // into the tee until done, then runs on_stream_end. capture by value keeps the - // response alive past the reset below, the detached thread holds its own reference + // response alive past the reset below, the request too since next() may reach + // into *request via &req if (!stream_done->load(std::memory_order_acquire) && response->tee) { + auto req_keep = request; auto resp_keep = response; - std::thread([resp_keep]() { + std::thread([req_keep, resp_keep]() { std::string c; while (true) { c.clear(); @@ -555,6 +557,7 @@ static void process_handler_response(server_http_req_ptr && request, server_http if (resp_keep->on_stream_end) { resp_keep->on_stream_end(); } + (void) req_keep; // keep the request alive until the drain is done }).detach(); } response.reset(); // trigger the destruction of the response object From 4e7c3a2d988a11d4a04124b8e4888d9d2b1244ef Mon Sep 17 00:00:00 2001 From: Pascal Date: Mon, 18 May 2026 22:27:24 +0200 Subject: [PATCH 10/39] ui: address review feedback from coder543 Forward Authorization to /v1/stream and /v1/streams fetches, the resumable routes must obey --api-key like the rest of the API. Wrap reader.read() in a try/catch, the underlying connection drop rejects with TypeError instead of resolving done=true, treat it as a premature end of stream so the existing resume loop kicks in. Freeze the model at session start in chatStreamingStates.model and thread it through cancel and resume, the dropdown selection may have changed since the POST and the server side identity is fixed at that time. --- tools/ui/src/lib/services/chat.service.ts | 38 +++++++++++++++---- .../src/lib/services/stream-resume.service.ts | 3 +- tools/ui/src/lib/stores/chat.svelte.ts | 36 ++++++++++++------ 3 files changed, 57 insertions(+), 20 deletions(-) diff --git a/tools/ui/src/lib/services/chat.service.ts b/tools/ui/src/lib/services/chat.service.ts index 1de49950b347..64e5ff768210 100644 --- a/tools/ui/src/lib/services/chat.service.ts +++ b/tools/ui/src/lib/services/chat.service.ts @@ -1,4 +1,4 @@ -import { getJsonHeaders } from '$lib/utils/api-headers'; +import { getAuthHeaders, getJsonHeaders } from '$lib/utils/api-headers'; import { formatAttachmentText } from '$lib/utils/formatters'; import { isAbortError } from '$lib/utils/abort'; import { streamIdentity } from '$lib/utils/stream-identity'; @@ -357,7 +357,8 @@ export class ChatService { onTimings, conversationId, signal, - onConnectionState + onConnectionState, + options.model ); return; @@ -493,7 +494,10 @@ export class ChatService { if (!conversationId) return; try { const id = streamIdentity(conversationId, model); - await fetch(`./v1/stream/${encodeURIComponent(id)}`, { method: 'DELETE' }); + await fetch(`./v1/stream/${encodeURIComponent(id)}`, { + method: 'DELETE', + headers: getAuthHeaders() + }); } catch (e) { console.warn('cancelServerStream failed:', e); } @@ -600,7 +604,8 @@ export class ChatService { onTimings?: (timings?: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => void, conversationId?: string, abortSignal?: AbortSignal, - onConnectionState?: (state: StreamConnectionState) => void + onConnectionState?: (state: StreamConnectionState) => void, + streamModel?: string | null ): Promise { let reader = response.body?.getReader(); @@ -699,7 +704,23 @@ export class ChatService { while (true) { if (abortSignal?.aborted) break; - const { done, value } = await reader.read(); + let done: boolean; + let value: Uint8Array | undefined; + try { + const r = await reader.read(); + done = r.done; + value = r.value; + } catch (readErr) { + // reader.read() rejects with TypeError when the underlying connection drops + // instead of just resolving with done=true. treat it like done so the outer + // loop swaps reader via the resume path + if (isAbortError(readErr)) { + throw readErr; + } + console.warn('reader.read() rejected, treating as premature end:', readErr); + done = true; + value = undefined; + } if (done) break; if (abortSignal?.aborted) break; @@ -807,12 +828,13 @@ export class ChatService { madeProgress = false; // the server resends starting at bytesParsed, discard any partial line we held - // it will be retransmitted from a clean line boundary. pass the active model name - // so the router routes the GET direct to the owning child, skips the loopback probe + // it will be retransmitted from a clean line boundary. reuse the model the POST was + // originally tagged with, the dropdown may have changed since but the server side + // identity is frozen at POST time const resumeResp = await resumeStream( conversationId, abortSignal, - selectedModelName() + streamModel ).catch(() => null); if (!resumeResp || resumeResp.status !== 200) { onConnectionState?.('lost'); diff --git a/tools/ui/src/lib/services/stream-resume.service.ts b/tools/ui/src/lib/services/stream-resume.service.ts index 879304c043e5..e1a9e3a40946 100644 --- a/tools/ui/src/lib/services/stream-resume.service.ts +++ b/tools/ui/src/lib/services/stream-resume.service.ts @@ -8,6 +8,7 @@ */ import { streamIdentity } from '$lib/utils/stream-identity'; +import { getAuthHeaders } from '$lib/utils/api-headers'; interface ResumableStreamState { bytesReceived: number; @@ -76,5 +77,5 @@ export async function resumeStream( const from = state?.bytesReceived ?? 0; const id = streamIdentity(conversationId, model); const url = `./v1/stream/${encodeURIComponent(id)}?from=${from}`; - return await fetch(url, { method: 'GET', signal }); + return await fetch(url, { method: 'GET', signal, headers: getAuthHeaders() }); } diff --git a/tools/ui/src/lib/stores/chat.svelte.ts b/tools/ui/src/lib/stores/chat.svelte.ts index 1e3022f412a9..df79bbddd62a 100644 --- a/tools/ui/src/lib/stores/chat.svelte.ts +++ b/tools/ui/src/lib/stores/chat.svelte.ts @@ -17,6 +17,7 @@ import { ChatService } from '$lib/services/chat.service'; import { selectActiveStream } from '$lib/services/stream-discovery.service'; import { getStreamState, clearStreamState } from '$lib/services/stream-resume.service'; import { streamIdentity } from '$lib/utils/stream-identity'; +import { getAuthHeaders } from '$lib/utils/api-headers'; import { conversationsStore } from '$lib/stores/conversations.svelte'; import { config } from '$lib/stores/settings.svelte'; import { agenticStore } from '$lib/stores/agentic.svelte'; @@ -75,7 +76,7 @@ class ChatStore { streamConnectionState = $state('streaming'); chatLoadingStates = new SvelteMap(); chatReasoningStates = new SvelteMap(); - chatStreamingStates = new SvelteMap(); + chatStreamingStates = new SvelteMap(); // convs that the backend reports as having a running session, populated by the global sync // at app mount and on visibilitychange. it does not overlap with chatLoadingStates which // tracks inferences driven by this browser, both are unioned to feed the sidebar spinners @@ -135,9 +136,9 @@ class ChatStore { if (convId === conversationsStore.activeConversation?.id) this.isReasoning = false; } } - private setChatStreaming(convId: string, response: string, messageId: string): void { + private setChatStreaming(convId: string, response: string, messageId: string, model?: string | null): void { this.touchConversationState(convId); - this.chatStreamingStates.set(convId, { response, messageId }); + this.chatStreamingStates.set(convId, { response, messageId, model: model ?? this.chatStreamingStates.get(convId)?.model }); if (convId === conversationsStore.activeConversation?.id) this.currentResponse = response; } private clearChatStreaming(convId: string): void { @@ -184,7 +185,9 @@ class ChatStore { if (!convId) return null; let listResp: Response; try { - listResp = await fetch(`./v1/streams?conversation_id=${encodeURIComponent(convId)}`); + listResp = await fetch(`./v1/streams?conversation_id=${encodeURIComponent(convId)}`, { + headers: getAuthHeaders() + }); } catch (e) { console.warn('probeServerStream fetch failed:', e); return null; @@ -235,7 +238,9 @@ class ChatStore { const id = streamId || streamIdentity(convId, selectedModelName()); let response: Response; try { - response = await fetch(`./v1/stream/${encodeURIComponent(id)}?from=0`); + response = await fetch(`./v1/stream/${encodeURIComponent(id)}?from=0`, { + headers: getAuthHeaders() + }); } catch (e) { console.error('attachServerStream replay fetch failed:', e); unlock(); @@ -329,7 +334,12 @@ class ChatStore { writeActive({ content: '', reasoningContent: undefined }); } - this.setChatStreaming(convId, existingContent, targetMessageId); + // extract the model suffix from the server side identity, the resume calls inside + // handleStreamResponse must reuse the model the session was originally tagged with, + // not the current dropdown selection which may have changed since + const sepIdx = id.indexOf('::'); + const attachedModel: string | null = sepIdx === -1 ? null : id.slice(sepIdx + 2); + this.setChatStreaming(convId, existingContent, targetMessageId, attachedModel); const abortController = this.getOrCreateAbortController(convId); let streamedContent = ''; @@ -394,7 +404,8 @@ class ChatStore { if (convId === conversationsStore.activeConversation?.id) { this.streamConnectionState = connState; } - } + }, + attachedModel ); } catch (e) { console.error('attachServerStream pipe crashed:', e); @@ -601,7 +612,7 @@ class ChatStore { async syncRemoteRunningStreams(): Promise { let sessions: ApiStreamSession[]; try { - const resp = await fetch('./v1/streams'); + const resp = await fetch('./v1/streams', { headers: getAuthHeaders() }); if (!resp.ok) return; const body = (await resp.json()) as unknown; if (!Array.isArray(body)) return; @@ -1344,9 +1355,12 @@ class ChatStore { await this.savePartialResponseIfNeeded(convId); this.setStreamingActive(false); // tell the server to stop the generation, not just to drop the HTTP socket. without this - // the detached drain keeps producing tokens until eos or max_tokens. the conv id is the - // session identity so the DELETE call is straight - void ChatService.cancelServerStream(convId, selectedModelName()); + // the detached drain keeps producing tokens until eos or max_tokens. use the model captured + // when the session started rather than the current dropdown, the dropdown may have changed + // since and the server side identity (conv id plus ::model suffix) is frozen at POST time + const streamStateForStop = this.chatStreamingStates.get(convId); + const modelForStop = streamStateForStop?.model ?? selectedModelName(); + void ChatService.cancelServerStream(convId, modelForStop); this.abortRequest(convId); this.setChatLoading(convId, false); this.clearChatStreaming(convId); From 13eaa7779a0dcdb47de6ebfd4fa5650a3171371d Mon Sep 17 00:00:00 2001 From: Pascal Date: Mon, 18 May 2026 22:41:47 +0200 Subject: [PATCH 11/39] format --- tools/ui/src/lib/services/chat.service.ts | 8 +++----- tools/ui/src/lib/stores/chat.svelte.ts | 18 +++++++++++++++--- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/tools/ui/src/lib/services/chat.service.ts b/tools/ui/src/lib/services/chat.service.ts index 64e5ff768210..8fdc3706aaa1 100644 --- a/tools/ui/src/lib/services/chat.service.ts +++ b/tools/ui/src/lib/services/chat.service.ts @@ -831,11 +831,9 @@ export class ChatService { // it will be retransmitted from a clean line boundary. reuse the model the POST was // originally tagged with, the dropdown may have changed since but the server side // identity is frozen at POST time - const resumeResp = await resumeStream( - conversationId, - abortSignal, - streamModel - ).catch(() => null); + const resumeResp = await resumeStream(conversationId, abortSignal, streamModel).catch( + () => null + ); if (!resumeResp || resumeResp.status !== 200) { onConnectionState?.('lost'); onError?.(new Error('Stream connection lost and could not be resumed')); diff --git a/tools/ui/src/lib/stores/chat.svelte.ts b/tools/ui/src/lib/stores/chat.svelte.ts index df79bbddd62a..88d43ddc3c4d 100644 --- a/tools/ui/src/lib/stores/chat.svelte.ts +++ b/tools/ui/src/lib/stores/chat.svelte.ts @@ -76,7 +76,10 @@ class ChatStore { streamConnectionState = $state('streaming'); chatLoadingStates = new SvelteMap(); chatReasoningStates = new SvelteMap(); - chatStreamingStates = new SvelteMap(); + chatStreamingStates = new SvelteMap< + string, + { response: string; messageId: string; model?: string | null } + >(); // convs that the backend reports as having a running session, populated by the global sync // at app mount and on visibilitychange. it does not overlap with chatLoadingStates which // tracks inferences driven by this browser, both are unioned to feed the sidebar spinners @@ -136,9 +139,18 @@ class ChatStore { if (convId === conversationsStore.activeConversation?.id) this.isReasoning = false; } } - private setChatStreaming(convId: string, response: string, messageId: string, model?: string | null): void { + private setChatStreaming( + convId: string, + response: string, + messageId: string, + model?: string | null + ): void { this.touchConversationState(convId); - this.chatStreamingStates.set(convId, { response, messageId, model: model ?? this.chatStreamingStates.get(convId)?.model }); + this.chatStreamingStates.set(convId, { + response, + messageId, + model: model ?? this.chatStreamingStates.get(convId)?.model + }); if (convId === conversationsStore.activeConversation?.id) this.currentResponse = response; } private clearChatStreaming(convId: string): void { From d8e330028a2a63fcb82585574f0b6d6375a70c08 Mon Sep 17 00:00:00 2001 From: Pascal Date: Mon, 18 May 2026 22:44:04 +0200 Subject: [PATCH 12/39] ui: remove unused selectedModelName --- tools/ui/src/lib/services/chat.service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ui/src/lib/services/chat.service.ts b/tools/ui/src/lib/services/chat.service.ts index 8fdc3706aaa1..5e865a212d0b 100644 --- a/tools/ui/src/lib/services/chat.service.ts +++ b/tools/ui/src/lib/services/chat.service.ts @@ -40,7 +40,7 @@ import type { DatabaseMessageExtraMcpResource, StreamConnectionState } from '$lib/types'; -import { modelsStore, selectedModelName } from '$lib/stores/models.svelte'; +import { modelsStore } from '$lib/stores/models.svelte'; import { settingsStore } from '../stores/settings.svelte'; import { capImageDataURLSize } from '../utils/cap-img-size'; From 316c1d30e7bd24e8e6c2331602ff6065f875587e Mon Sep 17 00:00:00 2001 From: Pascal Date: Mon, 18 May 2026 22:53:06 +0200 Subject: [PATCH 13/39] server-stream: poll session->is_cancelled() in stream_aware_should_stop Address review feedback from coder543. The cancel propagation through rd.stop() relies on the slot eventually processing the cancel task and posting a result that notifies the recv condvar, remove_waiting_task_ids does not notify directly. Add a defensive poll on session->is_cancelled() so the producer-side next() loop exits on its next iteration after cancel() without waiting for the cancel task to round trip through a slot. --- tools/server/server-stream.cpp | 55 +++++++++++++++++++++++++++++++--- tools/server/server-stream.h | 2 ++ 2 files changed, 53 insertions(+), 4 deletions(-) diff --git a/tools/server/server-stream.cpp b/tools/server/server-stream.cpp index 48ef5349c678..2dc90dc1efb2 100644 --- a/tools/server/server-stream.cpp +++ b/tools/server/server-stream.cpp @@ -27,6 +27,7 @@ stream_session::stream_session(std::string conversation_id_, size_t max_bytes_) , prefix_dropped(0) , cap_bytes(max_bytes_) , done(false) + , cancelled(false) , completed_ts(0) { buffer.reserve(64 * 1024); } @@ -127,6 +128,10 @@ void stream_session::set_stop_producer(std::function fn) { } void stream_session::cancel() { + // flip cancelled first so the producer-side stream_aware_should_stop can break out of the + // recv() wait even if remove_waiting_task_ids does not notify the condvar (the cancel task + // posted by rd.stop() will eventually notify, but we do not want to depend on that timing) + cancelled.store(true, std::memory_order_release); // copy the hook under the lock then invoke outside, the producer side may grab queue locks // and we do not want to hold our mu across that path std::function fn; @@ -139,6 +144,10 @@ void stream_session::cancel() { } } +bool stream_session::is_cancelled() const { + return cancelled.load(std::memory_order_acquire); +} + stream_session_manager::stream_session_manager() : running(false) , drain_shutdown(std::make_shared>(false)) { @@ -408,6 +417,35 @@ server_http_context::handler_t make_stream_delete_handler() { }; } +// per-response registry that lets stream_aware_should_stop look up the session attached to a +// given http response, so it can react to is_cancelled() without depending on the upstream +// cancel propagation (rd.stop() posts cancel tasks but does not notify the recv() condvar, +// the recv() unblock only happens when the slot processes the cancel and posts a result, which +// can take a while under load). entries are added by stream_session_attach_hooks and removed +// by on_stream_end or by the destructor of the response, but since we only key by raw pointer +// during the response lifetime that is enough +static std::mutex g_res_session_mu; +static std::unordered_map> g_res_session_map; + +static void register_res_session(server_http_res * res, const stream_session_ptr & s) { + std::lock_guard lock(g_res_session_mu); + g_res_session_map[res] = s; +} + +static void unregister_res_session(server_http_res * res) { + std::lock_guard lock(g_res_session_mu); + g_res_session_map.erase(res); +} + +static stream_session_ptr lookup_res_session(server_http_res * res) { + std::lock_guard lock(g_res_session_mu); + auto it = g_res_session_map.find(res); + if (it == g_res_session_map.end()) { + return nullptr; + } + return it->second.lock(); +} + void stream_session_attach_hooks(server_http_res & res, server_response_reader & rd, const std::map & headers) { // case insensitive scan for x-conversation-id. headers preserve the wire casing, an ASCII // tolower comparison is enough here, no locale machinery needed @@ -431,6 +469,8 @@ void stream_session_attach_hooks(server_http_res & res, server_response_reader & return; } auto session = g_stream_sessions.create_or_replace(conversation_id); + // register the res to session mapping so stream_aware_should_stop can poll is_cancelled + register_res_session(&res, session); // stop_producer may outlive the response (eviction by a later POST on the same conv_id, // late DELETE). guard with a shared alive flag, flipped by on_stream_end before rd dies auto alive = std::make_shared>(true); @@ -443,12 +483,14 @@ void stream_session_attach_hooks(server_http_res & res, server_response_reader & res.tee = [session](const char * d, size_t n) { session->append(d, n); }; - res.on_stream_end = [session, alive] { + auto * res_ptr = &res; + res.on_stream_end = [session, alive, res_ptr] { // flip alive first so any concurrent cancel (eg. from a create_or_replace racing with // our natural end) skips the now invalid rd. then detach the stop hook and finalize alive->store(false, std::memory_order_release); session->set_stop_producer(nullptr); session->finalize(); + unregister_res_session(res_ptr); }; } @@ -457,9 +499,14 @@ std::function stream_aware_should_stop(server_http_res * res, std::funct // out of scope, the std::function copy keeps the underlying target alive return [res, fallback = std::move(fallback)]() -> bool { if (res->tee) { - // a tee is attached, the session must outlive the http socket. ignore the peer - // disconnect signal, only an explicit DELETE through the session stop_producer - // hook is allowed to abort the producer + // tee attached: the session is the owner now, the peer disconnect signal is ignored. + // an explicit cancel (DELETE or session eviction) flips session->is_cancelled, which + // we poll here so the producer side breaks out of recv() without waiting for the cancel + // task to be processed by a slot. without a tee the legacy req.should_stop is used + auto session = lookup_res_session(res); + if (session && session->is_cancelled()) { + return true; + } return false; } return fallback(); diff --git a/tools/server/server-stream.h b/tools/server/server-stream.h index d21297922a08..643722d51918 100644 --- a/tools/server/server-stream.h +++ b/tools/server/server-stream.h @@ -48,6 +48,7 @@ struct stream_session { const std::function & should_stop); bool is_done() const; + bool is_cancelled() const; // true when cancel() has been invoked size_t total_size() const; // bytes that ever entered the session size_t dropped_prefix() const; // bytes evicted from the front due to cap int64_t completed_at() const; // 0 while alive, unix seconds after finalize @@ -68,6 +69,7 @@ struct stream_session { size_t prefix_dropped; size_t cap_bytes; std::atomic done; + std::atomic cancelled; std::atomic completed_ts; std::function stop_producer; // protected by mu }; From 448254325f84c55ae32a9c536fd4703266afc652 Mon Sep 17 00:00:00 2001 From: Pascal Date: Tue, 19 May 2026 03:25:05 +0200 Subject: [PATCH 14/39] server-stream, ui: replace GET /v1/streams with POST /v1/streams/lookup Address review feedback from coder543. Listing live sessions leaks the conversation_id of every concurrent user, which defeats the random UUID unguessability. The new route takes {conversation_ids: [...]} in the body and returns matches only for the ids the caller already owns, so foreign UUIDs stay private. The router fans out the same POST to every child and aggregates, the WebUI passes the convs visible in its sidebar. --- tools/server/server-models.cpp | 23 +++++------ tools/server/server-models.h | 2 +- tools/server/server-stream.cpp | 53 ++++++++++++++++++-------- tools/server/server-stream.h | 2 +- tools/server/server.cpp | 19 +++++---- tools/ui/src/lib/stores/chat.svelte.ts | 25 ++++++++++-- tools/ui/src/lib/types/api.d.ts | 9 +++-- 7 files changed, 86 insertions(+), 47 deletions(-) diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index 831aebe78bbf..9b038aa4c965 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -1624,7 +1624,10 @@ static std::optional probe_child_for_conv( if (conversation_id.empty()) { return std::nullopt; } - std::string child_path = "/v1/streams?conversation_id=" + encode_qs(conversation_id); + // POST /v1/streams/lookup with the one conv id we are probing. the child only returns a + // match if it owns that conv, listing is never exposed + json body = {{"conversation_ids", json::array({conversation_id})}}; + std::string body_str = body.dump(); for (auto & meta : models.get_all_meta()) { if (!meta.is_ready()) { continue; @@ -1633,7 +1636,7 @@ static std::optional probe_child_for_conv( cli.set_connection_timeout(0, 250 * 1000); cli.set_read_timeout(0, 250 * 1000); cli.set_write_timeout(0, 250 * 1000); - auto resp = cli.Get(child_path.c_str()); + auto resp = cli.Post("/v1/streams/lookup", body_str, "application/json"); if (!resp || resp->status != 200) { continue; } @@ -1945,17 +1948,11 @@ void server_models_routes::init_routes() { return std::unique_ptr(std::move(proxy)); }; - this->router_streams_list = [this](const server_http_req & req) { - // GET /v1/streams aggregates sessions from every ready child. the WebUI mounts and - // visibilitychanges use this to drive the sidebar spinners across convs. when a - // conversation_id filter is set we still fan out because the matching session can live - // on any child, the filter just narrows the response set + this->router_streams_lookup = [this](const server_http_req & req) { + // POST /v1/streams/lookup forwards the same body to every ready child and aggregates + // the results. the child responds only for the conv ids we asked about, never lists + // anything else, so the router never exposes ids the caller did not already know auto res = std::make_unique(); - std::string conversation_id = req.get_param("conversation_id"); - std::string child_path = "/v1/streams"; - if (!conversation_id.empty()) { - child_path += "?conversation_id=" + encode_qs(conversation_id); - } json aggregated = json::array(); for (auto & meta : models.get_all_meta()) { if (!meta.is_ready()) { @@ -1965,7 +1962,7 @@ void server_models_routes::init_routes() { cli.set_connection_timeout(0, 250 * 1000); cli.set_read_timeout(0, 250 * 1000); cli.set_write_timeout(0, 250 * 1000); - auto resp = cli.Get(child_path.c_str()); + auto resp = cli.Post("/v1/streams/lookup", req.body, "application/json"); if (!resp || resp->status != 200) { continue; } diff --git a/tools/server/server-models.h b/tools/server/server-models.h index 53029f5c17fc..af9c54793a82 100644 --- a/tools/server/server-models.h +++ b/tools/server/server-models.h @@ -274,7 +274,7 @@ struct server_models_routes { // the suffix is absent the get/delete paths fall back to a loopback probe and a fan out // respectively, the list path always fans out and aggregates server_http_context::handler_t router_stream_get; - server_http_context::handler_t router_streams_list; + server_http_context::handler_t router_streams_lookup; server_http_context::handler_t router_stream_delete; }; diff --git a/tools/server/server-stream.cpp b/tools/server/server-stream.cpp index 2dc90dc1efb2..7b5cad4140ec 100644 --- a/tools/server/server-stream.cpp +++ b/tools/server/server-stream.cpp @@ -359,28 +359,49 @@ server_http_context::handler_t make_stream_get_handler() { }; } -server_http_context::handler_t make_streams_list_handler() { +server_http_context::handler_t make_streams_lookup_handler() { return [](const server_http_req & req) -> server_http_res_ptr { - // GET /v1/streams returns sessions as a JSON array. with conversation_id set, every - // session whose key is exactly that id or starts with "::" matches, so a single - // call returns every per model variant for a given conv. without conversation_id, - // every live or recently completed session known to this server, used by the WebUI - // at mount and on visibilitychange to populate the sidebar spinners - std::string conversation_id = req.get_param("conversation_id"); + // POST /v1/streams/lookup with body {"conversation_ids": ["X", "Y", ...]} returns the + // matching sessions. you can only ask for ids you already know, the server never lists + // sessions it has not been asked about. for each requested id we match the exact key + // and any "::" variant, so a single lookup covers every per model session + // for that conv. used by the WebUI sidebar at mount and on visibilitychange + std::vector requested; + try { + json body = json::parse(req.body); + if (body.contains("conversation_ids") && body["conversation_ids"].is_array()) { + for (const auto & v : body["conversation_ids"]) { + if (v.is_string()) { + std::string id = v.get(); + if (!id.empty()) { + requested.push_back(std::move(id)); + } + } + } + } + } catch (const std::exception & e) { + auto res = std::make_unique(); + res->status = 400; + res->content_type = "application/json; charset=utf-8"; + res->data = safe_json_to_str({{"error", {{"message", std::string("invalid body: ") + e.what()}, + {"type", "invalid_request_error"}}}}); + return res; + } + std::vector sessions; - if (conversation_id.empty()) { - sessions = g_stream_sessions.list_all(); - } else { - const std::string with_sep = conversation_id + "::"; + if (!requested.empty()) { auto all = g_stream_sessions.list_all(); - for (auto & s : all) { - if (s->conversation_id == conversation_id) { - sessions.push_back(s); - } else if (s->conversation_id.compare(0, with_sep.size(), with_sep) == 0) { - sessions.push_back(s); + for (const auto & rid : requested) { + const std::string with_sep = rid + "::"; + for (auto & s : all) { + if (s->conversation_id == rid || + s->conversation_id.compare(0, with_sep.size(), with_sep) == 0) { + sessions.push_back(s); + } } } } + json arr = json::array(); for (auto & s : sessions) { arr.push_back({ diff --git a/tools/server/server-stream.h b/tools/server/server-stream.h index 643722d51918..78225ff4c59c 100644 --- a/tools/server/server-stream.h +++ b/tools/server/server-stream.h @@ -134,7 +134,7 @@ extern stream_session_manager g_stream_sessions; // through server-context's server_routes. keeps the resumable stream surface confined to // server-stream and server-http server_http_context::handler_t make_stream_get_handler(); -server_http_context::handler_t make_streams_list_handler(); +server_http_context::handler_t make_streams_lookup_handler(); server_http_context::handler_t make_stream_delete_handler(); // attach a resumable session to an outgoing streaming response. inspects the request headers diff --git a/tools/server/server.cpp b/tools/server/server.cpp index e97429e427b1..67bb0eff75f2 100644 --- a/tools/server/server.cpp +++ b/tools/server/server.cpp @@ -248,19 +248,22 @@ int llama_server(int argc, char ** argv) { // backed factories, the router binds proxies that route via the optional ::model suffix // (direct) or fall back to loopback probe and fan out (suffixless conv ids) server_http_context::handler_t stream_get_h; - server_http_context::handler_t streams_list_h; + server_http_context::handler_t streams_lookup_h; server_http_context::handler_t stream_delete_h; if (is_router_server) { - stream_get_h = models_routes->router_stream_get; - streams_list_h = models_routes->router_streams_list; - stream_delete_h = models_routes->router_stream_delete; + stream_get_h = models_routes->router_stream_get; + streams_lookup_h = models_routes->router_streams_lookup; + stream_delete_h = models_routes->router_stream_delete; } else { - stream_get_h = make_stream_get_handler(); - streams_list_h = make_streams_list_handler(); - stream_delete_h = make_stream_delete_handler(); + stream_get_h = make_stream_get_handler(); + streams_lookup_h = make_streams_lookup_handler(); + stream_delete_h = make_stream_delete_handler(); } ctx_http.get ("/v1/stream/:conv_id", ex_wrapper(stream_get_h)); - ctx_http.get ("/v1/streams", ex_wrapper(streams_list_h)); + // POST /v1/streams/lookup with body {"conversation_ids": [...]}. you can only ask for ids + // you already own (the WebUI passes the convs visible in its sidebar). the server never + // lists ids it has not been asked about, so a random caller cannot enumerate live sessions + ctx_http.post("/v1/streams/lookup", ex_wrapper(streams_lookup_h)); ctx_http.del_("/v1/stream/:conv_id", ex_wrapper(stream_delete_h)); // Google Cloud Platform (Vertex AI) compat diff --git a/tools/ui/src/lib/stores/chat.svelte.ts b/tools/ui/src/lib/stores/chat.svelte.ts index 88d43ddc3c4d..30b482e0be96 100644 --- a/tools/ui/src/lib/stores/chat.svelte.ts +++ b/tools/ui/src/lib/stores/chat.svelte.ts @@ -178,7 +178,7 @@ class ChatStore { /** * Server side stream discovery, split in three pieces: * - * probeServerStream(convId) -> hits GET /v1/streams?conversation_id, returns the session to attach + * probeServerStream(convId) -> hits POST /v1/streams/lookup with the conv id, returns the session to attach * to or null. Pure read, no side effect, no UI lock. Safe to fire in parallel with anything. * * attachServerStream(convId) -> flips the spinner immediately, fetches the replay stream @@ -197,8 +197,12 @@ class ChatStore { if (!convId) return null; let listResp: Response; try { - listResp = await fetch(`./v1/streams?conversation_id=${encodeURIComponent(convId)}`, { - headers: getAuthHeaders() + // POST the one conv id we are probing, the server only returns a match if it owns it, + // never lists ids the caller did not already provide + listResp = await fetch(`./v1/streams/lookup`, { + method: 'POST', + headers: { ...getAuthHeaders(), 'Content-Type': 'application/json' }, + body: JSON.stringify({ conversation_ids: [convId] }) }); } catch (e) { console.warn('probeServerStream fetch failed:', e); @@ -622,9 +626,22 @@ class ChatStore { * for sessions that finalized while the browser was elsewhere are dropped naturally. */ async syncRemoteRunningStreams(): Promise { + // only ask about conv ids the user already owns (the sidebar list). the server never lists + // ids the caller did not provide, so a random foreign UUID stays unguessable + const ids = conversationsStore.conversations.map((c) => c.id).filter((id) => !!id); + if (ids.length === 0) { + for (const id of Array.from(this.remoteRunningConvs)) { + this.remoteRunningConvs.delete(id); + } + return; + } let sessions: ApiStreamSession[]; try { - const resp = await fetch('./v1/streams', { headers: getAuthHeaders() }); + const resp = await fetch('./v1/streams/lookup', { + method: 'POST', + headers: { ...getAuthHeaders(), 'Content-Type': 'application/json' }, + body: JSON.stringify({ conversation_ids: ids }) + }); if (!resp.ok) return; const body = (await resp.json()) as unknown; if (!Array.isArray(body)) return; diff --git a/tools/ui/src/lib/types/api.d.ts b/tools/ui/src/lib/types/api.d.ts index 68fde6e60392..ec695ac61cc5 100644 --- a/tools/ui/src/lib/types/api.d.ts +++ b/tools/ui/src/lib/types/api.d.ts @@ -514,10 +514,11 @@ export interface ApiRouterModelsUnloadResponse { } /** - * Entry returned by GET /v1/streams (optional conversation_id query filter). One entry per - * live or recently completed background streaming session, keyed by its conversation_id. - * The WebUI uses this at mount and on visibilitychange to populate sidebar spinners and to - * reattach to an ongoing inference for the active conversation. + * Entry returned by POST /v1/streams/lookup. The client passes the conv ids it owns in the body + * and the server returns one entry per matching live or recently completed background streaming + * session, keyed by conversation_id. The WebUI uses this at mount and on visibilitychange to + * populate sidebar spinners and to reattach to an ongoing inference for the active conversation. + * The server never lists ids the client did not ask about, so foreign random UUIDs stay private. */ export interface ApiStreamSession { conversation_id: string; From fd5c81a9d2eff1227119110cdc6114c1877d6c04 Mon Sep 17 00:00:00 2001 From: Pascal Date: Tue, 19 May 2026 03:39:20 +0200 Subject: [PATCH 15/39] ui: read conv ids from IndexedDB in syncRemoteRunningStreams The conversations store is not hydrated yet at +layout onMount, so the sidebar spinners stayed off for background convs until the user clicked on them. Read straight from the DB to dodge the init race. --- tools/ui/src/lib/stores/chat.svelte.ts | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/tools/ui/src/lib/stores/chat.svelte.ts b/tools/ui/src/lib/stores/chat.svelte.ts index 30b482e0be96..62094f7ab123 100644 --- a/tools/ui/src/lib/stores/chat.svelte.ts +++ b/tools/ui/src/lib/stores/chat.svelte.ts @@ -626,9 +626,20 @@ class ChatStore { * for sessions that finalized while the browser was elsewhere are dropped naturally. */ async syncRemoteRunningStreams(): Promise { - // only ask about conv ids the user already owns (the sidebar list). the server never lists - // ids the caller did not provide, so a random foreign UUID stays unguessable - const ids = conversationsStore.conversations.map((c) => c.id).filter((id) => !!id); + // the conversations store loads from IndexedDB asynchronously, the +layout onMount caller + // fires before that finishes. read ids straight from the DB so the result does not depend + // on the store init race, and the sidebar spinners light up at first paint for every conv + // the user owns even if it has not been hydrated into the store yet + let ids: string[]; + try { + const all = await DatabaseService.getAllConversations(); + ids = all.map((c) => c.id).filter((id) => !!id); + } catch (e) { + console.warn('syncRemoteRunningStreams DB read failed:', e); + return; + } + // only ask about conv ids the user already owns. the server never lists ids the caller did + // not provide, so a random foreign UUID stays unguessable if (ids.length === 0) { for (const id of Array.from(this.remoteRunningConvs)) { this.remoteRunningConvs.delete(id); From 54be28f3085efea9c180934c63b99689d758d8c2 Mon Sep 17 00:00:00 2001 From: Pascal Date: Tue, 19 May 2026 04:10:24 +0200 Subject: [PATCH 16/39] server-models: deduplicate stream lookup timeouts behind one constant --- tools/server/server-models.cpp | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index 9b038aa4c965..7c69128e9083 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -93,6 +93,9 @@ struct server_subproc { } }; +// short loopback budget for the resumable stream router to child JSON calls (probe, lookup, +// delete). distinct from params.timeout_read/write which only applies to the generation proxy +static constexpr int STREAM_LOOKUP_TIMEOUT_MS = 250; static std::filesystem::path get_server_exec_path() { #if defined(_WIN32) @@ -1633,9 +1636,9 @@ static std::optional probe_child_for_conv( continue; } httplib::Client cli(CHILD_ADDR, meta.port); - cli.set_connection_timeout(0, 250 * 1000); - cli.set_read_timeout(0, 250 * 1000); - cli.set_write_timeout(0, 250 * 1000); + cli.set_connection_timeout(0, STREAM_LOOKUP_TIMEOUT_MS * 1000); + cli.set_read_timeout(0, STREAM_LOOKUP_TIMEOUT_MS * 1000); + cli.set_write_timeout(0, STREAM_LOOKUP_TIMEOUT_MS * 1000); auto resp = cli.Post("/v1/streams/lookup", body_str, "application/json"); if (!resp || resp->status != 200) { continue; @@ -1959,9 +1962,9 @@ void server_models_routes::init_routes() { continue; } httplib::Client cli(CHILD_ADDR, meta.port); - cli.set_connection_timeout(0, 250 * 1000); - cli.set_read_timeout(0, 250 * 1000); - cli.set_write_timeout(0, 250 * 1000); + cli.set_connection_timeout(0, STREAM_LOOKUP_TIMEOUT_MS * 1000); + cli.set_read_timeout(0, STREAM_LOOKUP_TIMEOUT_MS * 1000); + cli.set_write_timeout(0, STREAM_LOOKUP_TIMEOUT_MS * 1000); auto resp = cli.Post("/v1/streams/lookup", req.body, "application/json"); if (!resp || resp->status != 200) { continue; @@ -1998,9 +2001,9 @@ void server_models_routes::init_routes() { std::string model_hint = extract_model_from_conv(conv_id); auto delete_on = [&](int port) { httplib::Client cli(CHILD_ADDR, port); - cli.set_connection_timeout(0, 250 * 1000); - cli.set_read_timeout(0, 500 * 1000); - cli.set_write_timeout(0, 250 * 1000); + cli.set_connection_timeout(0, STREAM_LOOKUP_TIMEOUT_MS * 1000); + cli.set_read_timeout(0, STREAM_LOOKUP_TIMEOUT_MS * 1000); + cli.set_write_timeout(0, STREAM_LOOKUP_TIMEOUT_MS * 1000); auto resp = cli.Delete(child_path.c_str()); (void) resp; // best effort, 404 and network errors are equivalent to no op }; From a61cf9611426e08e417b5fb64f0c19263583ff2d Mon Sep 17 00:00:00 2001 From: Pascal Date: Tue, 19 May 2026 04:22:46 +0200 Subject: [PATCH 17/39] ui: extract visibility kick grace into a stream constant, bump to 1000 ms --- tools/ui/src/lib/constants/index.ts | 1 + tools/ui/src/lib/constants/stream.ts | 3 +++ tools/ui/src/lib/services/chat.service.ts | 5 +++-- 3 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 tools/ui/src/lib/constants/stream.ts diff --git a/tools/ui/src/lib/constants/index.ts b/tools/ui/src/lib/constants/index.ts index 4993ab647ad2..b982a5907275 100644 --- a/tools/ui/src/lib/constants/index.ts +++ b/tools/ui/src/lib/constants/index.ts @@ -46,6 +46,7 @@ export * from './routes'; export * from './sandbox'; export * from './settings-keys'; export * from './settings-registry'; +export * from './stream'; export * from './supported-file-types'; export * from './table-html-restorer'; export * from './title-generation'; diff --git a/tools/ui/src/lib/constants/stream.ts b/tools/ui/src/lib/constants/stream.ts new file mode 100644 index 000000000000..3d042451fc69 --- /dev/null +++ b/tools/ui/src/lib/constants/stream.ts @@ -0,0 +1,3 @@ +// grace window after a visibilitychange before we kick a reader whose socket likely died +// while the tab was hidden. covers brief background pauses without thrashing live streams +export const STREAM_VISIBILITY_KICK_MS = 1000; diff --git a/tools/ui/src/lib/services/chat.service.ts b/tools/ui/src/lib/services/chat.service.ts index 5e865a212d0b..369e0d3988b8 100644 --- a/tools/ui/src/lib/services/chat.service.ts +++ b/tools/ui/src/lib/services/chat.service.ts @@ -19,7 +19,8 @@ import { CONTROL_ACTION, SSE_LINE_SEPARATOR, SSE_DATA_PREFIX, - SSE_DONE_MARKER + SSE_DONE_MARKER, + STREAM_VISIBILITY_KICK_MS } from '$lib/constants'; import { AttachmentType, @@ -689,7 +690,7 @@ export class ChatService { if (!conversationId) return; // the bytes have been quiet for too long, the OS likely killed the socket // kicking the reader unblocks reader.read with done=true so the outer loop can resume - if (Date.now() - lastByteAt > 300) { + if (Date.now() - lastByteAt > STREAM_VISIBILITY_KICK_MS) { reader!.cancel().catch(() => {}); } }; From 81c394f8585ce0a8ed6df2e8d61c97aed9fadd6f Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Wed, 20 May 2026 00:28:32 +0200 Subject: [PATCH 18/39] make it safer & more simple --- tools/server/server-context.cpp | 19 +++-- tools/server/server-http.cpp | 64 ++++----------- tools/server/server-http.h | 22 ++--- tools/server/server-stream.cpp | 138 ++++++++++++++++---------------- tools/server/server-stream.h | 70 ++++++++++++---- 5 files changed, 160 insertions(+), 153 deletions(-) diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 2309c3455502..5c33a418f549 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -4023,6 +4023,15 @@ struct server_res_generator : server_http_res { queue_tasks.wait_until_no_sleep(); } } + ~server_res_generator() override { + // cleanup() must run while rd is still alive (rd is destroyed after this body returns) + if (spipe) { + spipe->cleanup(); + } + } + void stop() override { + rd.stop(); + } void ok(const json & response_data) { status = 200; data = safe_json_to_str(response_data); @@ -4211,8 +4220,6 @@ std::unique_ptr server_routes::handle_completions_impl( } }; - // delegate to server-stream so the tee-aware rule lives there, not here. without a - // tee attached this is bit identical to req.should_stop, the legacy flow is unchanged auto effective_should_stop = stream_aware_should_stop(res_this, req.should_stop); try { @@ -4307,11 +4314,9 @@ std::unique_ptr server_routes::handle_completions_impl( }; } - // delegate the X-Conversation-Id sniff and the three hook wirings (tee, on_stream_end, - // stop_producer) to server-stream. when the header is absent this is a no op, when set - // the session is created or replaced and the response carries the tee that mirrors every - // SSE chunk into the ring buffer, plus the cancel hook for DELETE /v1/stream/ - stream_session_attach_hooks(*res, res->rd, req.headers); + // attach a producer pipe to the response when X-Conversation-Id is present. + // the pipe mirrors SSE chunks into the ring buffer and wires up the cancel hook. + stream_session_attach_pipe(*res, req.headers); return res; } diff --git a/tools/server/server-http.cpp b/tools/server/server-http.cpp index d13ec78a1ebb..d39138484293 100644 --- a/tools/server/server-http.cpp +++ b/tools/server/server-http.cpp @@ -1,5 +1,6 @@ #include "common.h" #include "server-http.h" +#include "server-stream.h" #include "server-common.h" #include "ui.h" @@ -497,71 +498,34 @@ static void process_handler_response(server_http_req_ptr && request, server_http set_headers(res, response->headers); const std::string content_type = response->content_type; // convert to shared_ptr as both chunked_content_provider() and on_complete() need to use it - std::shared_ptr q_ptr = std::move(request); - std::shared_ptr r_ptr = std::move(response); - // shared flag, flipped to true the moment the producer signals next() == false on the - // normal wire path. on_complete uses it to decide whether to spawn the detached drain. - // covers every disconnect timing httplib can observe: between two chunks (peer dead - // detected before content_provider is called), during a chunk (sink.write fails), or - // never (the producer drained cleanly). without this httplib bails the provider when - // the peer is gone, on_complete fires, the shared_ptr resets, the underlying reader - // is destroyed, and the backend stops mid generation - auto stream_done = std::make_shared>(false); - - const auto chunked_content_provider = [response = r_ptr, stream_done](size_t, const httplib::DataSink & sink) -> bool { + std::shared_ptr q_ptr = std::move(request); + std::shared_ptr r_ptr = std::move(response); + + const auto chunked_content_provider = [response = r_ptr](size_t, httplib::DataSink & sink) -> bool { std::string chunk; const bool has_next = response->next(chunk); if (!chunk.empty()) { - // mirror to the tee first, the session must reflect the SSE stream regardless - // of whether the wire write succeeds for this chunk - if (response->tee) { - response->tee(chunk.data(), chunk.size()); + if (response->spipe) { + response->spipe->write(chunk.data(), chunk.size()); } if (!sink.write(chunk.data(), chunk.size())) { - // peer is gone mid chunk, on_complete will pick up the detached drain + // peer gone; if a pipe is attached keep producing into it, otherwise stop + if (response->spipe) { + return true; + } return false; } SRV_DBG("http: streamed chunk: %s\n", chunk.c_str()); } if (!has_next) { - stream_done->store(true, std::memory_order_release); - if (response->on_stream_end) { - response->on_stream_end(); - } sink.done(); SRV_DBG("%s", "http: stream ended\n"); } return has_next; }; - const auto on_complete = [request = q_ptr, response = r_ptr, stream_done](bool) mutable { - // if the producer is still running when httplib hands the response back, the peer - // is gone but the generation must keep going. detach a thread that pumps next() - // into the tee until done, then runs on_stream_end. capture by value keeps the - // response alive past the reset below, the request too since next() may reach - // into *request via &req - if (!stream_done->load(std::memory_order_acquire) && response->tee) { - auto req_keep = request; - auto resp_keep = response; - std::thread([req_keep, resp_keep]() { - std::string c; - while (true) { - c.clear(); - bool more = resp_keep->next(c); - if (!c.empty() && resp_keep->tee) { - resp_keep->tee(c.data(), c.size()); - } - if (!more) { - break; - } - } - if (resp_keep->on_stream_end) { - resp_keep->on_stream_end(); - } - (void) req_keep; // keep the request alive until the drain is done - }).detach(); - } - response.reset(); // trigger the destruction of the response object - request.reset(); // trigger the destruction of the request object + const auto on_complete = [request = q_ptr, response = r_ptr](bool) mutable { + response.reset(); // spipe destructor finalizes the session if attached + request.reset(); }; res.set_chunked_content_provider(content_type, chunked_content_provider, on_complete); } else { diff --git a/tools/server/server-http.h b/tools/server/server-http.h index d5743962d86d..1afec5fd9ad4 100644 --- a/tools/server/server-http.h +++ b/tools/server/server-http.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -10,6 +11,7 @@ #include struct common_params; +struct stream_pipe; // defined in server-stream.h // generator-like API for HTTP response generation // this object response with one of the 2 modes: @@ -23,22 +25,20 @@ struct server_http_res { std::string data; std::map headers; - // TODO: move this to a virtual function once we have proper polymorphism support + // if set, the stream survives a client disconnect: the http layer keeps calling next() and + // feeding chunks into the pipe even when sink.write fails, until next() returns false. + // the pipe destructor finalizes the session so no explicit on_stream_end callback is needed. + // shared_ptr used (not unique_ptr) so the forward-declared type is safe to delete here. + std::shared_ptr spipe; + std::function next = nullptr; bool is_stream() const { return next != nullptr; } - // optional, each chunk produced by next() is forwarded here before being written to the - // wire. on wire failure (peer gone), the http layer detaches a background drain that keeps - // invoking next() and forwarding to the tee until next() returns false. lets streams - // survive a client disconnect, the producer keeps writing into the tee even when nobody - // is listening on the original HTTP socket - std::function tee = nullptr; - - // optional, called when the stream ends on either path (wire drained to false, or detached - // drain reached false). used by the tee owner to finalize its sink, idempotent expected - std::function on_stream_end = nullptr; + // called when the session is cancelled (e.g. DELETE /v1/stream/). + // server_res_generator overrides this to stop its reader; the default is a no-op. + virtual void stop() {} virtual ~server_http_res() = default; }; diff --git a/tools/server/server-stream.cpp b/tools/server/server-stream.cpp index 7b5cad4140ec..47e768215a73 100644 --- a/tools/server/server-stream.cpp +++ b/tools/server/server-stream.cpp @@ -299,6 +299,63 @@ void stream_session_manager::gc_loop() { // process wide manager, lifecycle controlled by llama-server main() via start_gc/stop_gc stream_session_manager g_stream_sessions; +// stream_pipe --------------------------------------------------------------------------------- + +stream_pipe::stream_pipe(stream_session_ptr session, bool is_producer) + : session_(std::move(session)) + , is_producer_(is_producer) + , res_(nullptr) { +} + +stream_pipe::~stream_pipe() { + cleanup(); + if (is_producer_) { + session_->finalize(); + } +} + +void stream_pipe::cleanup() { + if (!is_producer_ || !alive_) { + return; + } + alive_->store(false, std::memory_order_release); + session_->set_stop_producer(nullptr); + alive_.reset(); +} + +bool stream_pipe::write(const char * data, size_t len) { + return session_->append(data, len); +} + +stream_read_status stream_pipe::read(size_t & offset, + const std::function & sink, + const std::function & should_stop) { + return session_->read_from(offset, sink, should_stop); +} + +bool stream_pipe::is_cancelled() const { + return session_->is_cancelled(); +} + +std::shared_ptr stream_pipe::create_producer(stream_session_ptr session, + server_http_res & res) { + auto alive = std::make_shared>(true); + auto * res_ptr = &res; + session->set_stop_producer([alive, res_ptr]() { + if (alive->load(std::memory_order_acquire)) { + res_ptr->stop(); + } + }); + auto pipe = std::shared_ptr(new stream_pipe(std::move(session), true)); + pipe->alive_ = std::move(alive); + pipe->res_ = res_ptr; + return pipe; +} + +std::shared_ptr stream_pipe::create_consumer(stream_session_ptr session) { + return std::shared_ptr(new stream_pipe(std::move(session), false)); +} + // helper, builds the standard error response and assigns it to a brand new http_res static server_http_res_ptr make_error_response(int status, const std::string & message, error_type type) { auto res = std::make_unique(); @@ -341,11 +398,12 @@ server_http_context::handler_t make_stream_get_handler() { // the next closure reads from the ring buffer at the requested offset, blocks until // bytes arrive or the session finalizes. exit each call after draining the available // chunk so set_chunked_content_provider gets a chance to flush to the socket - auto offset_ptr = std::make_shared(from); - auto session_capture = session; - res->next = [session_capture, offset_ptr, &req](std::string & output) -> bool { + auto offset_ptr = std::make_shared(from); + // consumer pipe: read-only, does not finalize the session on destruction + auto pipe = stream_pipe::create_consumer(session); + res->next = [pipe, offset_ptr, &req](std::string & output) -> bool { bool got_any = false; - session_capture->read_from(*offset_ptr, + pipe->read(*offset_ptr, [&](const char * d, size_t n) { output.append(d, n); *offset_ptr += n; @@ -438,38 +496,8 @@ server_http_context::handler_t make_stream_delete_handler() { }; } -// per-response registry that lets stream_aware_should_stop look up the session attached to a -// given http response, so it can react to is_cancelled() without depending on the upstream -// cancel propagation (rd.stop() posts cancel tasks but does not notify the recv() condvar, -// the recv() unblock only happens when the slot processes the cancel and posts a result, which -// can take a while under load). entries are added by stream_session_attach_hooks and removed -// by on_stream_end or by the destructor of the response, but since we only key by raw pointer -// during the response lifetime that is enough -static std::mutex g_res_session_mu; -static std::unordered_map> g_res_session_map; - -static void register_res_session(server_http_res * res, const stream_session_ptr & s) { - std::lock_guard lock(g_res_session_mu); - g_res_session_map[res] = s; -} - -static void unregister_res_session(server_http_res * res) { - std::lock_guard lock(g_res_session_mu); - g_res_session_map.erase(res); -} - -static stream_session_ptr lookup_res_session(server_http_res * res) { - std::lock_guard lock(g_res_session_mu); - auto it = g_res_session_map.find(res); - if (it == g_res_session_map.end()) { - return nullptr; - } - return it->second.lock(); -} - -void stream_session_attach_hooks(server_http_res & res, server_response_reader & rd, const std::map & headers) { - // case insensitive scan for x-conversation-id. headers preserve the wire casing, an ASCII - // tolower comparison is enough here, no locale machinery needed +void stream_session_attach_pipe(server_http_res & res, const std::map & headers) { + // case-insensitive scan for x-conversation-id static constexpr char target[] = "x-conversation-id"; static constexpr size_t target_len = sizeof(target) - 1; std::string conversation_id; @@ -490,45 +518,13 @@ void stream_session_attach_hooks(server_http_res & res, server_response_reader & return; } auto session = g_stream_sessions.create_or_replace(conversation_id); - // register the res to session mapping so stream_aware_should_stop can poll is_cancelled - register_res_session(&res, session); - // stop_producer may outlive the response (eviction by a later POST on the same conv_id, - // late DELETE). guard with a shared alive flag, flipped by on_stream_end before rd dies - auto alive = std::make_shared>(true); - auto * rd_ptr = &rd; - session->set_stop_producer([alive, rd_ptr] { - if (alive->load(std::memory_order_acquire)) { - rd_ptr->stop(); - } - }); - res.tee = [session](const char * d, size_t n) { - session->append(d, n); - }; - auto * res_ptr = &res; - res.on_stream_end = [session, alive, res_ptr] { - // flip alive first so any concurrent cancel (eg. from a create_or_replace racing with - // our natural end) skips the now invalid rd. then detach the stop hook and finalize - alive->store(false, std::memory_order_release); - session->set_stop_producer(nullptr); - session->finalize(); - unregister_res_session(res_ptr); - }; + res.spipe = stream_pipe::create_producer(session, res); } std::function stream_aware_should_stop(server_http_res * res, std::function fallback) { - // capture fallback by value, the closure stays valid even after the original local goes - // out of scope, the std::function copy keeps the underlying target alive return [res, fallback = std::move(fallback)]() -> bool { - if (res->tee) { - // tee attached: the session is the owner now, the peer disconnect signal is ignored. - // an explicit cancel (DELETE or session eviction) flips session->is_cancelled, which - // we poll here so the producer side breaks out of recv() without waiting for the cancel - // task to be processed by a slot. without a tee the legacy req.should_stop is used - auto session = lookup_res_session(res); - if (session && session->is_cancelled()) { - return true; - } - return false; + if (res->spipe) { + return res->spipe->is_cancelled(); } return fallback(); }; diff --git a/tools/server/server-stream.h b/tools/server/server-stream.h index 78225ff4c59c..b74df3412689 100644 --- a/tools/server/server-stream.h +++ b/tools/server/server-stream.h @@ -76,6 +76,52 @@ struct stream_session { using stream_session_ptr = std::shared_ptr; +// RAII wrapper around a stream_session that represents one end of the pipe. +// the producer side writes chunks and is responsible for finalizing; the consumer side reads. +// +// lifetime safety: the producer pipe holds a shared_ptr> alive that is also +// captured by the session's stop_producer hook. cleanup() sets alive=false and clears the +// hook; it must be called while the owning response object is still valid (i.e. before the +// reader it would call stop() on is destroyed). ~server_res_generator() does this explicitly. +struct stream_pipe { + ~stream_pipe(); + + // producer: append raw bytes to the session's ring buffer. + // returns false if the session is already finalized. + bool write(const char * data, size_t len); + + // consumer: drain bytes from offset, calling sink for each available chunk. + // blocks until more data arrives or the session finalizes. + // should_stop is polled periodically; returns OFFSET_LOST if offset fell below the prefix. + stream_read_status read(size_t & offset, + const std::function & sink, + const std::function & should_stop); + + // true if the session was cancelled (e.g. via DELETE /v1/stream/) + bool is_cancelled() const; + + // disarm the stop hook and mark the alive guard false; must be called while the + // object that stop_fn references (the response reader) is still alive. + // idempotent; ~stream_pipe() calls it automatically but callers can do it earlier. + void cleanup(); + + // factory: producer pipe. res.stop() is invoked when the session is cancelled. + // the alive guard ensures stop() is not called after cleanup() has run. + static std::shared_ptr create_producer(stream_session_ptr session, + server_http_res & res); + + // factory: consumer pipe (read-only; destructor does not finalize the session). + static std::shared_ptr create_consumer(stream_session_ptr session); + +private: + stream_session_ptr session_; + bool is_producer_; + std::shared_ptr> alive_; // only set for producer pipes + server_http_res * res_; // only set for producer pipes + + stream_pipe(stream_session_ptr session, bool is_producer); +}; + // owns all live sessions, runs a periodic GC to evict expired ones. // the map is keyed by conversation_id, so the invariant "one conv = at most one // live session" is enforced at the type level @@ -137,18 +183,14 @@ server_http_context::handler_t make_stream_get_handler(); server_http_context::handler_t make_streams_lookup_handler(); server_http_context::handler_t make_stream_delete_handler(); -// attach a resumable session to an outgoing streaming response. inspects the request headers -// for X-Conversation-Id, and when present creates or replaces a session on the global manager, -// then wires three closures on the response: tee mirrors each SSE chunk into the ring buffer, -// on_stream_end finalizes the session, and the session's stop_producer hook calls rd.stop() -// so the explicit DELETE /v1/stream/ route can abort the underlying reader. no op -// when the header is absent. server-context just calls this and never touches the manager -struct server_response_reader; // forward declare to avoid pulling server-queue.h into the header -void stream_session_attach_hooks(server_http_res & res, server_response_reader & rd, const std::map & headers); - -// build a should_stop closure that suppresses the peer disconnect signal when a tee is -// attached to the response. used by handler lambdas that pump a server_response_reader so -// the producer keeps running past F5, only an explicit DELETE through the stop_producer -// hook is allowed to abort it. without a tee the returned closure is bit identical to the -// fallback, the legacy non resumable flow is unchanged +// inspect request headers for X-Conversation-Id and, when present, create or replace a +// session on the global manager then attach a producer pipe to res. the pipe's stop_fn +// calls res.stop() (overridden by server_res_generator to stop its reader). no-op when +// the header is absent. server-context calls this from the server_res_generator constructor. +void stream_session_attach_pipe(server_http_res & res, const std::map & headers); + +// build a should_stop closure that suppresses peer-disconnect when a pipe is attached. +// when spipe is set, only an explicit cancel (DELETE /v1/stream/) stops the +// producer; peer disconnect is ignored so generation continues into the ring buffer. +// without a pipe the closure delegates to fallback, preserving the legacy non-resumable flow. std::function stream_aware_should_stop(server_http_res * res, std::function fallback); From cc90c0819c0f0857ded4f64b5e9ae12080fda7fe Mon Sep 17 00:00:00 2001 From: Pascal Date: Wed, 20 May 2026 09:49:41 +0200 Subject: [PATCH 19/39] server-stream: survive client disconnect via stream_pipe::finish_producer After the RAII rewrite the generation stopped the moment the client disconnected. httplib bails its content provider on the is_peer_alive check at the top of write_content_chunked, so returning true from the provider never keeps it producing: the response resets, rd is destroyed and its task gets cancelled. Reinstate the disconnect survival inside the pipe. stream_pipe gains finish_producer, which pumps the response next() into the ring buffer until the generation ends, and mark_producer_done for the clean wire end. server-http only triggers them: mark before sink.done on a clean close, finish in on_complete when the peer left early. No detach, no stream logic in server-http beyond the trigger, and the strict OAI path is untouched when no pipe is attached. Known limitation: finish_producer pumps synchronously on the http worker, so a disconnected stream keeps its worker busy until the generation ends. A follow-up will move the drain off the http worker so no worker is held. --- tools/server/server-http.cpp | 18 ++++++++++++++---- tools/server/server-http.h | 7 ++++--- tools/server/server-stream.cpp | 26 ++++++++++++++++++++++++++ tools/server/server-stream.h | 11 +++++++++++ 4 files changed, 55 insertions(+), 7 deletions(-) diff --git a/tools/server/server-http.cpp b/tools/server/server-http.cpp index d39138484293..e89680f05f04 100644 --- a/tools/server/server-http.cpp +++ b/tools/server/server-http.cpp @@ -505,25 +505,35 @@ static void process_handler_response(server_http_req_ptr && request, server_http std::string chunk; const bool has_next = response->next(chunk); if (!chunk.empty()) { + // mirror into the ring buffer first, the session must reflect every SSE chunk + // whether or not the wire write below succeeds if (response->spipe) { response->spipe->write(chunk.data(), chunk.size()); } if (!sink.write(chunk.data(), chunk.size())) { - // peer gone; if a pipe is attached keep producing into it, otherwise stop - if (response->spipe) { - return true; - } + // peer is gone, stop the wire path here. when a pipe is attached on_complete + // drains the rest of the generation into the ring buffer return false; } SRV_DBG("http: streamed chunk: %s\n", chunk.c_str()); } if (!has_next) { + // producer reached its natural end on the wire, the pipe skips its drain + if (response->spipe) { + response->spipe->mark_producer_done(); + } sink.done(); SRV_DBG("%s", "http: stream ended\n"); } return has_next; }; const auto on_complete = [request = q_ptr, response = r_ptr](bool) mutable { + // the peer may have dropped before the producer finished. when a pipe is attached it + // drains the rest of the generation into the ring buffer on this same worker, see + // stream_pipe::finish_producer + if (response->spipe) { + response->spipe->finish_producer(); + } response.reset(); // spipe destructor finalizes the session if attached request.reset(); }; diff --git a/tools/server/server-http.h b/tools/server/server-http.h index 1afec5fd9ad4..f2d8831d4e7e 100644 --- a/tools/server/server-http.h +++ b/tools/server/server-http.h @@ -25,9 +25,10 @@ struct server_http_res { std::string data; std::map headers; - // if set, the stream survives a client disconnect: the http layer keeps calling next() and - // feeding chunks into the pipe even when sink.write fails, until next() returns false. - // the pipe destructor finalizes the session so no explicit on_stream_end callback is needed. + // if set, the stream survives a client disconnect: when the peer leaves before the producer + // is done, on_complete calls spipe->finish_producer() to drain the rest of the generation into + // the ring buffer on the same worker. the pipe destructor finalizes the session so no explicit + // on_stream_end callback is needed. // shared_ptr used (not unique_ptr) so the forward-declared type is safe to delete here. std::shared_ptr spipe; diff --git a/tools/server/server-stream.cpp b/tools/server/server-stream.cpp index 47e768215a73..8b326d0a68a0 100644 --- a/tools/server/server-stream.cpp +++ b/tools/server/server-stream.cpp @@ -337,6 +337,32 @@ bool stream_pipe::is_cancelled() const { return session_->is_cancelled(); } +void stream_pipe::mark_producer_done() { + producer_done_ = true; +} + +void stream_pipe::finish_producer() { + // the peer dropped before the producer finished. httplib bails its content provider the moment + // is_peer_alive() goes false, so the rest of the generation is pumped here into the ring buffer + // on the caller's thread. stream_aware_should_stop ignores peer disconnect while a pipe is + // attached, so res_->next() runs to natural completion, only an explicit DELETE flips + // is_cancelled and cuts it short. is_producer_ guarantees res_ is set + if (!is_producer_ || producer_done_ || session_->is_cancelled()) { + return; + } + std::string chunk; + while (true) { + chunk.clear(); + bool has_next = res_->next(chunk); + if (!chunk.empty()) { + write(chunk.data(), chunk.size()); + } + if (!has_next) { + break; + } + } +} + std::shared_ptr stream_pipe::create_producer(stream_session_ptr session, server_http_res & res) { auto alive = std::make_shared>(true); diff --git a/tools/server/server-stream.h b/tools/server/server-stream.h index b74df3412689..52d059e98776 100644 --- a/tools/server/server-stream.h +++ b/tools/server/server-stream.h @@ -100,6 +100,16 @@ struct stream_pipe { // true if the session was cancelled (e.g. via DELETE /v1/stream/) bool is_cancelled() const; + // producer: record that next() reached its natural end on the wire, so finish_producer turns + // into a no-op. the http drain calls this right before it closes the stream cleanly + void mark_producer_done(); + + // producer: when the peer dropped before the producer finished, pump the response next() into + // the ring buffer until it reports done. runs on the caller's thread (the http worker, from + // on_complete), no extra thread. no-op for a consumer pipe, an already finished producer, or a + // cancelled session. only an explicit DELETE flips is_cancelled and cuts the drain short + void finish_producer(); + // disarm the stop hook and mark the alive guard false; must be called while the // object that stop_fn references (the response reader) is still alive. // idempotent; ~stream_pipe() calls it automatically but callers can do it earlier. @@ -116,6 +126,7 @@ struct stream_pipe { private: stream_session_ptr session_; bool is_producer_; + bool producer_done_ = false; // producer only, set on clean wire end std::shared_ptr> alive_; // only set for producer pipes server_http_res * res_; // only set for producer pipes From 1b4d9a8258c967d88ca43d117b2178e38800b676 Mon Sep 17 00:00:00 2001 From: Pascal Date: Wed, 20 May 2026 10:25:20 +0200 Subject: [PATCH 20/39] server-stream: drain disconnected streams on a manager owned thread The previous commit pumped the post disconnect drain synchronously in on_complete, on the http worker, so a disconnected stream kept its worker busy until the generation ended. Under a wave of reloads or tab closes that pins workers from the pool. Move the drain off the http worker. on_complete now hands the response to stream_session_manager::adopt_orphan, which pumps it to completion on a manager owned thread and releases the worker at once. One thread per disconnected stream still generating, stored in a list, joined and reaped on the next adopt, by the GC, and at shutdown. No detach, the thread lifecycle is fully owned by the manager. needs_drain gates the handoff so a cleanly finished stream never spawns a thread, and the strict OAI path stays untouched when no pipe is attached. stop_gc now cancels sessions before finalizing them, so an in flight drain sees is_cancelled and exits instead of blocking the shutdown join until the generation ends naturally. --- tools/server/server-http.cpp | 11 +++--- tools/server/server-http.h | 8 ++--- tools/server/server-stream.cpp | 61 +++++++++++++++++++++++++++++++--- tools/server/server-stream.h | 31 +++++++++++++++-- 4 files changed, 95 insertions(+), 16 deletions(-) diff --git a/tools/server/server-http.cpp b/tools/server/server-http.cpp index e89680f05f04..2c007c965a78 100644 --- a/tools/server/server-http.cpp +++ b/tools/server/server-http.cpp @@ -528,11 +528,12 @@ static void process_handler_response(server_http_req_ptr && request, server_http return has_next; }; const auto on_complete = [request = q_ptr, response = r_ptr](bool) mutable { - // the peer may have dropped before the producer finished. when a pipe is attached it - // drains the rest of the generation into the ring buffer on this same worker, see - // stream_pipe::finish_producer - if (response->spipe) { - response->spipe->finish_producer(); + // the peer may have dropped before the producer finished. when a drain is still owed, + // hand the response to a manager owned thread that pumps it to completion, so this http + // worker is released at once. see stream_session_manager::adopt_orphan + if (response->spipe && response->spipe->needs_drain()) { + g_stream_sessions.adopt_orphan(std::move(response), std::move(request)); + return; } response.reset(); // spipe destructor finalizes the session if attached request.reset(); diff --git a/tools/server/server-http.h b/tools/server/server-http.h index f2d8831d4e7e..cb27bc85990a 100644 --- a/tools/server/server-http.h +++ b/tools/server/server-http.h @@ -25,10 +25,10 @@ struct server_http_res { std::string data; std::map headers; - // if set, the stream survives a client disconnect: when the peer leaves before the producer - // is done, on_complete calls spipe->finish_producer() to drain the rest of the generation into - // the ring buffer on the same worker. the pipe destructor finalizes the session so no explicit - // on_stream_end callback is needed. + // if set, the stream survives a client disconnect: when the peer leaves before the producer is + // done, on_complete hands the response to a manager owned thread that drains the rest of the + // generation into the ring buffer, releasing the http worker at once. the pipe destructor + // finalizes the session so no explicit on_stream_end callback is needed. // shared_ptr used (not unique_ptr) so the forward-declared type is safe to delete here. std::shared_ptr spipe; diff --git a/tools/server/server-stream.cpp b/tools/server/server-stream.cpp index 8b326d0a68a0..c7f9892195e2 100644 --- a/tools/server/server-stream.cpp +++ b/tools/server/server-stream.cpp @@ -230,6 +230,37 @@ void stream_session_manager::evict_and_cancel(const std::string & conversation_i s->finalize(); } +void stream_session_manager::reap_orphans_locked() { + for (auto it = orphans.begin(); it != orphans.end(); ) { + if (it->finished.load(std::memory_order_acquire)) { + if (it->thread.joinable()) { + it->thread.join(); + } + it = orphans.erase(it); + } else { + ++it; + } + } +} + +void stream_session_manager::adopt_orphan(std::shared_ptr res, + std::shared_ptr req) { + std::lock_guard lock(orphan_mu); + reap_orphans_locked(); + auto & node = orphans.emplace_back(); + auto * finished = &node.finished; + node.thread = std::thread([res = std::move(res), req = std::move(req), finished]() mutable { + // pump the rest of the generation into the ring buffer, then drop the response and the + // request. dropping res finalizes the session through the producer pipe destructor + if (res->spipe) { + res->spipe->finish_producer(); + } + res.reset(); + req.reset(); + finished->store(true, std::memory_order_release); + }); +} + void stream_session_manager::start_gc() { if (running.exchange(true)) { return; @@ -249,7 +280,8 @@ void stream_session_manager::stop_gc() { gc_thread.join(); } } - // finalize all live sessions so no reader ever hangs + // cancel then finalize all live sessions so any in flight orphan drain unblocks and no reader + // ever hangs std::vector snapshot; { std::unique_lock lock(map_mu); @@ -260,8 +292,19 @@ void stream_session_manager::stop_gc() { sessions.clear(); } for (auto & s : snapshot) { + s->cancel(); s->finalize(); } + // join the orphan drain threads now that their producers are cancelled + { + std::lock_guard lock(orphan_mu); + for (auto & o : orphans) { + if (o.thread.joinable()) { + o.thread.join(); + } + } + orphans.clear(); + } } void stream_session_manager::gc_loop() { @@ -293,6 +336,11 @@ void stream_session_manager::gc_loop() { for (auto & s : to_drop) { s->finalize(); } + // reap any orphan drain threads that finished since the last pass + { + std::lock_guard lock(orphan_mu); + reap_orphans_locked(); + } } } @@ -344,9 +392,10 @@ void stream_pipe::mark_producer_done() { void stream_pipe::finish_producer() { // the peer dropped before the producer finished. httplib bails its content provider the moment // is_peer_alive() goes false, so the rest of the generation is pumped here into the ring buffer - // on the caller's thread. stream_aware_should_stop ignores peer disconnect while a pipe is - // attached, so res_->next() runs to natural completion, only an explicit DELETE flips - // is_cancelled and cuts it short. is_producer_ guarantees res_ is set + // on the orphan drain thread that on_complete handed this response to. stream_aware_should_stop + // ignores peer disconnect while a pipe is attached, so res_->next() runs to natural completion, + // only an explicit DELETE flips is_cancelled and cuts it short. is_producer_ guarantees res_ is + // set, the guard also covers a DELETE that races between adopt_orphan and this thread starting if (!is_producer_ || producer_done_ || session_->is_cancelled()) { return; } @@ -363,6 +412,10 @@ void stream_pipe::finish_producer() { } } +bool stream_pipe::needs_drain() const { + return is_producer_ && !producer_done_ && !session_->is_cancelled(); +} + std::shared_ptr stream_pipe::create_producer(stream_session_ptr session, server_http_res & res) { auto alive = std::make_shared>(true); diff --git a/tools/server/server-stream.h b/tools/server/server-stream.h index 52d059e98776..592d9877ab54 100644 --- a/tools/server/server-stream.h +++ b/tools/server/server-stream.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -105,11 +106,16 @@ struct stream_pipe { void mark_producer_done(); // producer: when the peer dropped before the producer finished, pump the response next() into - // the ring buffer until it reports done. runs on the caller's thread (the http worker, from - // on_complete), no extra thread. no-op for a consumer pipe, an already finished producer, or a - // cancelled session. only an explicit DELETE flips is_cancelled and cuts the drain short + // the ring buffer until it reports done. runs on the caller's thread (the orphan drain thread, + // see stream_session_manager::adopt_orphan), no-op for a consumer pipe, an already finished + // producer, or a cancelled session. only an explicit DELETE flips is_cancelled and cuts short void finish_producer(); + // producer: true when a disconnect drain is still owed, the producer has not reached its + // natural end and the session is not cancelled. on_complete checks this before handing the + // response to an orphan drain thread, so a cleanly finished stream never spawns one + bool needs_drain() const; + // disarm the stop hook and mark the alive guard false; must be called while the // object that stop_fn references (the response reader) is still alive. // idempotent; ~stream_pipe() calls it automatically but callers can do it earlier. @@ -161,6 +167,12 @@ class stream_session_manager { // signal the producer to cancel asap then evict, used by the explicit user Stop path void evict_and_cancel(const std::string & conversation_id); + // take ownership of a response whose client disconnected mid generation and pump it to + // completion on a manager owned thread, so the http worker is released at once. the response + // and its request stay alive until the drain finishes, then both are dropped here. finished + // threads are joined and reaped on the next adopt, by the GC, and at shutdown + void adopt_orphan(std::shared_ptr res, std::shared_ptr req); + void start_gc(); void stop_gc(); @@ -170,6 +182,9 @@ class stream_session_manager { private: void gc_loop(); + // join and erase the orphan drain threads that have already finished, caller holds orphan_mu + void reap_orphans_locked(); + mutable std::shared_mutex map_mu; std::unordered_map sessions; // key: conversation_id std::thread gc_thread; @@ -177,6 +192,16 @@ class stream_session_manager { std::mutex gc_wake_mu; std::condition_variable gc_wake_cv; std::shared_ptr> drain_shutdown; + + // one entry per disconnected stream still generating. the drain thread flips finished on exit + // so reap can join without blocking. std::list keeps the node address stable for the flag + // pointer captured by the thread + struct orphan_drain { + std::thread thread; + std::atomic finished{false}; + }; + std::mutex orphan_mu; + std::list orphans; }; // the process wide stream session manager. defined in server-stream.cpp so the symbol From 4a5cbac59212e689ab4f7cd74086f7add4cd1656 Mon Sep 17 00:00:00 2001 From: Pascal Date: Wed, 20 May 2026 12:09:21 +0200 Subject: [PATCH 21/39] ui: add missing JSDoc --- tools/ui/src/lib/components/app/chat/index.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/ui/src/lib/components/app/chat/index.ts b/tools/ui/src/lib/components/app/chat/index.ts index 4d039d056b5e..d7004d3ae377 100644 --- a/tools/ui/src/lib/components/app/chat/index.ts +++ b/tools/ui/src/lib/components/app/chat/index.ts @@ -684,4 +684,10 @@ export { default as ChatScreenProcessingInfo } from './ChatScreen/ChatScreenProc */ export { default as ChatScreenServerError } from './ChatScreen/ChatScreenServerError.svelte'; +/** + * Stream resume status indicator. Shows a small "Reconnecting to the stream..." + * banner with a spinner while `chatStore.streamConnectionState` is `resuming`, + * i.e. after a dropped connection is reattaching to the live SSE replay buffer. + * Renders nothing otherwise. Shown inside ChatScreen only on an active conversation route. + */ export { default as ChatScreenStreamResumeStatus } from './ChatScreen/ChatScreenStreamResumeStatus.svelte'; From cffee543e0fe4babb04f10d599c667c73ccc89f8 Mon Sep 17 00:00:00 2001 From: Pascal Date: Wed, 20 May 2026 12:36:02 +0200 Subject: [PATCH 22/39] server-stream: drain on the http worker, drop the manager thread Address @ngxson review: httplib runs a large dynamic pool and a worker blocked in next() sits on a condvar instead of burning cpu, so draining the rest of the generation on that worker is fine and much simpler than a dedicated thread. on_complete calls finish_producer directly again. Removes adopt_orphan, the orphan thread list and its reaping, the stop_gc session cancel that only existed to unblock those threads, and the now dead drain_shutdown flag. --- tools/server/server-http.cpp | 12 +++---- tools/server/server-stream.cpp | 65 +++------------------------------- tools/server/server-stream.h | 35 ++---------------- 3 files changed, 14 insertions(+), 98 deletions(-) diff --git a/tools/server/server-http.cpp b/tools/server/server-http.cpp index 2c007c965a78..81407f1f7197 100644 --- a/tools/server/server-http.cpp +++ b/tools/server/server-http.cpp @@ -528,12 +528,12 @@ static void process_handler_response(server_http_req_ptr && request, server_http return has_next; }; const auto on_complete = [request = q_ptr, response = r_ptr](bool) mutable { - // the peer may have dropped before the producer finished. when a drain is still owed, - // hand the response to a manager owned thread that pumps it to completion, so this http - // worker is released at once. see stream_session_manager::adopt_orphan - if (response->spipe && response->spipe->needs_drain()) { - g_stream_sessions.adopt_orphan(std::move(response), std::move(request)); - return; + // the peer may have dropped before the producer finished. when a pipe is attached, drain + // the rest of the generation into the ring buffer here, on this http worker. httplib + // runs a large dynamic pool and the worker blocks in next() on a condvar rather than + // burning cpu, so holding it until the generation ends is fine. see finish_producer + if (response->spipe) { + response->spipe->finish_producer(); } response.reset(); // spipe destructor finalizes the session if attached request.reset(); diff --git a/tools/server/server-stream.cpp b/tools/server/server-stream.cpp index c7f9892195e2..9535dc3b84a5 100644 --- a/tools/server/server-stream.cpp +++ b/tools/server/server-stream.cpp @@ -149,8 +149,7 @@ bool stream_session::is_cancelled() const { } stream_session_manager::stream_session_manager() - : running(false) - , drain_shutdown(std::make_shared>(false)) { + : running(false) { } stream_session_manager::~stream_session_manager() { @@ -230,37 +229,6 @@ void stream_session_manager::evict_and_cancel(const std::string & conversation_i s->finalize(); } -void stream_session_manager::reap_orphans_locked() { - for (auto it = orphans.begin(); it != orphans.end(); ) { - if (it->finished.load(std::memory_order_acquire)) { - if (it->thread.joinable()) { - it->thread.join(); - } - it = orphans.erase(it); - } else { - ++it; - } - } -} - -void stream_session_manager::adopt_orphan(std::shared_ptr res, - std::shared_ptr req) { - std::lock_guard lock(orphan_mu); - reap_orphans_locked(); - auto & node = orphans.emplace_back(); - auto * finished = &node.finished; - node.thread = std::thread([res = std::move(res), req = std::move(req), finished]() mutable { - // pump the rest of the generation into the ring buffer, then drop the response and the - // request. dropping res finalizes the session through the producer pipe destructor - if (res->spipe) { - res->spipe->finish_producer(); - } - res.reset(); - req.reset(); - finished->store(true, std::memory_order_release); - }); -} - void stream_session_manager::start_gc() { if (running.exchange(true)) { return; @@ -269,7 +237,6 @@ void stream_session_manager::start_gc() { } void stream_session_manager::stop_gc() { - drain_shutdown->store(true, std::memory_order_release); bool was_running = running.exchange(false); if (was_running) { { @@ -280,8 +247,7 @@ void stream_session_manager::stop_gc() { gc_thread.join(); } } - // cancel then finalize all live sessions so any in flight orphan drain unblocks and no reader - // ever hangs + // finalize all live sessions so no reader ever hangs std::vector snapshot; { std::unique_lock lock(map_mu); @@ -292,19 +258,8 @@ void stream_session_manager::stop_gc() { sessions.clear(); } for (auto & s : snapshot) { - s->cancel(); s->finalize(); } - // join the orphan drain threads now that their producers are cancelled - { - std::lock_guard lock(orphan_mu); - for (auto & o : orphans) { - if (o.thread.joinable()) { - o.thread.join(); - } - } - orphans.clear(); - } } void stream_session_manager::gc_loop() { @@ -336,11 +291,6 @@ void stream_session_manager::gc_loop() { for (auto & s : to_drop) { s->finalize(); } - // reap any orphan drain threads that finished since the last pass - { - std::lock_guard lock(orphan_mu); - reap_orphans_locked(); - } } } @@ -392,10 +342,9 @@ void stream_pipe::mark_producer_done() { void stream_pipe::finish_producer() { // the peer dropped before the producer finished. httplib bails its content provider the moment // is_peer_alive() goes false, so the rest of the generation is pumped here into the ring buffer - // on the orphan drain thread that on_complete handed this response to. stream_aware_should_stop - // ignores peer disconnect while a pipe is attached, so res_->next() runs to natural completion, - // only an explicit DELETE flips is_cancelled and cuts it short. is_producer_ guarantees res_ is - // set, the guard also covers a DELETE that races between adopt_orphan and this thread starting + // on the http worker, from on_complete. stream_aware_should_stop ignores peer disconnect while a + // pipe is attached, so res_->next() runs to natural completion, only an explicit DELETE flips + // is_cancelled and cuts it short. is_producer_ guarantees res_ is set if (!is_producer_ || producer_done_ || session_->is_cancelled()) { return; } @@ -412,10 +361,6 @@ void stream_pipe::finish_producer() { } } -bool stream_pipe::needs_drain() const { - return is_producer_ && !producer_done_ && !session_->is_cancelled(); -} - std::shared_ptr stream_pipe::create_producer(stream_session_ptr session, server_http_res & res) { auto alive = std::make_shared>(true); diff --git a/tools/server/server-stream.h b/tools/server/server-stream.h index 592d9877ab54..c2758c801be1 100644 --- a/tools/server/server-stream.h +++ b/tools/server/server-stream.h @@ -7,7 +7,6 @@ #include #include #include -#include #include #include #include @@ -106,16 +105,11 @@ struct stream_pipe { void mark_producer_done(); // producer: when the peer dropped before the producer finished, pump the response next() into - // the ring buffer until it reports done. runs on the caller's thread (the orphan drain thread, - // see stream_session_manager::adopt_orphan), no-op for a consumer pipe, an already finished - // producer, or a cancelled session. only an explicit DELETE flips is_cancelled and cuts short + // the ring buffer until it reports done. runs on the caller's thread (the http worker, from + // on_complete), no-op for a consumer pipe, an already finished producer, or a cancelled session. + // only an explicit DELETE flips is_cancelled and cuts the drain short void finish_producer(); - // producer: true when a disconnect drain is still owed, the producer has not reached its - // natural end and the session is not cancelled. on_complete checks this before handing the - // response to an orphan drain thread, so a cleanly finished stream never spawns one - bool needs_drain() const; - // disarm the stop hook and mark the alive guard false; must be called while the // object that stop_fn references (the response reader) is still alive. // idempotent; ~stream_pipe() calls it automatically but callers can do it earlier. @@ -167,41 +161,18 @@ class stream_session_manager { // signal the producer to cancel asap then evict, used by the explicit user Stop path void evict_and_cancel(const std::string & conversation_id); - // take ownership of a response whose client disconnected mid generation and pump it to - // completion on a manager owned thread, so the http worker is released at once. the response - // and its request stay alive until the drain finishes, then both are dropped here. finished - // threads are joined and reaped on the next adopt, by the GC, and at shutdown - void adopt_orphan(std::shared_ptr res, std::shared_ptr req); - void start_gc(); void stop_gc(); - // shared atomic flipped to true on stop_gc, drain threads poll it to exit cleanly - std::shared_ptr> shutdown_flag() const { return drain_shutdown; } - private: void gc_loop(); - // join and erase the orphan drain threads that have already finished, caller holds orphan_mu - void reap_orphans_locked(); - mutable std::shared_mutex map_mu; std::unordered_map sessions; // key: conversation_id std::thread gc_thread; std::atomic running; std::mutex gc_wake_mu; std::condition_variable gc_wake_cv; - std::shared_ptr> drain_shutdown; - - // one entry per disconnected stream still generating. the drain thread flips finished on exit - // so reap can join without blocking. std::list keeps the node address stable for the flag - // pointer captured by the thread - struct orphan_drain { - std::thread thread; - std::atomic finished{false}; - }; - std::mutex orphan_mu; - std::list orphans; }; // the process wide stream session manager. defined in server-stream.cpp so the symbol From 231e3a0e4b97e4265608a4402894d37f3513af1d Mon Sep 17 00:00:00 2001 From: Pascal Date: Wed, 20 May 2026 13:00:00 +0200 Subject: [PATCH 23/39] server-stream: split stream_pipe into producer and consumer classes Address @ngxson review: one class covering both ends was messy. stream_pipe is now a base holding the session and is_cancelled, with stream_pipe_producer (write, mark_producer_done, finish_producer, cleanup, finalizes on destruct) and stream_pipe_consumer (read only, no finalize) deriving from it. Drops the is_producer_ discriminator and its runtime guards, the type now encodes the role. res.spipe is retyped to shared_ptr since it is only ever a producer. No behavior change. --- tools/server/server-http.h | 9 ++-- tools/server/server-stream.cpp | 72 ++++++++++++++------------ tools/server/server-stream.h | 93 +++++++++++++++++++--------------- 3 files changed, 96 insertions(+), 78 deletions(-) diff --git a/tools/server/server-http.h b/tools/server/server-http.h index cb27bc85990a..335219e234af 100644 --- a/tools/server/server-http.h +++ b/tools/server/server-http.h @@ -11,7 +11,7 @@ #include struct common_params; -struct stream_pipe; // defined in server-stream.h +struct stream_pipe_producer; // defined in server-stream.h // generator-like API for HTTP response generation // this object response with one of the 2 modes: @@ -26,11 +26,10 @@ struct server_http_res { std::map headers; // if set, the stream survives a client disconnect: when the peer leaves before the producer is - // done, on_complete hands the response to a manager owned thread that drains the rest of the - // generation into the ring buffer, releasing the http worker at once. the pipe destructor - // finalizes the session so no explicit on_stream_end callback is needed. + // done, on_complete drains the rest of the generation into the ring buffer on the http worker. + // the producer pipe destructor finalizes the session so no explicit on_stream_end is needed. // shared_ptr used (not unique_ptr) so the forward-declared type is safe to delete here. - std::shared_ptr spipe; + std::shared_ptr spipe; std::function next = nullptr; bool is_stream() const { diff --git a/tools/server/server-stream.cpp b/tools/server/server-stream.cpp index 9535dc3b84a5..3e7f51808175 100644 --- a/tools/server/server-stream.cpp +++ b/tools/server/server-stream.cpp @@ -299,21 +299,27 @@ stream_session_manager g_stream_sessions; // stream_pipe --------------------------------------------------------------------------------- -stream_pipe::stream_pipe(stream_session_ptr session, bool is_producer) - : session_(std::move(session)) - , is_producer_(is_producer) - , res_(nullptr) { +stream_pipe::stream_pipe(stream_session_ptr session) + : session_(std::move(session)) { } -stream_pipe::~stream_pipe() { +bool stream_pipe::is_cancelled() const { + return session_->is_cancelled(); +} + +// stream_pipe_producer + +stream_pipe_producer::stream_pipe_producer(stream_session_ptr session) + : stream_pipe(std::move(session)) { +} + +stream_pipe_producer::~stream_pipe_producer() { cleanup(); - if (is_producer_) { - session_->finalize(); - } + session_->finalize(); } -void stream_pipe::cleanup() { - if (!is_producer_ || !alive_) { +void stream_pipe_producer::cleanup() { + if (!alive_) { return; } alive_->store(false, std::memory_order_release); @@ -321,31 +327,21 @@ void stream_pipe::cleanup() { alive_.reset(); } -bool stream_pipe::write(const char * data, size_t len) { +bool stream_pipe_producer::write(const char * data, size_t len) { return session_->append(data, len); } -stream_read_status stream_pipe::read(size_t & offset, - const std::function & sink, - const std::function & should_stop) { - return session_->read_from(offset, sink, should_stop); -} - -bool stream_pipe::is_cancelled() const { - return session_->is_cancelled(); -} - -void stream_pipe::mark_producer_done() { +void stream_pipe_producer::mark_producer_done() { producer_done_ = true; } -void stream_pipe::finish_producer() { +void stream_pipe_producer::finish_producer() { // the peer dropped before the producer finished. httplib bails its content provider the moment // is_peer_alive() goes false, so the rest of the generation is pumped here into the ring buffer // on the http worker, from on_complete. stream_aware_should_stop ignores peer disconnect while a // pipe is attached, so res_->next() runs to natural completion, only an explicit DELETE flips - // is_cancelled and cuts it short. is_producer_ guarantees res_ is set - if (!is_producer_ || producer_done_ || session_->is_cancelled()) { + // is_cancelled and cuts it short + if (producer_done_ || session_->is_cancelled()) { return; } std::string chunk; @@ -361,8 +357,8 @@ void stream_pipe::finish_producer() { } } -std::shared_ptr stream_pipe::create_producer(stream_session_ptr session, - server_http_res & res) { +std::shared_ptr stream_pipe_producer::create(stream_session_ptr session, + server_http_res & res) { auto alive = std::make_shared>(true); auto * res_ptr = &res; session->set_stop_producer([alive, res_ptr]() { @@ -370,14 +366,26 @@ std::shared_ptr stream_pipe::create_producer(stream_session_ptr ses res_ptr->stop(); } }); - auto pipe = std::shared_ptr(new stream_pipe(std::move(session), true)); + auto pipe = std::shared_ptr(new stream_pipe_producer(std::move(session))); pipe->alive_ = std::move(alive); pipe->res_ = res_ptr; return pipe; } -std::shared_ptr stream_pipe::create_consumer(stream_session_ptr session) { - return std::shared_ptr(new stream_pipe(std::move(session), false)); +// stream_pipe_consumer + +stream_pipe_consumer::stream_pipe_consumer(stream_session_ptr session) + : stream_pipe(std::move(session)) { +} + +stream_read_status stream_pipe_consumer::read(size_t & offset, + const std::function & sink, + const std::function & should_stop) { + return session_->read_from(offset, sink, should_stop); +} + +std::shared_ptr stream_pipe_consumer::create(stream_session_ptr session) { + return std::shared_ptr(new stream_pipe_consumer(std::move(session))); } // helper, builds the standard error response and assigns it to a brand new http_res @@ -424,7 +432,7 @@ server_http_context::handler_t make_stream_get_handler() { // chunk so set_chunked_content_provider gets a chance to flush to the socket auto offset_ptr = std::make_shared(from); // consumer pipe: read-only, does not finalize the session on destruction - auto pipe = stream_pipe::create_consumer(session); + auto pipe = stream_pipe_consumer::create(session); res->next = [pipe, offset_ptr, &req](std::string & output) -> bool { bool got_any = false; pipe->read(*offset_ptr, @@ -542,7 +550,7 @@ void stream_session_attach_pipe(server_http_res & res, const std::map stream_aware_should_stop(server_http_res * res, std::function fallback) { diff --git a/tools/server/server-stream.h b/tools/server/server-stream.h index c2758c801be1..955dd97b731a 100644 --- a/tools/server/server-stream.h +++ b/tools/server/server-stream.h @@ -76,61 +76,72 @@ struct stream_session { using stream_session_ptr = std::shared_ptr; -// RAII wrapper around a stream_session that represents one end of the pipe. -// the producer side writes chunks and is responsible for finalizing; the consumer side reads. -// -// lifetime safety: the producer pipe holds a shared_ptr> alive that is also -// captured by the session's stop_producer hook. cleanup() sets alive=false and clears the -// hook; it must be called while the owning response object is still valid (i.e. before the -// reader it would call stop() on is destroyed). ~server_res_generator() does this explicitly. +// one end of a stream_session pipe. the base holds the session and the shared query, the +// producer and consumer ends derive from it. virtual dtor so each end runs its own teardown: +// the producer finalizes the session, the consumer leaves it untouched struct stream_pipe { - ~stream_pipe(); - - // producer: append raw bytes to the session's ring buffer. - // returns false if the session is already finalized. - bool write(const char * data, size_t len); - - // consumer: drain bytes from offset, calling sink for each available chunk. - // blocks until more data arrives or the session finalizes. - // should_stop is polled periodically; returns OFFSET_LOST if offset fell below the prefix. - stream_read_status read(size_t & offset, - const std::function & sink, - const std::function & should_stop); + virtual ~stream_pipe() = default; // true if the session was cancelled (e.g. via DELETE /v1/stream/) bool is_cancelled() const; - // producer: record that next() reached its natural end on the wire, so finish_producer turns - // into a no-op. the http drain calls this right before it closes the stream cleanly +protected: + explicit stream_pipe(stream_session_ptr session); + + stream_session_ptr session_; +}; + +// producer end: writes chunks into the ring buffer and owns the session lifetime, finalizing it +// on destruction. +// +// lifetime safety: holds a shared_ptr> alive also captured by the session's +// stop_producer hook. cleanup() sets alive=false and clears the hook; it must run while the +// response the hook calls stop() on is still alive. ~server_res_generator() does this explicitly. +struct stream_pipe_producer : stream_pipe { + ~stream_pipe_producer() override; + + // append raw bytes to the session's ring buffer, returns false if already finalized + bool write(const char * data, size_t len); + + // record that next() reached its natural end on the wire, so finish_producer turns into a + // no-op. the http drain calls this right before it closes the stream cleanly void mark_producer_done(); - // producer: when the peer dropped before the producer finished, pump the response next() into - // the ring buffer until it reports done. runs on the caller's thread (the http worker, from - // on_complete), no-op for a consumer pipe, an already finished producer, or a cancelled session. - // only an explicit DELETE flips is_cancelled and cuts the drain short + // when the peer dropped before the producer finished, pump the response next() into the ring + // buffer until it reports done. runs on the http worker, from on_complete. no-op for an + // already finished producer or a cancelled session, only a DELETE flips is_cancelled and cuts + // the drain short void finish_producer(); - // disarm the stop hook and mark the alive guard false; must be called while the - // object that stop_fn references (the response reader) is still alive. - // idempotent; ~stream_pipe() calls it automatically but callers can do it earlier. + // disarm the stop hook and drop the alive guard, must run while the response the hook + // references is still alive. idempotent, the destructor calls it too void cleanup(); - // factory: producer pipe. res.stop() is invoked when the session is cancelled. - // the alive guard ensures stop() is not called after cleanup() has run. - static std::shared_ptr create_producer(stream_session_ptr session, - server_http_res & res); - - // factory: consumer pipe (read-only; destructor does not finalize the session). - static std::shared_ptr create_consumer(stream_session_ptr session); + // res.stop() is invoked when the session is cancelled, the alive guard ensures stop() is not + // called after cleanup() has run + static std::shared_ptr create(stream_session_ptr session, server_http_res & res); private: - stream_session_ptr session_; - bool is_producer_; - bool producer_done_ = false; // producer only, set on clean wire end - std::shared_ptr> alive_; // only set for producer pipes - server_http_res * res_; // only set for producer pipes + explicit stream_pipe_producer(stream_session_ptr session); - stream_pipe(stream_session_ptr session, bool is_producer); + bool producer_done_ = false; + std::shared_ptr> alive_; + server_http_res * res_ = nullptr; +}; + +// consumer end: read-only replay of the ring buffer, the destructor does not finalize the session +struct stream_pipe_consumer : stream_pipe { + // drain bytes from offset, calling sink for each available chunk. blocks until more data + // arrives or the session finalizes. should_stop is polled, returns OFFSET_LOST if offset + // fell below the dropped prefix + stream_read_status read(size_t & offset, + const std::function & sink, + const std::function & should_stop); + + static std::shared_ptr create(stream_session_ptr session); + +private: + explicit stream_pipe_consumer(stream_session_ptr session); }; // owns all live sessions, runs a periodic GC to evict expired ones. From 4524ea7c85b283aa259e7c4b7a00fed18e5d5177 Mon Sep 17 00:00:00 2001 From: Pascal Date: Wed, 20 May 2026 13:18:50 +0200 Subject: [PATCH 24/39] server-stream: rename producer methods to unix pipe semantics Address @ngxson review: mark_producer_done becomes done(), finish_producer becomes close(), matching a unix pipe write end. The producer_done_ member follows as done_. write() is unchanged. No behavior change. --- tools/server/server-http.cpp | 8 ++++---- tools/server/server-stream.cpp | 8 ++++---- tools/server/server-stream.h | 18 +++++++++--------- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/tools/server/server-http.cpp b/tools/server/server-http.cpp index 81407f1f7197..0f0c9c7da0a2 100644 --- a/tools/server/server-http.cpp +++ b/tools/server/server-http.cpp @@ -518,9 +518,9 @@ static void process_handler_response(server_http_req_ptr && request, server_http SRV_DBG("http: streamed chunk: %s\n", chunk.c_str()); } if (!has_next) { - // producer reached its natural end on the wire, the pipe skips its drain + // producer reached its natural end on the wire, a later close() skips the drain if (response->spipe) { - response->spipe->mark_producer_done(); + response->spipe->done(); } sink.done(); SRV_DBG("%s", "http: stream ended\n"); @@ -531,9 +531,9 @@ static void process_handler_response(server_http_req_ptr && request, server_http // the peer may have dropped before the producer finished. when a pipe is attached, drain // the rest of the generation into the ring buffer here, on this http worker. httplib // runs a large dynamic pool and the worker blocks in next() on a condvar rather than - // burning cpu, so holding it until the generation ends is fine. see finish_producer + // burning cpu, so holding it until the generation ends is fine. see stream_pipe_producer::close if (response->spipe) { - response->spipe->finish_producer(); + response->spipe->close(); } response.reset(); // spipe destructor finalizes the session if attached request.reset(); diff --git a/tools/server/server-stream.cpp b/tools/server/server-stream.cpp index 3e7f51808175..26ffa4b1c73b 100644 --- a/tools/server/server-stream.cpp +++ b/tools/server/server-stream.cpp @@ -331,17 +331,17 @@ bool stream_pipe_producer::write(const char * data, size_t len) { return session_->append(data, len); } -void stream_pipe_producer::mark_producer_done() { - producer_done_ = true; +void stream_pipe_producer::done() { + done_ = true; } -void stream_pipe_producer::finish_producer() { +void stream_pipe_producer::close() { // the peer dropped before the producer finished. httplib bails its content provider the moment // is_peer_alive() goes false, so the rest of the generation is pumped here into the ring buffer // on the http worker, from on_complete. stream_aware_should_stop ignores peer disconnect while a // pipe is attached, so res_->next() runs to natural completion, only an explicit DELETE flips // is_cancelled and cuts it short - if (producer_done_ || session_->is_cancelled()) { + if (done_ || session_->is_cancelled()) { return; } std::string chunk; diff --git a/tools/server/server-stream.h b/tools/server/server-stream.h index 955dd97b731a..4b8aa9528ae2 100644 --- a/tools/server/server-stream.h +++ b/tools/server/server-stream.h @@ -103,15 +103,15 @@ struct stream_pipe_producer : stream_pipe { // append raw bytes to the session's ring buffer, returns false if already finalized bool write(const char * data, size_t len); - // record that next() reached its natural end on the wire, so finish_producer turns into a - // no-op. the http drain calls this right before it closes the stream cleanly - void mark_producer_done(); + // record that the producer reached its natural end on the wire, so a later close() turns into + // a no-op. the http drain calls this right before it closes the stream cleanly + void done(); - // when the peer dropped before the producer finished, pump the response next() into the ring - // buffer until it reports done. runs on the http worker, from on_complete. no-op for an - // already finished producer or a cancelled session, only a DELETE flips is_cancelled and cuts - // the drain short - void finish_producer(); + // close the producer end. when the peer dropped before the producer finished, pump the + // response next() into the ring buffer until it reports done. runs on the http worker, from + // on_complete. no-op once done() has fired or the session is cancelled, only a DELETE flips + // is_cancelled and cuts the drain short + void close(); // disarm the stop hook and drop the alive guard, must run while the response the hook // references is still alive. idempotent, the destructor calls it too @@ -124,7 +124,7 @@ struct stream_pipe_producer : stream_pipe { private: explicit stream_pipe_producer(stream_session_ptr session); - bool producer_done_ = false; + bool done_ = false; std::shared_ptr> alive_; server_http_res * res_ = nullptr; }; From 52884f18905cc782771b344492512f435f72d9ad Mon Sep 17 00:00:00 2001 From: Pascal Date: Thu, 21 May 2026 21:01:59 +0200 Subject: [PATCH 25/39] server, ui: route resumable streams via a conv map, persist resume identity Address ngxson review: drop the polling probe, proxy_post records a conv_id -> model map and the stream routes resolve the owning child with one lookup. The map is the single source of truth, the ::model suffix stays for child session uniqueness but the router never parses it. UI: the server keys a session by the POST time identity (conv::model), but reload probed with the bare conv id and missed model tagged sessions, so F5 stopped the stream and sidebar spinners stayed off. Persist the model and rebuild the exact identity on resume, single conv and bulk sidebar both send it. Add unit coverage for the identity round trip. --- tools/server/server-http.cpp | 29 ++- tools/server/server-models.cpp | 177 +++++++++--------- tools/server/server-models.h | 16 ++ tools/server/server-stream.cpp | 19 +- tools/server/server-stream.h | 4 + tools/ui/src/lib/services/chat.service.ts | 4 +- .../src/lib/services/stream-resume.service.ts | 28 ++- tools/ui/src/lib/stores/chat.svelte.ts | 30 ++- tools/ui/tests/unit/stream-resume.test.ts | 54 +++++- 9 files changed, 258 insertions(+), 103 deletions(-) diff --git a/tools/server/server-http.cpp b/tools/server/server-http.cpp index 0f0c9c7da0a2..13509cb6de1e 100644 --- a/tools/server/server-http.cpp +++ b/tools/server/server-http.cpp @@ -457,13 +457,40 @@ static void set_headers(httplib::Response & res, const std::map int { + if (c >= '0' && c <= '9') return c - '0'; + if (c >= 'a' && c <= 'f') return c - 'a' + 10; + if (c >= 'A' && c <= 'F') return c - 'A' + 10; + return -1; + }; + int hi = hex(in[i + 1]); + int lo = hex(in[i + 2]); + if (hi >= 0 && lo >= 0) { + out.push_back(char((hi << 4) | lo)); + i += 2; + continue; + } + } + out.push_back(in[i]); + } + return out; +} + static std::map get_params(const httplib::Request & req) { std::map params; for (const auto & [key, value] : req.params) { params[key] = value; } for (const auto & [key, value] : req.path_params) { - params[key] = value; + params[key] = decode_path_component(value); } return params; } diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index 7c69128e9083..00c85df6c38a 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -1,6 +1,7 @@ #include "server-common.h" #include "server-models.h" #include "server-context.h" +#include "server-stream.h" #include "build-info.h" #include "preset.h" @@ -1368,6 +1369,34 @@ void server_models::handle_child_state(const std::string & name, const std::stri } } +void server_models::remember_conv_model(const std::string & conv_id, const std::string & model) { + if (conv_id.empty() || model.empty()) { + return; + } + std::lock_guard lock(conv_model_mu); + conv_model_map[conv_id] = model; +} + +std::optional server_models::lookup_conv_model(const std::string & conv_id) { + if (conv_id.empty()) { + return std::nullopt; + } + std::lock_guard lock(conv_model_mu); + auto it = conv_model_map.find(conv_id); + if (it == conv_model_map.end()) { + return std::nullopt; + } + return it->second; +} + +void server_models::forget_conv_model(const std::string & conv_id) { + if (conv_id.empty()) { + return; + } + std::lock_guard lock(conv_model_mu); + conv_model_map.erase(conv_id); +} + // // server_child // @@ -1605,52 +1634,22 @@ static std::string encode_qs(const std::string & in) { return out; } -// extract the optional model suffix from a conversation_id. the WebUI encodes the active model -// name after :: when the user has explicitly picked a model, so the router can route stream -// lookups direct to the right child without probing every other one. returns empty when no -// separator is present, in which case the caller falls back to probing or fan out -static std::string extract_model_from_conv(const std::string & conv_id) { - static constexpr char SEP[] = "::"; - static constexpr size_t SEP_LEN = sizeof(SEP) - 1; - auto pos = conv_id.rfind(SEP); - if (pos == std::string::npos) { - return std::string(); - } - return conv_id.substr(pos + SEP_LEN); -} - -// loopback probe across every ready child, returns the meta of the first one that reports a -// live or recently completed session for this conv. only called as a fallback when the conv -// id carries no :: suffix -static std::optional probe_child_for_conv( +// resolve the child that owns a conversation's stream session via the conv_id -> model map +// populated when the POST was routed. single map lookup then a meta lookup, no polling, no +// parsing of the conv id. returns nullopt when nothing maps, the caller answers not found and +// the client recovers +static std::optional resolve_child_for_conv( server_models & models, const std::string & conversation_id) { if (conversation_id.empty()) { return std::nullopt; } - // POST /v1/streams/lookup with the one conv id we are probing. the child only returns a - // match if it owns that conv, listing is never exposed - json body = {{"conversation_ids", json::array({conversation_id})}}; - std::string body_str = body.dump(); - for (auto & meta : models.get_all_meta()) { - if (!meta.is_ready()) { - continue; - } - httplib::Client cli(CHILD_ADDR, meta.port); - cli.set_connection_timeout(0, STREAM_LOOKUP_TIMEOUT_MS * 1000); - cli.set_read_timeout(0, STREAM_LOOKUP_TIMEOUT_MS * 1000); - cli.set_write_timeout(0, STREAM_LOOKUP_TIMEOUT_MS * 1000); - auto resp = cli.Post("/v1/streams/lookup", body_str, "application/json"); - if (!resp || resp->status != 200) { - continue; - } - try { - json arr = json::parse(resp->body); - if (arr.is_array() && !arr.empty()) { - return meta; - } - } catch (const std::exception &) { - continue; - } + auto tracked = models.lookup_conv_model(conversation_id); + if (!tracked.has_value()) { + return std::nullopt; + } + auto meta = models.get_meta(*tracked); + if (meta.has_value() && meta->is_ready()) { + return meta; } return std::nullopt; } @@ -1703,6 +1702,13 @@ void server_models_routes::init_routes() { if (!router_validate_model(name, models, autoload, error_res)) { return error_res; } + // remember which child serves this conversation so the resumable stream routes can route + // straight to it without polling. key on the exact conv id from the header, the same value + // the GET and DELETE routes receive in their path, no parsing either side + std::string conv_id = stream_conv_id_from_headers(req.headers); + if (!conv_id.empty()) { + models.remember_conv_model(conv_id, name); + } return models.proxy_request(req, method, name, true); // update last usage for POST request only }; @@ -1905,26 +1911,16 @@ void server_models_routes::init_routes() { }; this->router_stream_get = [this](const server_http_req & req) { - // GET /v1/stream/?from=N. when the conv carries a ::model suffix, route - // straight to that child, otherwise loopback probe every ready child. returns 404 - // when no child currently owns a session for this conv + // GET /v1/stream/?from=N. resolve the owning child from the conv_id -> model + // map (no polling), 404 when nothing maps. a stale map entry just forwards to a child + // that answers not found, the client recovers auto res = std::make_unique(); std::string conv_id = req.get_param("conv_id"); if (conv_id.empty()) { res_err(res, format_error_response("Missing conversation id in path", ERROR_TYPE_INVALID_REQUEST)); return res; } - std::optional owner; - std::string model_hint = extract_model_from_conv(conv_id); - if (!model_hint.empty()) { - auto direct = models.get_meta(model_hint); - if (direct.has_value() && direct->is_ready()) { - owner = direct; - } - } - if (!owner.has_value()) { - owner = probe_child_for_conv(models, conv_id); - } + std::optional owner = resolve_child_for_conv(models, conv_id); if (!owner.has_value()) { res_err(res, format_error_response("Stream not found or expired", ERROR_TYPE_NOT_FOUND)); return res; @@ -1952,20 +1948,44 @@ void server_models_routes::init_routes() { }; this->router_streams_lookup = [this](const server_http_req & req) { - // POST /v1/streams/lookup forwards the same body to every ready child and aggregates - // the results. the child responds only for the conv ids we asked about, never lists - // anything else, so the router never exposes ids the caller did not already know + // POST /v1/streams/lookup. resolve each requested conv id to its owning child via the + // map, group the ids per child, and query only the children that actually own some of + // them instead of fanning out to every ready child. a child only answers for the ids + // it owns, never lists anything else auto res = std::make_unique(); - json aggregated = json::array(); - for (auto & meta : models.get_all_meta()) { - if (!meta.is_ready()) { + std::vector requested; + try { + json body = json::parse(req.body); + if (body.contains("conversation_ids") && body["conversation_ids"].is_array()) { + for (const auto & v : body["conversation_ids"]) { + if (v.is_string() && !v.get().empty()) { + requested.push_back(v.get()); + } + } + } + } catch (const std::exception &) { + res_ok(res, json::array()); + return res; + } + + // group requested ids by the child port that owns them, drop ids that map to nothing + std::unordered_map per_child; + for (const auto & cid : requested) { + auto owner = resolve_child_for_conv(models, cid); + if (!owner.has_value()) { continue; } - httplib::Client cli(CHILD_ADDR, meta.port); + per_child[owner->port].push_back(cid); + } + + json aggregated = json::array(); + for (auto & [port, ids] : per_child) { + json child_body = {{"conversation_ids", ids}}; + httplib::Client cli(CHILD_ADDR, port); cli.set_connection_timeout(0, STREAM_LOOKUP_TIMEOUT_MS * 1000); cli.set_read_timeout(0, STREAM_LOOKUP_TIMEOUT_MS * 1000); cli.set_write_timeout(0, STREAM_LOOKUP_TIMEOUT_MS * 1000); - auto resp = cli.Post("/v1/streams/lookup", req.body, "application/json"); + auto resp = cli.Post("/v1/streams/lookup", child_body.dump(), "application/json"); if (!resp || resp->status != 200) { continue; } @@ -1988,9 +2008,9 @@ void server_models_routes::init_routes() { }; this->router_stream_delete = [this](const server_http_req & req) { - // DELETE /v1/stream/. with a ::model suffix we forward to that child only, - // otherwise fan out across every ready child. each child runs an idempotent - // evict_and_cancel so a child without the session returns 204 and does nothing + // DELETE /v1/stream/. resolve the owning child via the map and forward only to + // it. evict_and_cancel is idempotent on the child, a stale map entry just hits a child + // that has nothing to cancel and returns 204 auto res = std::make_unique(); std::string conv_id = req.get_param("conv_id"); if (conv_id.empty()) { @@ -1998,30 +2018,17 @@ void server_models_routes::init_routes() { return res; } std::string child_path = "/v1/stream/" + encode_qs(conv_id); - std::string model_hint = extract_model_from_conv(conv_id); - auto delete_on = [&](int port) { - httplib::Client cli(CHILD_ADDR, port); + auto owner = resolve_child_for_conv(models, conv_id); + if (owner.has_value()) { + httplib::Client cli(CHILD_ADDR, owner->port); cli.set_connection_timeout(0, STREAM_LOOKUP_TIMEOUT_MS * 1000); cli.set_read_timeout(0, STREAM_LOOKUP_TIMEOUT_MS * 1000); cli.set_write_timeout(0, STREAM_LOOKUP_TIMEOUT_MS * 1000); auto resp = cli.Delete(child_path.c_str()); (void) resp; // best effort, 404 and network errors are equivalent to no op - }; - if (!model_hint.empty()) { - auto direct = models.get_meta(model_hint); - if (direct.has_value() && direct->is_ready()) { - delete_on(direct->port); - res->status = 204; - res->content_type = "application/json"; - return res; - } - } - for (auto & meta : models.get_all_meta()) { - if (!meta.is_ready()) { - continue; - } - delete_on(meta.port); } + // drop the tracking entry, the session is being torn down + models.forget_conv_model(conv_id); res->status = 204; res->content_type = "application/json"; return res; diff --git a/tools/server/server-models.h b/tools/server/server-models.h index af9c54793a82..1a28a9e36334 100644 --- a/tools/server/server-models.h +++ b/tools/server/server-models.h @@ -11,7 +11,10 @@ #include #include #include +#include #include +#include +#include /** * state diagram: @@ -126,6 +129,13 @@ struct server_models { // if true, the next get_meta() will trigger a reload of model list bool need_reload = false; + // maps a conversation id to the model name that currently serves its stream session, so the + // resumable stream routes can go straight to the owning child instead of polling every one. + // populated when proxy_request forwards a POST carrying an X-Conversation-Id, the entry is + // best effort: if it is stale the child simply answers not found and the client recovers + std::mutex conv_model_mu; + std::unordered_map conv_model_map; + common_preset_context ctx_preset; common_params base_params; @@ -215,6 +225,12 @@ struct server_models { // state = ready -> payload = model_info (json), or {} if wakeup from sleeping // state = sleeping -> payload = {} void handle_child_state(const std::string & name, const std::string & raw_input); + + // conv_id -> model name tracking for the resumable stream routes (thread-safe) + // remember is called when a POST is routed to a child, lookup/forget by the stream routes + void remember_conv_model(const std::string & conv_id, const std::string & model); + std::optional lookup_conv_model(const std::string & conv_id); + void forget_conv_model(const std::string & conv_id); }; struct server_child { diff --git a/tools/server/server-stream.cpp b/tools/server/server-stream.cpp index 26ffa4b1c73b..44f9864299d7 100644 --- a/tools/server/server-stream.cpp +++ b/tools/server/server-stream.cpp @@ -342,19 +342,25 @@ void stream_pipe_producer::close() { // pipe is attached, so res_->next() runs to natural completion, only an explicit DELETE flips // is_cancelled and cuts it short if (done_ || session_->is_cancelled()) { + SRV_INF("stream_pipe close: skip drain (done=%d cancelled=%d) conv=%s\n", + done_ ? 1 : 0, session_->is_cancelled() ? 1 : 0, session_->conversation_id.c_str()); return; } + SRV_INF("stream_pipe close: draining conv=%s\n", session_->conversation_id.c_str()); + size_t drained = 0; std::string chunk; while (true) { chunk.clear(); bool has_next = res_->next(chunk); if (!chunk.empty()) { write(chunk.data(), chunk.size()); + drained += chunk.size(); } if (!has_next) { break; } } + SRV_INF("stream_pipe close: drain ended conv=%s bytes=%zu\n", session_->conversation_id.c_str(), drained); } std::shared_ptr stream_pipe_producer::create(stream_session_ptr session, @@ -528,11 +534,10 @@ server_http_context::handler_t make_stream_delete_handler() { }; } -void stream_session_attach_pipe(server_http_res & res, const std::map & headers) { +std::string stream_conv_id_from_headers(const std::map & headers) { // case-insensitive scan for x-conversation-id static constexpr char target[] = "x-conversation-id"; static constexpr size_t target_len = sizeof(target) - 1; - std::string conversation_id; for (const auto & [hk, hv] : headers) { if (hk.size() != target_len) continue; bool match = true; @@ -542,10 +547,16 @@ void stream_session_attach_pipe(server_http_res & res, const std::map & headers) { + std::string conversation_id = stream_conv_id_from_headers(headers); + SRV_INF("stream_session_attach_pipe: conv_id=%s (empty=%d)\n", + conversation_id.c_str(), conversation_id.empty() ? 1 : 0); if (conversation_id.empty()) { return; } diff --git a/tools/server/server-stream.h b/tools/server/server-stream.h index 4b8aa9528ae2..e782d772bd4d 100644 --- a/tools/server/server-stream.h +++ b/tools/server/server-stream.h @@ -201,6 +201,10 @@ server_http_context::handler_t make_stream_get_handler(); server_http_context::handler_t make_streams_lookup_handler(); server_http_context::handler_t make_stream_delete_handler(); +// extract the X-Conversation-Id header value (case-insensitive), empty when absent. exposed +// so the router can read the conv id off a forwarded POST to track which child serves it +std::string stream_conv_id_from_headers(const std::map & headers); + // inspect request headers for X-Conversation-Id and, when present, create or replace a // session on the global manager then attach a producer pipe to res. the pipe's stop_fn // calls res.stop() (overridden by server_res_generator to stop its reader). no-op when diff --git a/tools/ui/src/lib/services/chat.service.ts b/tools/ui/src/lib/services/chat.service.ts index 369e0d3988b8..4cbadcc1daf3 100644 --- a/tools/ui/src/lib/services/chat.service.ts +++ b/tools/ui/src/lib/services/chat.service.ts @@ -627,7 +627,7 @@ export class ChatService { let madeProgress = true; const encoder = new TextEncoder(); if (conversationId) { - saveStreamState(conversationId, 0); + saveStreamState(conversationId, 0, streamModel); } onConnectionState?.('streaming'); @@ -744,7 +744,7 @@ export class ChatService { if (conversationId) { const tailBytes = encoder.encode(chunk).byteLength; bytesParsed = segmentStartOffset + segmentBytesRead - tailBytes; - saveStreamState(conversationId, bytesParsed); + saveStreamState(conversationId, bytesParsed, streamModel); } for (const line of lines) { diff --git a/tools/ui/src/lib/services/stream-resume.service.ts b/tools/ui/src/lib/services/stream-resume.service.ts index e1a9e3a40946..40439393655c 100644 --- a/tools/ui/src/lib/services/stream-resume.service.ts +++ b/tools/ui/src/lib/services/stream-resume.service.ts @@ -13,6 +13,10 @@ import { getAuthHeaders } from '$lib/utils/api-headers'; interface ResumableStreamState { bytesReceived: number; updatedAt: number; + + // model frozen at POST time, lets a reload rebuild the exact conv::model identity the + // server keyed the session under. null when the POST carried no explicit model + model?: string | null; } const STORAGE_PREFIX = 'llamacpp.stream.resume.'; @@ -21,12 +25,17 @@ function storageKey(conversationId: string): string { return STORAGE_PREFIX + conversationId; } -export function saveStreamState(conversationId: string, bytesReceived: number): void { +export function saveStreamState( + conversationId: string, + bytesReceived: number, + model?: string | null +): void { if (!conversationId) return; try { const state: ResumableStreamState = { bytesReceived, - updatedAt: Date.now() + updatedAt: Date.now(), + model: model ?? null }; localStorage.setItem(storageKey(conversationId), JSON.stringify(state)); } catch { @@ -56,6 +65,21 @@ export function clearStreamState(conversationId: string): void { } } +/** + * Rebuild the stream identity for a resume. The model persisted at POST time wins, including a + * stored null which means the POST carried no explicit model so the identity stays the bare + * conv id. Only fall back to the caller supplied current model when nothing was persisted, e.g. + * a state written before the model field existed. + */ +export function resumeStreamIdentity( + conversationId: string, + state: ResumableStreamState | null, + fallbackModel: string | null +): string { + const model = state && state.model !== undefined ? state.model : fallbackModel; + return streamIdentity(conversationId, model); +} + /** * Reconnect to an interrupted stream for this conversation. Returns the fetch * Response so the existing SSE parser can drain it just like a fresh stream. diff --git a/tools/ui/src/lib/stores/chat.svelte.ts b/tools/ui/src/lib/stores/chat.svelte.ts index 62094f7ab123..d86bdeeff3b6 100644 --- a/tools/ui/src/lib/stores/chat.svelte.ts +++ b/tools/ui/src/lib/stores/chat.svelte.ts @@ -15,7 +15,11 @@ import { SvelteMap, SvelteSet } from 'svelte/reactivity'; import { DatabaseService } from '$lib/services/database.service'; import { ChatService } from '$lib/services/chat.service'; import { selectActiveStream } from '$lib/services/stream-discovery.service'; -import { getStreamState, clearStreamState } from '$lib/services/stream-resume.service'; +import { + getStreamState, + clearStreamState, + resumeStreamIdentity +} from '$lib/services/stream-resume.service'; import { streamIdentity } from '$lib/utils/stream-identity'; import { getAuthHeaders } from '$lib/utils/api-headers'; import { conversationsStore } from '$lib/stores/conversations.svelte'; @@ -440,8 +444,14 @@ class ChatStore { this.discoveringConvs.add(convId); try { - // primary path: ask the server which sessions exist for this conversation - const serverTarget = await this.probeServerStream(convId); + // the model is frozen at POST time, rebuild the exact conv::model identity from the + // persisted state so the lookup key matches what the server stored. null means a single + // model conv with no ::suffix, only guess from the dropdown with no persisted state + const localState = getStreamState(convId); + const streamId = resumeStreamIdentity(convId, localState, selectedModelName()); + + // primary path: ask the server which sessions exist for this identity + const serverTarget = await this.probeServerStream(streamId); if (serverTarget) { // pass the full server side identity (may carry a ::model suffix) so the GET routes // straight to the owning session, no probe or fan out @@ -450,13 +460,12 @@ class ChatStore { } // fallback: local state remembers an interrupted byte offset for this conv, the server may - // still have a live session matching that conv id (we just lost the bytes mid stream). try to - // attach with conv id only, the server probe inside attachServerStream tells us if it exists - const localState = getStreamState(convId); + // still have a live session matching that identity (we just lost the bytes mid stream). retry + // with the frozen identity, the server probe inside attachServerStream tells us if it exists if (!localState) { return; } - await this.attachServerStream(convId); + await this.attachServerStream(convId, streamId); // if attachServerStream failed (session gone, TTL expired), clear the local state to avoid retrying forever if (!this.chatStreamingStates.has(convId) && !this.chatLoadingStates.get(convId)) { clearStreamState(convId); @@ -646,12 +655,17 @@ class ChatStore { } return; } + // the lookup is keyed by the identity frozen at POST time (conv::model when the stream + // carried an explicit model), rebuild it per conv from the persisted state so a running + // session started with a model still matches. a single model conv stays a bare id, and + // the server response is mapped back to the bare id below for the sidebar set + const lookupIds = ids.map((id) => resumeStreamIdentity(id, getStreamState(id), null)); let sessions: ApiStreamSession[]; try { const resp = await fetch('./v1/streams/lookup', { method: 'POST', headers: { ...getAuthHeaders(), 'Content-Type': 'application/json' }, - body: JSON.stringify({ conversation_ids: ids }) + body: JSON.stringify({ conversation_ids: lookupIds }) }); if (!resp.ok) return; const body = (await resp.json()) as unknown; diff --git a/tools/ui/tests/unit/stream-resume.test.ts b/tools/ui/tests/unit/stream-resume.test.ts index 715d5d6c77d0..84a91af82ac8 100644 --- a/tools/ui/tests/unit/stream-resume.test.ts +++ b/tools/ui/tests/unit/stream-resume.test.ts @@ -23,7 +23,8 @@ beforeAll(() => { import { saveStreamState, getStreamState, - clearStreamState + clearStreamState, + resumeStreamIdentity } from '$lib/services/stream-resume.service'; describe('stream-resume.service', () => { @@ -75,4 +76,55 @@ describe('stream-resume.service', () => { localStorage.setItem('llamacpp.stream.resume.conv-a', '{not-json'); expect(getStreamState('conv-a')).toBeNull(); }); + + it('persists the model alongside the byte count', () => { + saveStreamState('conv-a', 10, 'model-x'); + expect(getStreamState('conv-a')!.model).toBe('model-x'); + }); + + it('stores a null model when none is provided', () => { + saveStreamState('conv-a', 10); + expect(getStreamState('conv-a')!.model).toBeNull(); + }); + + it('overwrites the model on a new save for the same conversation', () => { + saveStreamState('conv-a', 10, 'model-x'); + saveStreamState('conv-a', 20, 'model-y'); + expect(getStreamState('conv-a')!.model).toBe('model-y'); + }); + + describe('resumeStreamIdentity', () => { + it('appends the persisted model so the resume key matches the frozen POST identity', () => { + saveStreamState('conv-a', 10, 'model-x'); + expect(resumeStreamIdentity('conv-a', getStreamState('conv-a'), 'dropdown')).toBe( + 'conv-a::model-x' + ); + }); + + it('keeps the bare conv id when the persisted model is null', () => { + saveStreamState('conv-a', 10); + expect(resumeStreamIdentity('conv-a', getStreamState('conv-a'), 'dropdown')).toBe('conv-a'); + }); + + it('falls back to the current model only when no state is persisted', () => { + expect(resumeStreamIdentity('conv-a', null, 'dropdown')).toBe('conv-a::dropdown'); + }); + + it('ignores the fallback when a state exists, the persisted value is authoritative', () => { + saveStreamState('conv-a', 10, 'model-x'); + expect(resumeStreamIdentity('conv-a', getStreamState('conv-a'), 'dropdown')).toBe( + 'conv-a::model-x' + ); + }); + + it('falls back when a legacy state has no model field', () => { + localStorage.setItem( + 'llamacpp.stream.resume.conv-a', + JSON.stringify({ bytesReceived: 10, updatedAt: 1 }) + ); + expect(resumeStreamIdentity('conv-a', getStreamState('conv-a'), 'dropdown')).toBe( + 'conv-a::dropdown' + ); + }); + }); }); From eeb62ddc25e63aef4477fa33401fedff26bcb4d6 Mon Sep 17 00:00:00 2001 From: Pascal Date: Fri, 22 May 2026 08:28:45 +0200 Subject: [PATCH 26/39] ui: resolve continue target by id to stop cross-conversation flash on switch --- tools/ui/src/lib/stores/chat.svelte.ts | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/tools/ui/src/lib/stores/chat.svelte.ts b/tools/ui/src/lib/stores/chat.svelte.ts index d86bdeeff3b6..45a6188b148d 100644 --- a/tools/ui/src/lib/stores/chat.svelte.ts +++ b/tools/ui/src/lib/stores/chat.svelte.ts @@ -1829,7 +1829,11 @@ class ChatStore { const updateStreamingContent = (fullContent: string) => { this.setChatStreaming(msg.convId, fullContent, msg.id); - conversationsStore.updateMessageAtIndex(idx, { content: fullContent }); + // resolve the row by id on every write, switching to another conv mid continue makes + // this a no op instead of writing positionally into the now displayed conversation + conversationsStore.updateMessageAtIndex(conversationsStore.findMessageIndex(msg.id), { + content: fullContent + }); }; const abortController = this.getOrCreateAbortController(msg.convId); @@ -1855,7 +1859,7 @@ class ChatStore { hasReceivedContent = true; // mark streaming state so a stop mid-thinking can persist the partial reasoning this.setChatStreaming(msg.convId, originalContent + appendedContent, msg.id); - conversationsStore.updateMessageAtIndex(idx, { + conversationsStore.updateMessageAtIndex(conversationsStore.findMessageIndex(msg.id), { reasoningContent: originalReasoning + appendedReasoning }); this.setChatReasoning(msg.convId, true); @@ -1896,7 +1900,7 @@ class ChatStore { timings }); - conversationsStore.updateMessageAtIndex(idx, { + conversationsStore.updateMessageAtIndex(conversationsStore.findMessageIndex(msg.id), { content: fullContent, reasoningContent: fullReasoning, timestamp: Date.now(), @@ -1918,11 +1922,14 @@ class ChatStore { timestamp: Date.now() }); - conversationsStore.updateMessageAtIndex(idx, { - content: originalContent + appendedContent, - reasoningContent: originalReasoning + appendedReasoning || undefined, - timestamp: Date.now() - }); + conversationsStore.updateMessageAtIndex( + conversationsStore.findMessageIndex(msg.id), + { + content: originalContent + appendedContent, + reasoningContent: originalReasoning + appendedReasoning || undefined, + timestamp: Date.now() + } + ); } this.setChatLoading(msg.convId, false); @@ -1939,7 +1946,7 @@ class ChatStore { reasoningContent: originalReasoning + appendedReasoning || undefined, timestamp: Date.now() }); - conversationsStore.updateMessageAtIndex(idx, { + conversationsStore.updateMessageAtIndex(conversationsStore.findMessageIndex(msg.id), { content: originalContent + appendedContent, reasoningContent: originalReasoning + appendedReasoning || undefined, timestamp: Date.now() From c572968c76fba737fb28780b54bb2fdcf6ab1b9e Mon Sep 17 00:00:00 2001 From: Pascal Date: Fri, 22 May 2026 09:28:12 +0200 Subject: [PATCH 27/39] ui: skip stream resume when the abort is intentional --- tools/ui/src/lib/services/chat.service.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/ui/src/lib/services/chat.service.ts b/tools/ui/src/lib/services/chat.service.ts index 4cbadcc1daf3..48be630c2525 100644 --- a/tools/ui/src/lib/services/chat.service.ts +++ b/tools/ui/src/lib/services/chat.service.ts @@ -835,6 +835,8 @@ export class ChatService { const resumeResp = await resumeStream(conversationId, abortSignal, streamModel).catch( () => null ); + // an abort landing during the resume request is intentional, not a lost connection + if (abortSignal?.aborted) break; if (!resumeResp || resumeResp.status !== 200) { onConnectionState?.('lost'); onError?.(new Error('Stream connection lost and could not be resumed')); From 374ddbf1081f4d224e0ea87e9e3db266921e5d4c Mon Sep 17 00:00:00 2001 From: Pascal Date: Sat, 23 May 2026 13:02:23 +0200 Subject: [PATCH 28/39] server: move the conv id to model map into a self contained tracker Address review from ngxson: server_models held two mutexes side by side, the global one and a bare conv_model_mu guarding a loose map, which made the locking hard to follow. Wrap the map and its lock in a small conv_model_tracker struct that owns its mutex, one mutex per struct. The remember, lookup and forget methods move inline into the tracker, server_models exposes a single conv_models member and the routes call models.conv_models.lookup and friends. No behavior change, the map stays the single source of truth for routing resumable streams to a child. --- tools/server/server-models.cpp | 34 ++-------------------- tools/server/server-models.h | 52 ++++++++++++++++++++++++++-------- 2 files changed, 43 insertions(+), 43 deletions(-) diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index 00c85df6c38a..5f066f0ee759 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -1369,34 +1369,6 @@ void server_models::handle_child_state(const std::string & name, const std::stri } } -void server_models::remember_conv_model(const std::string & conv_id, const std::string & model) { - if (conv_id.empty() || model.empty()) { - return; - } - std::lock_guard lock(conv_model_mu); - conv_model_map[conv_id] = model; -} - -std::optional server_models::lookup_conv_model(const std::string & conv_id) { - if (conv_id.empty()) { - return std::nullopt; - } - std::lock_guard lock(conv_model_mu); - auto it = conv_model_map.find(conv_id); - if (it == conv_model_map.end()) { - return std::nullopt; - } - return it->second; -} - -void server_models::forget_conv_model(const std::string & conv_id) { - if (conv_id.empty()) { - return; - } - std::lock_guard lock(conv_model_mu); - conv_model_map.erase(conv_id); -} - // // server_child // @@ -1643,7 +1615,7 @@ static std::optional resolve_child_for_conv( if (conversation_id.empty()) { return std::nullopt; } - auto tracked = models.lookup_conv_model(conversation_id); + auto tracked = models.conv_models.lookup(conversation_id); if (!tracked.has_value()) { return std::nullopt; } @@ -1707,7 +1679,7 @@ void server_models_routes::init_routes() { // the GET and DELETE routes receive in their path, no parsing either side std::string conv_id = stream_conv_id_from_headers(req.headers); if (!conv_id.empty()) { - models.remember_conv_model(conv_id, name); + models.conv_models.remember(conv_id, name); } return models.proxy_request(req, method, name, true); // update last usage for POST request only }; @@ -2028,7 +2000,7 @@ void server_models_routes::init_routes() { (void) resp; // best effort, 404 and network errors are equivalent to no op } // drop the tracking entry, the session is being torn down - models.forget_conv_model(conv_id); + models.conv_models.forget(conv_id); res->status = 204; res->content_type = "application/json"; return res; diff --git a/tools/server/server-models.h b/tools/server/server-models.h index 1a28a9e36334..3109ecd780d2 100644 --- a/tools/server/server-models.h +++ b/tools/server/server-models.h @@ -129,12 +129,43 @@ struct server_models { // if true, the next get_meta() will trigger a reload of model list bool need_reload = false; - // maps a conversation id to the model name that currently serves its stream session, so the - // resumable stream routes can go straight to the owning child instead of polling every one. - // populated when proxy_request forwards a POST carrying an X-Conversation-Id, the entry is - // best effort: if it is stale the child simply answers not found and the client recovers - std::mutex conv_model_mu; - std::unordered_map conv_model_map; + // conv_id -> model name that currently serves its stream session, lets the resumable stream + // routes go straight to the owning child instead of polling every one. populated when + // proxy_request forwards a POST carrying an X-Conversation-Id. best effort: a stale entry just + // makes the child answer not found and the client recovers. owns its lock, one mutex per struct + struct conv_model_tracker { + void remember(const std::string & conv_id, const std::string & model) { + if (conv_id.empty() || model.empty()) { + return; + } + std::lock_guard lock(mu); + map[conv_id] = model; + } + + std::optional lookup(const std::string & conv_id) { + if (conv_id.empty()) { + return std::nullopt; + } + std::lock_guard lock(mu); + auto it = map.find(conv_id); + if (it == map.end()) { + return std::nullopt; + } + return it->second; + } + + void forget(const std::string & conv_id) { + if (conv_id.empty()) { + return; + } + std::lock_guard lock(mu); + map.erase(conv_id); + } + + private: + std::mutex mu; + std::unordered_map map; + }; common_preset_context ctx_preset; @@ -155,6 +186,9 @@ struct server_models { void notify_sse(const std::string & event, const std::string & model_id, const json & data = nullptr); public: + // conv_id -> model tracker for the resumable stream routes, owns its lock + conv_model_tracker conv_models; + server_models(const common_params & params, int argc, char ** argv); server_response sse; // for real-time updates via SSE endpoint @@ -225,12 +259,6 @@ struct server_models { // state = ready -> payload = model_info (json), or {} if wakeup from sleeping // state = sleeping -> payload = {} void handle_child_state(const std::string & name, const std::string & raw_input); - - // conv_id -> model name tracking for the resumable stream routes (thread-safe) - // remember is called when a POST is routed to a child, lookup/forget by the stream routes - void remember_conv_model(const std::string & conv_id, const std::string & model); - std::optional lookup_conv_model(const std::string & conv_id); - void forget_conv_model(const std::string & conv_id); }; struct server_child { From 8667f14d0d05419f78b5a5e6fc60134e10a9b2ed Mon Sep 17 00:00:00 2001 From: Pascal Date: Sat, 23 May 2026 20:59:13 +0200 Subject: [PATCH 29/39] ui: replace stream magic values with enums and shared constants Address review from allozaur: lift the inline literals around the resumable stream code into named symbols so the intent is explicit and reusable. --- .../ChatScreenStreamResumeStatus.svelte | 3 ++- tools/ui/src/lib/constants/api-endpoints.ts | 6 +++++ tools/ui/src/lib/constants/stream.ts | 6 +++++ tools/ui/src/lib/enums/chat.enums.ts | 9 ++++++++ tools/ui/src/lib/enums/index.ts | 1 + tools/ui/src/lib/services/chat.service.ts | 21 +++++++++-------- .../src/lib/services/stream-resume.service.ts | 3 ++- tools/ui/src/lib/stores/chat.svelte.ts | 13 +++++++---- tools/ui/src/lib/types/index.ts | 1 - tools/ui/src/lib/types/settings.d.ts | 11 +++------ tools/ui/src/lib/utils/abort.ts | 23 +++++++++++++------ 11 files changed, 65 insertions(+), 32 deletions(-) diff --git a/tools/ui/src/lib/components/app/chat/ChatScreen/ChatScreenStreamResumeStatus.svelte b/tools/ui/src/lib/components/app/chat/ChatScreen/ChatScreenStreamResumeStatus.svelte index 21fa0d9a97a9..b3abe4c66080 100644 --- a/tools/ui/src/lib/components/app/chat/ChatScreen/ChatScreenStreamResumeStatus.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatScreen/ChatScreenStreamResumeStatus.svelte @@ -1,11 +1,12 @@ -{#if state === 'resuming'} +{#if state === StreamConnectionState.RESUMING}
bytes flowing normally, resuming -> waiting on /v1/stream/:id reconnect, lost -> unrecoverable - streamConnectionState = $state('streaming'); + streamConnectionState = $state(StreamConnectionState.STREAMING); chatLoadingStates = new SvelteMap(); chatReasoningStates = new SvelteMap(); chatStreamingStates = new SvelteMap< diff --git a/tools/ui/src/lib/types/index.ts b/tools/ui/src/lib/types/index.ts index 26b5f6c98fb0..cbe0538be6e3 100644 --- a/tools/ui/src/lib/types/index.ts +++ b/tools/ui/src/lib/types/index.ts @@ -87,7 +87,6 @@ export type { SettingsConfigValue, SettingsFieldConfig, SettingsChatServiceOptions, - StreamConnectionState, SettingsConfigType, SettingsExportType, ParameterValue, diff --git a/tools/ui/src/lib/types/settings.d.ts b/tools/ui/src/lib/types/settings.d.ts index b181c6825418..9c787cd958f7 100644 --- a/tools/ui/src/lib/types/settings.d.ts +++ b/tools/ui/src/lib/types/settings.d.ts @@ -4,9 +4,10 @@ import type { OpenAIToolDefinition } from './mcp'; import type { DatabaseMessageExtra } from './database'; import type { ParameterSource, - ReasoningEffort, SyncableParameterType, - SettingsFieldType + SettingsFieldType, + StreamConnectionState, + ReasoningEffort } from '$lib/enums'; import type { Icon } from '@lucide/svelte'; import type { Component } from 'svelte'; @@ -122,12 +123,6 @@ export interface SettingsChatServiceOptions { onConnectionState?: (state: StreamConnectionState) => void; } -// Connection lifecycle for resumable streams -// streaming: data is flowing from the initial POST or a successful resume GET -// resuming : the server connection was lost and the client is attempting a resume -// lost : the stream is unrecoverable, the user must restart -export type StreamConnectionState = 'streaming' | 'resuming' | 'lost'; - export type SettingsConfigType = typeof SETTING_CONFIG_DEFAULT & { [key: string]: SettingsConfigValue; }; diff --git a/tools/ui/src/lib/utils/abort.ts b/tools/ui/src/lib/utils/abort.ts index e1771d92c948..67246b22f6fc 100644 --- a/tools/ui/src/lib/utils/abort.ts +++ b/tools/ui/src/lib/utils/abort.ts @@ -6,6 +6,17 @@ * when needed (e.g., user stops generation, navigates away, etc.). */ +// the standard DOMException name for a cancelled operation +const ABORT_ERROR_NAME = 'AbortError'; + +// browser specific TypeError messages emitted when a fetch reader is cut by page unload, +// navigation, or a transient network drop. functionally aborts, not actionable errors +const ABORT_LIKE_MESSAGE_PATTERNS = [ + /input stream/i, // Firefox: stream cut at unload + /network connection was lost/i, // Safari: transient network drop + /load failed/i // Safari: page navigation during fetch +]; + /** * Throws an AbortError if the signal is aborted. * Use this at the start of async operations to fail fast. @@ -23,7 +34,7 @@ */ export function throwIfAborted(signal?: AbortSignal): void { if (signal?.aborted) { - throw new DOMException('Operation was aborted', 'AbortError'); + throw new DOMException('Operation was aborted', ABORT_ERROR_NAME); } } @@ -48,11 +59,11 @@ export function throwIfAborted(signal?: AbortSignal): void { * ``` */ export function isAbortError(error: unknown): boolean { - if (error instanceof DOMException && error.name === 'AbortError') { + if (error instanceof DOMException && error.name === ABORT_ERROR_NAME) { return true; } if (error instanceof Error) { - if (error.name === 'AbortError') { + if (error.name === ABORT_ERROR_NAME) { return true; } // browser specific patterns emitted when a fetch reader is interrupted by page @@ -60,9 +71,7 @@ export function isAbortError(error: unknown): boolean { // not actionable application errors, so they should not surface as red console logs if (error instanceof TypeError) { const msg = error.message ?? ''; - if (/input stream/i.test(msg)) return true; // Firefox: stream cut at unload - if (/network connection was lost/i.test(msg)) return true; // Safari: transient network drop - if (/load failed/i.test(msg)) return true; // Safari: page navigation during fetch + if (ABORT_LIKE_MESSAGE_PATTERNS.some((re) => re.test(msg))) return true; } } return false; @@ -144,7 +153,7 @@ export async function withAbortSignal(promise: Promise, signal?: AbortSign return new Promise((resolve, reject) => { const abortHandler = () => { - reject(new DOMException('Operation was aborted', 'AbortError')); + reject(new DOMException('Operation was aborted', ABORT_ERROR_NAME)); }; signal.addEventListener('abort', abortHandler, { once: true }); From d5c85aacc783224e802d71d1fcc883bcade3ee8a Mon Sep 17 00:00:00 2001 From: Pascal Date: Sat, 23 May 2026 21:12:17 +0200 Subject: [PATCH 30/39] ui: fold the stream resume and discovery helpers into ChatService Address review from allozaur: drop the two standalone stream-*.service files. They were used only by the chat service and store, carried no shared state, and did not follow the static class pattern the other services use, so a separate abstraction was not warranted. Move the helpers onto ChatService as static methods. No behavior change, tests now exercise them through ChatService. --- tools/ui/src/lib/services/chat.service.ts | 134 ++++++++++++++++-- .../lib/services/stream-discovery.service.ts | 28 ---- .../src/lib/services/stream-resume.service.ts | 106 -------------- tools/ui/src/lib/stores/chat.svelte.ts | 18 +-- tools/ui/tests/unit/stream-discovery.test.ts | 28 ++-- tools/ui/tests/unit/stream-resume.test.ts | 87 ++++++------ 6 files changed, 186 insertions(+), 215 deletions(-) delete mode 100644 tools/ui/src/lib/services/stream-discovery.service.ts delete mode 100644 tools/ui/src/lib/services/stream-resume.service.ts diff --git a/tools/ui/src/lib/services/chat.service.ts b/tools/ui/src/lib/services/chat.service.ts index 09320ed5ada3..2bcfc70680c0 100644 --- a/tools/ui/src/lib/services/chat.service.ts +++ b/tools/ui/src/lib/services/chat.service.ts @@ -2,11 +2,6 @@ import { getAuthHeaders, getJsonHeaders } from '$lib/utils/api-headers'; import { formatAttachmentText } from '$lib/utils/formatters'; import { isAbortError } from '$lib/utils/abort'; import { streamIdentity } from '$lib/utils/stream-identity'; -import { - saveStreamState, - clearStreamState, - resumeStream -} from '$lib/services/stream-resume.service'; import { ATTACHMENT_LABEL_PDF_FILE, ATTACHMENT_LABEL_MCP_PROMPT, @@ -35,7 +30,8 @@ import { import type { ApiChatMessageContentPart, ApiChatMessageData, - ApiChatCompletionToolCall + ApiChatCompletionToolCall, + ApiStreamSession } from '$lib/types/api'; import type { AudioInputFormat, @@ -63,6 +59,21 @@ function getAudioInputFormat(mimeType: string): AudioInputFormat { return FileTypeAudio.MP3; } +interface ResumableStreamState { + bytesReceived: number; + updatedAt: number; + + // model frozen at POST time, lets a reload rebuild the exact conv::model identity the + // server keyed the session under. null when the POST carried no explicit model + model?: string | null; +} + +const STREAM_RESUME_STORAGE_PREFIX = 'llamacpp.stream.resume.'; + +function streamStorageKey(conversationId: string): string { + return STREAM_RESUME_STORAGE_PREFIX + conversationId; +} + export class ChatService { /** * @@ -505,6 +516,103 @@ export class ChatService { } } + /** + * Pick the running session to splice into when discoverActiveStream lists candidates for a + * conversation. Finalized sessions are not candidates: their final content was already written + * to the DB by the original onComplete handler, so attaching to them would replay a buffer that + * may not match what the DB holds. A continue session's buffer holds only the appended deltas, + * not the pre continue prefix, so replaying it as a fresh generation would erase the original. + * + * Among running sessions we tie break on the most recent started_at, which covers the case of + * multiple inferences left running on the same conversation. + */ + static selectActiveStream( + sessions: ApiStreamSession[] | null | undefined + ): ApiStreamSession | null { + if (!Array.isArray(sessions) || sessions.length === 0) { + return null; + } + const running = sessions.filter((s) => !s.is_done); + if (running.length === 0) { + return null; + } + return running.reduce((best, cur) => (cur.started_at > best.started_at ? cur : best)); + } + + // persist the running byte count and the frozen model for a conversation, a later visit + // resumes the SSE replay at the right offset under the same conv::model identity + static saveStreamState( + conversationId: string, + bytesReceived: number, + model?: string | null + ): void { + if (!conversationId) return; + try { + const state: ResumableStreamState = { + bytesReceived, + updatedAt: Date.now(), + model: model ?? null + }; + localStorage.setItem(streamStorageKey(conversationId), JSON.stringify(state)); + } catch { + // localStorage may be full or disabled, silently ignore + } + } + + static getStreamState(conversationId: string): ResumableStreamState | null { + if (!conversationId) return null; + try { + const raw = localStorage.getItem(streamStorageKey(conversationId)); + if (!raw) return null; + const parsed = JSON.parse(raw) as ResumableStreamState; + if (!parsed || typeof parsed.bytesReceived !== 'number') return null; + return parsed; + } catch { + return null; + } + } + + static clearStreamState(conversationId: string): void { + if (!conversationId) return; + try { + localStorage.removeItem(streamStorageKey(conversationId)); + } catch { + // nothing to do + } + } + + /** + * Rebuild the stream identity for a resume. The model persisted at POST time wins, including a + * stored null which means the POST carried no explicit model so the identity stays the bare conv + * id. Only fall back to the caller supplied current model when nothing was persisted. + */ + static resumeStreamIdentity( + conversationId: string, + state: ResumableStreamState | null, + fallbackModel: string | null + ): string { + const model = state && state.model !== undefined ? state.model : fallbackModel; + return streamIdentity(conversationId, model); + } + + /** + * Reconnect to an interrupted stream for this conversation. Returns the fetch Response so the + * existing SSE parser drains it like a fresh stream. The server returns 200 on success, 404 if + * no session exists for the conv_id, and 400 if the offset is below the dropped prefix. + */ + static async resumeStream( + conversationId: string, + signal?: AbortSignal, + model?: string | null + ): Promise { + if (!conversationId) return null; + const state = ChatService.getStreamState(conversationId); + const from = state?.bytesReceived ?? 0; + const id = streamIdentity(conversationId, model); + const url = `${API_STREAM.BASE}/${encodeURIComponent(id)}?from=${from}`; + return await fetch(url, { method: 'GET', signal, headers: getAuthHeaders() }); + } + static async preEncode( messages: ApiChatMessageData[] | (DatabaseMessage & { extra?: DatabaseMessageExtra[] })[], model?: string | null, @@ -628,7 +736,7 @@ export class ChatService { let madeProgress = true; const encoder = new TextEncoder(); if (conversationId) { - saveStreamState(conversationId, 0, streamModel); + ChatService.saveStreamState(conversationId, 0, streamModel); } onConnectionState?.(StreamConnectionState.STREAMING); @@ -745,7 +853,7 @@ export class ChatService { if (conversationId) { const tailBytes = encoder.encode(chunk).byteLength; bytesParsed = segmentStartOffset + segmentBytesRead - tailBytes; - saveStreamState(conversationId, bytesParsed, streamModel); + ChatService.saveStreamState(conversationId, bytesParsed, streamModel); } for (const line of lines) { @@ -833,9 +941,11 @@ export class ChatService { // it will be retransmitted from a clean line boundary. reuse the model the POST was // originally tagged with, the dropdown may have changed since but the server side // identity is frozen at POST time - const resumeResp = await resumeStream(conversationId, abortSignal, streamModel).catch( - () => null - ); + const resumeResp = await ChatService.resumeStream( + conversationId, + abortSignal, + streamModel + ).catch(() => null); // an abort landing during the resume request is intentional, not a lost connection if (abortSignal?.aborted) break; if (!resumeResp || resumeResp.status !== 200) { @@ -865,7 +975,7 @@ export class ChatService { finalizeOpenToolCallBatch(); if (conversationId) { - clearStreamState(conversationId); + ChatService.clearStreamState(conversationId); } const finalToolCalls = diff --git a/tools/ui/src/lib/services/stream-discovery.service.ts b/tools/ui/src/lib/services/stream-discovery.service.ts deleted file mode 100644 index 3792e4ad0122..000000000000 --- a/tools/ui/src/lib/services/stream-discovery.service.ts +++ /dev/null @@ -1,28 +0,0 @@ -import type { ApiStreamSession } from '$lib/types'; - -/** - * Pick the running session to splice into when discoverActiveStream lists candidates for - * a conversation. Finalized sessions are not candidates: their final content was already - * written to the DB by the original onComplete handler, so attaching to them would replay - * a buffer that may not match what the DB holds. In particular a continue session's buffer - * holds only the appended deltas, not the pre continue prefix, so replaying it as a fresh - * generation would erase the original assistant content. - * - * Among running sessions we tie break on the most recent started_at, which covers the - * pathological case of multiple inferences left running on the same conversation (eg user - * spawned two tabs). - * - * Returns null when no running session exists or the input is empty. - */ -export function selectActiveStream( - sessions: ApiStreamSession[] | null | undefined -): ApiStreamSession | null { - if (!Array.isArray(sessions) || sessions.length === 0) { - return null; - } - const running = sessions.filter((s) => !s.is_done); - if (running.length === 0) { - return null; - } - return running.reduce((best, cur) => (cur.started_at > best.started_at ? cur : best)); -} diff --git a/tools/ui/src/lib/services/stream-resume.service.ts b/tools/ui/src/lib/services/stream-resume.service.ts deleted file mode 100644 index f55ee969a7b3..000000000000 --- a/tools/ui/src/lib/services/stream-resume.service.ts +++ /dev/null @@ -1,106 +0,0 @@ -/** - * Stream resume persistence and reconnection helper. - * - * Tracks the running byte count for an in flight streaming generation per - * conversation_id, so a later visit can resume the SSE replay at the right - * offset. The conversation_id is the session identity end to end (server map, - * client localStorage, /v1/stream/ routes), no extra opaque token. - */ - -import { streamIdentity } from '$lib/utils/stream-identity'; -import { getAuthHeaders } from '$lib/utils/api-headers'; -import { API_STREAM } from '$lib/constants'; - -interface ResumableStreamState { - bytesReceived: number; - updatedAt: number; - - // model frozen at POST time, lets a reload rebuild the exact conv::model identity the - // server keyed the session under. null when the POST carried no explicit model - model?: string | null; -} - -const STORAGE_PREFIX = 'llamacpp.stream.resume.'; - -function storageKey(conversationId: string): string { - return STORAGE_PREFIX + conversationId; -} - -export function saveStreamState( - conversationId: string, - bytesReceived: number, - model?: string | null -): void { - if (!conversationId) return; - try { - const state: ResumableStreamState = { - bytesReceived, - updatedAt: Date.now(), - model: model ?? null - }; - localStorage.setItem(storageKey(conversationId), JSON.stringify(state)); - } catch { - // localStorage may be full or disabled, silently ignore - } -} - -export function getStreamState(conversationId: string): ResumableStreamState | null { - if (!conversationId) return null; - try { - const raw = localStorage.getItem(storageKey(conversationId)); - if (!raw) return null; - const parsed = JSON.parse(raw) as ResumableStreamState; - if (!parsed || typeof parsed.bytesReceived !== 'number') return null; - return parsed; - } catch { - return null; - } -} - -export function clearStreamState(conversationId: string): void { - if (!conversationId) return; - try { - localStorage.removeItem(storageKey(conversationId)); - } catch { - // nothing to do - } -} - -/** - * Rebuild the stream identity for a resume. The model persisted at POST time wins, including a - * stored null which means the POST carried no explicit model so the identity stays the bare - * conv id. Only fall back to the caller supplied current model when nothing was persisted, e.g. - * a state written before the model field existed. - */ -export function resumeStreamIdentity( - conversationId: string, - state: ResumableStreamState | null, - fallbackModel: string | null -): string { - const model = state && state.model !== undefined ? state.model : fallbackModel; - return streamIdentity(conversationId, model); -} - -/** - * Reconnect to an interrupted stream for this conversation. Returns the fetch - * Response so the existing SSE parser can drain it just like a fresh stream. - * The caller is expected to feed the running byte count back via - * saveStreamState as more data flows. - * - * The server returns 200 with text/event-stream on success, 404 if no session - * exists for the conv_id (already evicted or never created), and 400 if the - * requested offset is below the dropped prefix (buffer cap was hit and head - * bytes were lost). - */ -export async function resumeStream( - conversationId: string, - signal?: AbortSignal, - model?: string | null -): Promise { - if (!conversationId) return null; - const state = getStreamState(conversationId); - const from = state?.bytesReceived ?? 0; - const id = streamIdentity(conversationId, model); - const url = `${API_STREAM.BASE}/${encodeURIComponent(id)}?from=${from}`; - return await fetch(url, { method: 'GET', signal, headers: getAuthHeaders() }); -} diff --git a/tools/ui/src/lib/stores/chat.svelte.ts b/tools/ui/src/lib/stores/chat.svelte.ts index fc0314cf1415..94c02cfd56af 100644 --- a/tools/ui/src/lib/stores/chat.svelte.ts +++ b/tools/ui/src/lib/stores/chat.svelte.ts @@ -14,12 +14,6 @@ import { SvelteMap, SvelteSet } from 'svelte/reactivity'; import { DatabaseService } from '$lib/services/database.service'; import { ChatService } from '$lib/services/chat.service'; -import { selectActiveStream } from '$lib/services/stream-discovery.service'; -import { - getStreamState, - clearStreamState, - resumeStreamIdentity -} from '$lib/services/stream-resume.service'; import { streamIdentity } from '$lib/utils/stream-identity'; import { getAuthHeaders } from '$lib/utils/api-headers'; import { conversationsStore } from '$lib/stores/conversations.svelte'; @@ -228,7 +222,7 @@ class ChatStore { console.warn('probeServerStream JSON parse failed:', e); return null; } - return selectActiveStream(sessions); + return ChatService.selectActiveStream(sessions); } async attachServerStream(convId: string, streamId?: string): Promise { @@ -452,8 +446,8 @@ class ChatStore { // the model is frozen at POST time, rebuild the exact conv::model identity from the // persisted state so the lookup key matches what the server stored. null means a single // model conv with no ::suffix, only guess from the dropdown with no persisted state - const localState = getStreamState(convId); - const streamId = resumeStreamIdentity(convId, localState, selectedModelName()); + const localState = ChatService.getStreamState(convId); + const streamId = ChatService.resumeStreamIdentity(convId, localState, selectedModelName()); // primary path: ask the server which sessions exist for this identity const serverTarget = await this.probeServerStream(streamId); @@ -473,7 +467,7 @@ class ChatStore { await this.attachServerStream(convId, streamId); // if attachServerStream failed (session gone, TTL expired), clear the local state to avoid retrying forever if (!this.chatStreamingStates.has(convId) && !this.chatLoadingStates.get(convId)) { - clearStreamState(convId); + ChatService.clearStreamState(convId); } } finally { this.discoveringConvs.delete(convId); @@ -664,7 +658,9 @@ class ChatStore { // carried an explicit model), rebuild it per conv from the persisted state so a running // session started with a model still matches. a single model conv stays a bare id, and // the server response is mapped back to the bare id below for the sidebar set - const lookupIds = ids.map((id) => resumeStreamIdentity(id, getStreamState(id), null)); + const lookupIds = ids.map((id) => + ChatService.resumeStreamIdentity(id, ChatService.getStreamState(id), null) + ); let sessions: ApiStreamSession[]; try { const resp = await fetch('./v1/streams/lookup', { diff --git a/tools/ui/tests/unit/stream-discovery.test.ts b/tools/ui/tests/unit/stream-discovery.test.ts index 33027afdf08b..a428e5df8a39 100644 --- a/tools/ui/tests/unit/stream-discovery.test.ts +++ b/tools/ui/tests/unit/stream-discovery.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { selectActiveStream } from '$lib/services/stream-discovery.service'; +import { ChatService } from '$lib/services/chat.service'; import type { ApiStreamSession } from '$lib/types'; function makeSession(overrides: Partial): ApiStreamSession { @@ -15,51 +15,51 @@ function makeSession(overrides: Partial): ApiStreamSession { describe('selectActiveStream', () => { it('returns null on empty input', () => { - expect(selectActiveStream([])).toBeNull(); + expect(ChatService.selectActiveStream([])).toBeNull(); }); it('returns null on null or undefined input', () => { - expect(selectActiveStream(null)).toBeNull(); - expect(selectActiveStream(undefined)).toBeNull(); + expect(ChatService.selectActiveStream(null)).toBeNull(); + expect(ChatService.selectActiveStream(undefined)).toBeNull(); }); it('returns the single session when it is running', () => { const s = makeSession({ conversation_id: 'only', is_done: false, started_at: 42 }); - expect(selectActiveStream([s])).toBe(s); + expect(ChatService.selectActiveStream([s])).toBe(s); }); it('returns null when the single session is finalized', () => { const s = makeSession({ conversation_id: 'only', is_done: true, started_at: 42 }); - expect(selectActiveStream([s])).toBeNull(); + expect(ChatService.selectActiveStream([s])).toBeNull(); }); it('prefers a still running session over a finalized one regardless of started_at', () => { const finalized = makeSession({ conversation_id: 'old', is_done: true, started_at: 1000 }); const running = makeSession({ conversation_id: 'new', is_done: false, started_at: 10 }); - expect(selectActiveStream([finalized, running])?.conversation_id).toBe('new'); - expect(selectActiveStream([running, finalized])?.conversation_id).toBe('new'); + expect(ChatService.selectActiveStream([finalized, running])?.conversation_id).toBe('new'); + expect(ChatService.selectActiveStream([running, finalized])?.conversation_id).toBe('new'); }); it('among running sessions, picks the most recently started one', () => { const a = makeSession({ conversation_id: 'a', is_done: false, started_at: 100 }); const b = makeSession({ conversation_id: 'b', is_done: false, started_at: 200 }); const c = makeSession({ conversation_id: 'c', is_done: false, started_at: 150 }); - expect(selectActiveStream([a, b, c])?.conversation_id).toBe('b'); - expect(selectActiveStream([c, a, b])?.conversation_id).toBe('b'); + expect(ChatService.selectActiveStream([a, b, c])?.conversation_id).toBe('b'); + expect(ChatService.selectActiveStream([c, a, b])?.conversation_id).toBe('b'); }); it('returns null when all sessions are finalized, the DB already holds the content', () => { const a = makeSession({ conversation_id: 'a', is_done: true, started_at: 10 }); const b = makeSession({ conversation_id: 'b', is_done: true, started_at: 30 }); const c = makeSession({ conversation_id: 'c', is_done: true, started_at: 20 }); - expect(selectActiveStream([a, b, c])).toBeNull(); + expect(ChatService.selectActiveStream([a, b, c])).toBeNull(); }); it('keeps the first match on ties when both are running with identical started_at', () => { // reduce visits left to right, the initial accumulator stays unless a strictly greater value appears const a = makeSession({ conversation_id: 'first', is_done: false, started_at: 50 }); const b = makeSession({ conversation_id: 'second', is_done: false, started_at: 50 }); - expect(selectActiveStream([a, b])?.conversation_id).toBe('first'); + expect(ChatService.selectActiveStream([a, b])?.conversation_id).toBe('first'); }); it('handles a typical realistic mix: two finalized old, one freshly running, one freshly finalized', () => { @@ -67,6 +67,8 @@ describe('selectActiveStream', () => { const old2 = makeSession({ conversation_id: 'old2', is_done: true, started_at: 200 }); const freshFin = makeSession({ conversation_id: 'freshFin', is_done: true, started_at: 500 }); const running = makeSession({ conversation_id: 'running', is_done: false, started_at: 400 }); - expect(selectActiveStream([old1, old2, freshFin, running])?.conversation_id).toBe('running'); + expect(ChatService.selectActiveStream([old1, old2, freshFin, running])?.conversation_id).toBe( + 'running' + ); }); }); diff --git a/tools/ui/tests/unit/stream-resume.test.ts b/tools/ui/tests/unit/stream-resume.test.ts index 84a91af82ac8..60bd19867f49 100644 --- a/tools/ui/tests/unit/stream-resume.test.ts +++ b/tools/ui/tests/unit/stream-resume.test.ts @@ -20,14 +20,9 @@ beforeAll(() => { (globalThis as unknown as { localStorage: Storage }).localStorage = polyfill; }); -import { - saveStreamState, - getStreamState, - clearStreamState, - resumeStreamIdentity -} from '$lib/services/stream-resume.service'; - -describe('stream-resume.service', () => { +import { ChatService } from '$lib/services/chat.service'; + +describe('ChatService stream resume', () => { beforeEach(() => { localStorage.clear(); }); @@ -36,85 +31,87 @@ describe('stream-resume.service', () => { }); it('returns null when no state exists for the conversation', () => { - expect(getStreamState('conv-a')).toBeNull(); + expect(ChatService.getStreamState('conv-a')).toBeNull(); }); it('saves and reads back the byte count', () => { - saveStreamState('conv-a', 4242); - const got = getStreamState('conv-a'); + ChatService.saveStreamState('conv-a', 4242); + const got = ChatService.getStreamState('conv-a'); expect(got).not.toBeNull(); expect(got!.bytesReceived).toBe(4242); expect(typeof got!.updatedAt).toBe('number'); }); it('overwrites the previous byte count on a new save for the same conversation', () => { - saveStreamState('conv-a', 100); - saveStreamState('conv-a', 200); - const got = getStreamState('conv-a'); + ChatService.saveStreamState('conv-a', 100); + ChatService.saveStreamState('conv-a', 200); + const got = ChatService.getStreamState('conv-a'); expect(got!.bytesReceived).toBe(200); }); it('keeps states for distinct conversations isolated', () => { - saveStreamState('conv-a', 10); - saveStreamState('conv-b', 20); - expect(getStreamState('conv-a')!.bytesReceived).toBe(10); - expect(getStreamState('conv-b')!.bytesReceived).toBe(20); + ChatService.saveStreamState('conv-a', 10); + ChatService.saveStreamState('conv-b', 20); + expect(ChatService.getStreamState('conv-a')!.bytesReceived).toBe(10); + expect(ChatService.getStreamState('conv-b')!.bytesReceived).toBe(20); }); it('clears the state for a given conversation', () => { - saveStreamState('conv-a', 10); - clearStreamState('conv-a'); - expect(getStreamState('conv-a')).toBeNull(); + ChatService.saveStreamState('conv-a', 10); + ChatService.clearStreamState('conv-a'); + expect(ChatService.getStreamState('conv-a')).toBeNull(); }); it('ignores empty conversation id on save', () => { - saveStreamState('', 1); - expect(getStreamState('')).toBeNull(); + ChatService.saveStreamState('', 1); + expect(ChatService.getStreamState('')).toBeNull(); }); it('returns null on corrupted storage payload', () => { localStorage.setItem('llamacpp.stream.resume.conv-a', '{not-json'); - expect(getStreamState('conv-a')).toBeNull(); + expect(ChatService.getStreamState('conv-a')).toBeNull(); }); it('persists the model alongside the byte count', () => { - saveStreamState('conv-a', 10, 'model-x'); - expect(getStreamState('conv-a')!.model).toBe('model-x'); + ChatService.saveStreamState('conv-a', 10, 'model-x'); + expect(ChatService.getStreamState('conv-a')!.model).toBe('model-x'); }); it('stores a null model when none is provided', () => { - saveStreamState('conv-a', 10); - expect(getStreamState('conv-a')!.model).toBeNull(); + ChatService.saveStreamState('conv-a', 10); + expect(ChatService.getStreamState('conv-a')!.model).toBeNull(); }); it('overwrites the model on a new save for the same conversation', () => { - saveStreamState('conv-a', 10, 'model-x'); - saveStreamState('conv-a', 20, 'model-y'); - expect(getStreamState('conv-a')!.model).toBe('model-y'); + ChatService.saveStreamState('conv-a', 10, 'model-x'); + ChatService.saveStreamState('conv-a', 20, 'model-y'); + expect(ChatService.getStreamState('conv-a')!.model).toBe('model-y'); }); describe('resumeStreamIdentity', () => { it('appends the persisted model so the resume key matches the frozen POST identity', () => { - saveStreamState('conv-a', 10, 'model-x'); - expect(resumeStreamIdentity('conv-a', getStreamState('conv-a'), 'dropdown')).toBe( - 'conv-a::model-x' - ); + ChatService.saveStreamState('conv-a', 10, 'model-x'); + expect( + ChatService.resumeStreamIdentity('conv-a', ChatService.getStreamState('conv-a'), 'dropdown') + ).toBe('conv-a::model-x'); }); it('keeps the bare conv id when the persisted model is null', () => { - saveStreamState('conv-a', 10); - expect(resumeStreamIdentity('conv-a', getStreamState('conv-a'), 'dropdown')).toBe('conv-a'); + ChatService.saveStreamState('conv-a', 10); + expect( + ChatService.resumeStreamIdentity('conv-a', ChatService.getStreamState('conv-a'), 'dropdown') + ).toBe('conv-a'); }); it('falls back to the current model only when no state is persisted', () => { - expect(resumeStreamIdentity('conv-a', null, 'dropdown')).toBe('conv-a::dropdown'); + expect(ChatService.resumeStreamIdentity('conv-a', null, 'dropdown')).toBe('conv-a::dropdown'); }); it('ignores the fallback when a state exists, the persisted value is authoritative', () => { - saveStreamState('conv-a', 10, 'model-x'); - expect(resumeStreamIdentity('conv-a', getStreamState('conv-a'), 'dropdown')).toBe( - 'conv-a::model-x' - ); + ChatService.saveStreamState('conv-a', 10, 'model-x'); + expect( + ChatService.resumeStreamIdentity('conv-a', ChatService.getStreamState('conv-a'), 'dropdown') + ).toBe('conv-a::model-x'); }); it('falls back when a legacy state has no model field', () => { @@ -122,9 +119,9 @@ describe('stream-resume.service', () => { 'llamacpp.stream.resume.conv-a', JSON.stringify({ bytesReceived: 10, updatedAt: 1 }) ); - expect(resumeStreamIdentity('conv-a', getStreamState('conv-a'), 'dropdown')).toBe( - 'conv-a::dropdown' - ); + expect( + ChatService.resumeStreamIdentity('conv-a', ChatService.getStreamState('conv-a'), 'dropdown') + ).toBe('conv-a::dropdown'); }); }); }); From 7962c303bbcfa6625e5a3df052254a63971adf37 Mon Sep 17 00:00:00 2001 From: Pascal Date: Tue, 26 May 2026 19:06:33 +0200 Subject: [PATCH 31/39] docs: document the SSE replay buffer in server README-dev Add the resumable streaming section, list stream_session_manager in the backend component inventory, and link PR 23226 in the related PRs. --- tools/server/README-dev.md | 54 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/tools/server/README-dev.md b/tools/server/README-dev.md index 5959745e473a..dfc9004de549 100644 --- a/tools/server/README-dev.md +++ b/tools/server/README-dev.md @@ -57,6 +57,7 @@ The core architecture consists of the following components: - `server_tokens`: Unified representation of token sequences (supports both text and multimodal tokens); used by `server_task` and `server_slot`. - `server_prompt_checkpoint`: For recurrent (e.g., RWKV) and SWA models, stores snapshots of KV cache state. Enables reuse when subsequent requests share the same prompt prefix, saving redundant computation. - `server_models`: Standalone component for managing multiple backend instances (used in router mode). It is completely independent of `server_context`. +- `stream_session_manager`: Process wide owner of resumable SSE stream sessions (`g_stream_sessions`), keyed by conversation id. Backs the replay buffer that lets a client reattach to a generation after an HTTP disconnect. See the "Resumable streaming" section below. ```mermaid graph TD @@ -117,6 +118,58 @@ Here is an example trace of an API request for text completion: - As the response is stateless, `server_res_generator` calls `response->update()` to update the response with the current state. - `server_res_generator` then calls `response->to_json()` and passes the response to the HTTP layer. +### Resumable streaming (SSE replay buffer) + +By default a streaming generation is bound to its HTTP socket: when the socket drops (refresh, tab close, mobile background, transient network) the generation aborts and the live stream is lost. This feature keeps the generation running server side and lets a client reattach. + +It is opt in via the `X-Conversation-Id` header on `POST /v1/chat/completions`. Without the header the OAI strict path is unchanged. The conversation id is the only identity end to end (server map key, client localStorage key, route path), with an optional `::model` suffix for direct routing in router mode. + +The feature lives entirely in `server-stream.{h,cpp}` and rests on three types: + +- `stream_session`: a bounded ring buffer (4 MiB cap, oldest bytes drop first) plus a condvar. `append` pushes raw SSE bytes, `read_from` drains from any offset and blocks for live bytes or finalize, `finalize` wakes readers, `cancel` stops the producer. One conv maps to at most one live session. +- `stream_session_manager` (`g_stream_sessions`): owns all sessions keyed by conv id, enforces the one conv one session invariant via `create_or_replace`, and runs a GC thread that drops completed sessions past their TTL. +- `stream_pipe_producer` / `stream_pipe_consumer`: the write and read ends. The producer owns the session lifetime and finalizes it on destruction; the consumer is read only and never finalizes, so a reader detaching cannot kill a running generation. + +Producer side: `server_res_generator` attaches a producer pipe when the header is present. The HTTP content provider mirrors every chunk into the ring before writing it to the socket. While a pipe is attached, `stream_aware_should_stop` ignores peer disconnect, so a dropped socket does not stop generation: only an explicit `DELETE` does. When the peer leaves early, `on_complete` calls `close()`, which drains the rest of the generation into the ring on the http worker. + +Lifetime safety: the producer pipe holds a shared `alive` flag also captured by the session cancel hook. `~server_res_generator` calls `cleanup()` to clear that hook while the reader is still alive, so a `cancel` arriving during teardown can never call `stop()` on a freed response. This ordering is the most fragile part of the feature: finalizing or destroying the producer before `cleanup()` runs reintroduces a use after free. + +Consumer side: `GET /v1/stream/?from=N` opens a `text/event-stream` that replays buffered bytes from offset `N` and blocks for live bytes, so the browser reattaches like a fresh EventSource. An offset below the dropped prefix returns 400. + +Routes: + +- `GET /v1/stream/:conv_id?from=N`: replay or live reattach. +- `POST /v1/streams/lookup` with `{"conversation_ids": [...]}`: returns session status only for ids the caller already owns. There is no listing route, so live sessions cannot be enumerated (an earlier `GET /v1/streams` was removed for exactly this reason). +- `DELETE /v1/stream/:conv_id`: explicit Stop, idempotent (`evict_and_cancel`). + +Router mode binds the same paths to proxy handlers. A `conv_id -> child` map (`conv_models`), populated when a POST is routed, resolves the owning child in one lookup with no polling. The lookup groups ids per child; GET and DELETE proxy straight to the owner. This loopback REST hop is expected to move to a websocket IPC later, swapping only the transport. + +Lifecycle: `g_stream_sessions.start_gc()` runs in main after common init, `stop_gc()` runs first in `clean_up()` and finalizes every live session so no reader hangs. Reader blocking and the post drop drain both run on httplib worker threads, which block on a condvar rather than spin. + +| Constant | Value | Role | +| --- | --- | --- | +| `STREAM_SESSION_TTL_SECONDS` | 300 | retention of a completed session before GC | +| `STREAM_SESSION_MAX_BYTES` | 4 MiB | ring cap per session | +| `STREAM_SESSION_GC_INTERVAL_SECONDS` | 60 | GC tick | +| `STREAM_READ_WAKE_INTERVAL_MS` | 200 | read_from wake to recheck should_stop | +| `STREAM_LOOKUP_TIMEOUT_MS` | 250 | router to child loopback budget | + +```mermaid +graph TD + Client -- "POST + X-Conversation-Id" --> RG[server_res_generator] + RG -- attach --> Prod[stream_pipe_producer] + Prod -- "write, drain on peer drop" --> Sess + subgraph g_stream_sessions + Sess[stream_session: ring buffer, 4 MiB] + GC[GC thread] -- drop after TTL --> Sess + end + Sess -- read_from offset --> Cons[stream_pipe_consumer] + Cons -- "GET /v1/stream/:id?from=N" --> Client + DEL[DELETE /v1/stream/:id] -- evict_and_cancel --> Sess +``` + +The diagram shows the buffer touch points. The live wire (chunks streamed to the original client during a normal generation) is the producer's default output, described under "Producer side" above. + ### Testing `llama-server` includes an automated test suite based on `pytest`. @@ -223,6 +276,7 @@ The flow for downloading a new model: - Speculative decoding: https://github.com/ggml-org/llama.cpp/pull/17808 and rework in https://github.com/ggml-org/llama.cpp/pull/17808 - INI presets: https://github.com/ggml-org/llama.cpp/pull/17859 (+ refactoring: https://github.com/ggml-org/llama.cpp/pull/18169) - Sleeping mode: https://github.com/ggml-org/llama.cpp/pull/18228 +- Resumable streaming (SSE replay buffer): https://github.com/ggml-org/llama.cpp/pull/23226 From 99f8ae01597579b0f27f486547c629a92a0c1d7d Mon Sep 17 00:00:00 2001 From: Pascal Date: Tue, 2 Jun 2026 16:05:25 +0200 Subject: [PATCH 32/39] ui: align attachServerStream call with onCompletionId param in handleStreamResponse --- tools/ui/src/lib/stores/chat.svelte.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/ui/src/lib/stores/chat.svelte.ts b/tools/ui/src/lib/stores/chat.svelte.ts index 94c02cfd56af..83af31886d4d 100644 --- a/tools/ui/src/lib/stores/chat.svelte.ts +++ b/tools/ui/src/lib/stores/chat.svelte.ts @@ -417,6 +417,7 @@ class ChatStore { undefined, undefined, undefined, + undefined, convId, abortController.signal, (connState: StreamConnectionState) => { From 2f4be74e44d221e5daf5a957b3e604b431a8daf0 Mon Sep 17 00:00:00 2001 From: Pascal Date: Mon, 8 Jun 2026 14:42:23 +0200 Subject: [PATCH 33/39] server-http: rename del_ to del to match get and post --- tools/server/server-http.cpp | 17 ----------------- tools/server/server-http.h | 1 - tools/server/server.cpp | 2 +- 3 files changed, 1 insertion(+), 19 deletions(-) diff --git a/tools/server/server-http.cpp b/tools/server/server-http.cpp index 13509cb6de1e..532562e6b441 100644 --- a/tools/server/server-http.cpp +++ b/tools/server/server-http.cpp @@ -590,23 +590,6 @@ void server_http_context::get(const std::string & path, const server_http_contex }); } -void server_http_context::del_(const std::string & path, const server_http_context::handler_t & handler) const { - handlers.emplace(path, handler); - pimpl->srv->Delete(path_prefix + path, [handler](const httplib::Request & req, httplib::Response & res) { - server_http_req_ptr request = std::make_unique(server_http_req{ - get_params(req), - get_headers(req), - req.path, - build_query_string(req), - req.body, - {}, - req.is_connection_closed - }); - server_http_res_ptr response = handler(*request); - process_handler_response(std::move(request), response, res); - }); -} - void server_http_context::post(const std::string & path, const server_http_context::handler_t & handler) const { handlers.emplace(path, handler); pimpl->srv->Post(path_prefix + path, [handler](const httplib::Request & req, httplib::Response & res) { diff --git a/tools/server/server-http.h b/tools/server/server-http.h index 335219e234af..455a0170619f 100644 --- a/tools/server/server-http.h +++ b/tools/server/server-http.h @@ -98,7 +98,6 @@ struct server_http_context { void get(const std::string & path, const handler_t & handler) const; void post(const std::string & path, const handler_t & handler) const; void del(const std::string & path, const handler_t & handler) const; - void del_(const std::string & path, const handler_t & handler) const; // Register the Google Cloud Platform (Vertex AI) compat (AIP_PREDICT_ROUTE env var, or /predict) // Must be called AFTER all other API routes are registered diff --git a/tools/server/server.cpp b/tools/server/server.cpp index 67bb0eff75f2..1e1c121ffbdf 100644 --- a/tools/server/server.cpp +++ b/tools/server/server.cpp @@ -264,7 +264,7 @@ int llama_server(int argc, char ** argv) { // you already own (the WebUI passes the convs visible in its sidebar). the server never // lists ids it has not been asked about, so a random caller cannot enumerate live sessions ctx_http.post("/v1/streams/lookup", ex_wrapper(streams_lookup_h)); - ctx_http.del_("/v1/stream/:conv_id", ex_wrapper(stream_delete_h)); + ctx_http.del ("/v1/stream/:conv_id", ex_wrapper(stream_delete_h)); // Google Cloud Platform (Vertex AI) compat ctx_http.register_gcp_compat(); From 5a3a9817847888c2c3a810b1b71f0d662ae160e0 Mon Sep 17 00:00:00 2001 From: Pascal Date: Thu, 18 Jun 2026 19:46:41 +0200 Subject: [PATCH 34/39] ui: address review feedback from allozaur --- tools/ui/src/lib/constants/storage.ts | 3 +++ tools/ui/src/lib/services/chat.service.ts | 5 ++--- tools/ui/tests/unit/stream-resume.test.ts | 5 +++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/tools/ui/src/lib/constants/storage.ts b/tools/ui/src/lib/constants/storage.ts index 8d425b96b762..0180a76fb67c 100644 --- a/tools/ui/src/lib/constants/storage.ts +++ b/tools/ui/src/lib/constants/storage.ts @@ -26,6 +26,9 @@ export const THINKING_ENABLED_DEFAULT_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.th export const REASONING_EFFORT_DEFAULT_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.reasoningEffortDefault`; export const USER_OVERRIDES_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.userOverrides`; +/** Key prefix for per-conversation resumable stream state, conversationId is appended */ +export const STREAM_RESUME_LOCALSTORAGE_KEY_PREFIX = `${STORAGE_APP_NAME}.streamResume.`; + // Deprecated old key names (kept for backward compat while users migrate) /** @deprecated Use {@link ALWAYS_ALLOWED_TOOLS_LOCALSTORAGE_KEY} instead */ export const DEPRECATED_ALWAYS_ALLOWED_TOOLS_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME_DEPRECATED}.alwaysAllowedTools`; diff --git a/tools/ui/src/lib/services/chat.service.ts b/tools/ui/src/lib/services/chat.service.ts index 2bcfc70680c0..704aea3d4913 100644 --- a/tools/ui/src/lib/services/chat.service.ts +++ b/tools/ui/src/lib/services/chat.service.ts @@ -16,6 +16,7 @@ import { SSE_DATA_PREFIX, SSE_DONE_MARKER, STREAM_VISIBILITY_KICK_MS, + STREAM_RESUME_LOCALSTORAGE_KEY_PREFIX, API_STREAM } from '$lib/constants'; import { @@ -68,10 +69,8 @@ interface ResumableStreamState { model?: string | null; } -const STREAM_RESUME_STORAGE_PREFIX = 'llamacpp.stream.resume.'; - function streamStorageKey(conversationId: string): string { - return STREAM_RESUME_STORAGE_PREFIX + conversationId; + return STREAM_RESUME_LOCALSTORAGE_KEY_PREFIX + conversationId; } export class ChatService { diff --git a/tools/ui/tests/unit/stream-resume.test.ts b/tools/ui/tests/unit/stream-resume.test.ts index 60bd19867f49..f52f2cabd53d 100644 --- a/tools/ui/tests/unit/stream-resume.test.ts +++ b/tools/ui/tests/unit/stream-resume.test.ts @@ -21,6 +21,7 @@ beforeAll(() => { }); import { ChatService } from '$lib/services/chat.service'; +import { STREAM_RESUME_LOCALSTORAGE_KEY_PREFIX } from '$lib/constants'; describe('ChatService stream resume', () => { beforeEach(() => { @@ -68,7 +69,7 @@ describe('ChatService stream resume', () => { }); it('returns null on corrupted storage payload', () => { - localStorage.setItem('llamacpp.stream.resume.conv-a', '{not-json'); + localStorage.setItem(`${STREAM_RESUME_LOCALSTORAGE_KEY_PREFIX}conv-a`, '{not-json'); expect(ChatService.getStreamState('conv-a')).toBeNull(); }); @@ -116,7 +117,7 @@ describe('ChatService stream resume', () => { it('falls back when a legacy state has no model field', () => { localStorage.setItem( - 'llamacpp.stream.resume.conv-a', + `${STREAM_RESUME_LOCALSTORAGE_KEY_PREFIX}conv-a`, JSON.stringify({ bytesReceived: 10, updatedAt: 1 }) ); expect( From 9ddbaf8dcc65ca777d238278f3a0e5939bd0c74e Mon Sep 17 00:00:00 2001 From: Pascal Date: Thu, 25 Jun 2026 14:07:50 +0200 Subject: [PATCH 35/39] ui: drop duplicate SSE constants, keep sse.ts canonical --- tools/ui/src/lib/constants/stream.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tools/ui/src/lib/constants/stream.ts b/tools/ui/src/lib/constants/stream.ts index 632c797f45f7..3d042451fc69 100644 --- a/tools/ui/src/lib/constants/stream.ts +++ b/tools/ui/src/lib/constants/stream.ts @@ -1,9 +1,3 @@ // grace window after a visibilitychange before we kick a reader whose socket likely died // while the tab was hidden. covers brief background pauses without thrashing live streams export const STREAM_VISIBILITY_KICK_MS = 1000; - -// marks the end of an SSE completion stream, the server sends it as the final data payload -export const SSE_DONE_MARKER = '[DONE]'; - -// prefix of an SSE data line, the payload starts right after it -export const SSE_DATA_PREFIX = 'data: '; From 6eaafab316d5bf81314f7601c69f9fb64d9221ed Mon Sep 17 00:00:00 2001 From: Pascal Date: Thu, 25 Jun 2026 15:04:43 +0200 Subject: [PATCH 36/39] ui: use svelte:document for the visibilitychange listener address review from allozaur: replace the manual document.addEventListener in onMount with a declarative . svelte handles attach, detach and SSR, so the typeof document guard and the onMount cleanup go away. onMount keeps only the first load snapshot. --- tools/ui/src/routes/+layout.svelte | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/tools/ui/src/routes/+layout.svelte b/tools/ui/src/routes/+layout.svelte index 0bedc725db0c..38848786e998 100644 --- a/tools/ui/src/routes/+layout.svelte +++ b/tools/ui/src/routes/+layout.svelte @@ -155,20 +155,18 @@ onMount(() => { updateFavicon(); - // global snapshot of backend running streams. populates the sidebar spinners so the user - // sees at a glance every conv that has a live inference, even ones not yet opened. snapshot - // only, no polling: refresh happens on mount and on visibilitychange via the effect below + // snapshot of every backend running stream on first load, populates the sidebar spinners + // so the user sees each conv that has a live inference, even ones not opened yet void chatStore.syncRemoteRunningStreams(); - - if (typeof document === 'undefined') return; - const onVisibility = () => { - if (document.visibilityState !== 'visible') return; - void chatStore.syncRemoteRunningStreams(); - }; - document.addEventListener('visibilitychange', onVisibility); - return () => document.removeEventListener('visibilitychange', onVisibility); }); + // refresh that snapshot when the tab returns to the foreground, a stream may have advanced + // or ended while it was hidden. snapshot only, no polling + function handleVisibilityChange() { + if (document.visibilityState !== 'visible') return; + void chatStore.syncRemoteRunningStreams(); + } + $effect(() => { void theme.isSystemDark; @@ -293,6 +291,7 @@ +
From ac02404e7393ffdfc106989f25b9ed7d727ced8e Mon Sep 17 00:00:00 2001 From: Pascal Date: Thu, 25 Jun 2026 15:59:22 +0200 Subject: [PATCH 37/39] server: trim redundant stream drain comments Address review from ngxson --- tools/server/server-http.cpp | 8 ++------ tools/server/server-stream.cpp | 7 ++----- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/tools/server/server-http.cpp b/tools/server/server-http.cpp index 532562e6b441..82f34edac069 100644 --- a/tools/server/server-http.cpp +++ b/tools/server/server-http.cpp @@ -538,8 +538,7 @@ static void process_handler_response(server_http_req_ptr && request, server_http response->spipe->write(chunk.data(), chunk.size()); } if (!sink.write(chunk.data(), chunk.size())) { - // peer is gone, stop the wire path here. when a pipe is attached on_complete - // drains the rest of the generation into the ring buffer + // peer is gone, stop the wire path here return false; } SRV_DBG("http: streamed chunk: %s\n", chunk.c_str()); @@ -555,10 +554,7 @@ static void process_handler_response(server_http_req_ptr && request, server_http return has_next; }; const auto on_complete = [request = q_ptr, response = r_ptr](bool) mutable { - // the peer may have dropped before the producer finished. when a pipe is attached, drain - // the rest of the generation into the ring buffer here, on this http worker. httplib - // runs a large dynamic pool and the worker blocks in next() on a condvar rather than - // burning cpu, so holding it until the generation ends is fine. see stream_pipe_producer::close + // on a dropped peer, close() drains the rest of the generation into the ring buffer if (response->spipe) { response->spipe->close(); } diff --git a/tools/server/server-stream.cpp b/tools/server/server-stream.cpp index 44f9864299d7..d1cdfa6b1832 100644 --- a/tools/server/server-stream.cpp +++ b/tools/server/server-stream.cpp @@ -336,11 +336,8 @@ void stream_pipe_producer::done() { } void stream_pipe_producer::close() { - // the peer dropped before the producer finished. httplib bails its content provider the moment - // is_peer_alive() goes false, so the rest of the generation is pumped here into the ring buffer - // on the http worker, from on_complete. stream_aware_should_stop ignores peer disconnect while a - // pipe is attached, so res_->next() runs to natural completion, only an explicit DELETE flips - // is_cancelled and cuts it short + // httplib bails its content provider the moment is_peer_alive() goes false, so pump the rest + // of the generation into the ring buffer here. a DELETE flips is_cancelled and cuts it short if (done_ || session_->is_cancelled()) { SRV_INF("stream_pipe close: skip drain (done=%d cancelled=%d) conv=%s\n", done_ ? 1 : 0, session_->is_cancelled() ? 1 : 0, session_->conversation_id.c_str()); From 8b591a5e0cfde86718c2a7d75ed7a7a24f4bbab5 Mon Sep 17 00:00:00 2001 From: Pascal Date: Thu, 25 Jun 2026 16:16:23 +0200 Subject: [PATCH 38/39] server: balance and clean up stream comments remove redundant comments and tighten the verbose ones across the resumable stream code, keeping the concurrency and lifetime rationale that is not obvious from the code. also fix two stale comments in server.cpp and server-models.h that still described the old ::model suffix probe and fan out routing, now replaced by the conv_id -> model map Address review from ngxson --- tools/server/server-http.h | 7 ++--- tools/server/server-models.cpp | 16 ++++------ tools/server/server-models.h | 6 ++-- tools/server/server-stream.cpp | 6 ++-- tools/server/server-stream.h | 57 +++++++++++++--------------------- tools/server/server.cpp | 10 +++--- 6 files changed, 38 insertions(+), 64 deletions(-) diff --git a/tools/server/server-http.h b/tools/server/server-http.h index 455a0170619f..350813183671 100644 --- a/tools/server/server-http.h +++ b/tools/server/server-http.h @@ -25,10 +25,9 @@ struct server_http_res { std::string data; std::map headers; - // if set, the stream survives a client disconnect: when the peer leaves before the producer is - // done, on_complete drains the rest of the generation into the ring buffer on the http worker. - // the producer pipe destructor finalizes the session so no explicit on_stream_end is needed. - // shared_ptr used (not unique_ptr) so the forward-declared type is safe to delete here. + // if set, the stream survives a client disconnect: the producer pipe keeps draining into the + // ring buffer and finalizes the session on destruction, so no explicit on_stream_end is needed. + // shared_ptr (not unique_ptr) so the forward-declared type is safe to delete here. std::shared_ptr spipe; std::function next = nullptr; diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index 5f066f0ee759..44fc97fa87a5 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -1586,9 +1586,8 @@ static bool is_autoload(const common_params & params, const server_http_req & re } } -// percent encode a single query string or path component value. covers reserved chars without -// dragging in httplib::detail. used by the resumable stream routes to forward conversation_id -// to children safely +// percent encode one query or path component, covers reserved chars without pulling in +// httplib::detail. used by the stream routes to forward conversation_id to children safely static std::string encode_qs(const std::string & in) { std::string out; out.reserve(in.size() * 3); @@ -1674,9 +1673,8 @@ void server_models_routes::init_routes() { if (!router_validate_model(name, models, autoload, error_res)) { return error_res; } - // remember which child serves this conversation so the resumable stream routes can route - // straight to it without polling. key on the exact conv id from the header, the same value - // the GET and DELETE routes receive in their path, no parsing either side + // remember which child serves this conversation so the stream routes can route straight + // to it without polling, keyed on the exact conv id from the header std::string conv_id = stream_conv_id_from_headers(req.headers); if (!conv_id.empty()) { models.conv_models.remember(conv_id, name); @@ -1884,8 +1882,7 @@ void server_models_routes::init_routes() { this->router_stream_get = [this](const server_http_req & req) { // GET /v1/stream/?from=N. resolve the owning child from the conv_id -> model - // map (no polling), 404 when nothing maps. a stale map entry just forwards to a child - // that answers not found, the client recovers + // map, 404 when nothing maps auto res = std::make_unique(); std::string conv_id = req.get_param("conv_id"); if (conv_id.empty()) { @@ -1981,8 +1978,7 @@ void server_models_routes::init_routes() { this->router_stream_delete = [this](const server_http_req & req) { // DELETE /v1/stream/. resolve the owning child via the map and forward only to - // it. evict_and_cancel is idempotent on the child, a stale map entry just hits a child - // that has nothing to cancel and returns 204 + // it, evict_and_cancel is idempotent on the child auto res = std::make_unique(); std::string conv_id = req.get_param("conv_id"); if (conv_id.empty()) { diff --git a/tools/server/server-models.h b/tools/server/server-models.h index 3109ecd780d2..62bed8725b5b 100644 --- a/tools/server/server-models.h +++ b/tools/server/server-models.h @@ -313,10 +313,8 @@ struct server_models_routes { server_http_context::handler_t post_router_models; server_http_context::handler_t del_router_models; - // router side handlers for the resumable streaming routes. each conversation_id may carry - // an optional ::model suffix to enable direct routing without probing every child. when - // the suffix is absent the get/delete paths fall back to a loopback probe and a fan out - // respectively, the list path always fans out and aggregates + // router side handlers for the resumable streaming routes. each resolves the child that owns + // a conversation through the conv_id -> model map, no probing or fan out server_http_context::handler_t router_stream_get; server_http_context::handler_t router_streams_lookup; server_http_context::handler_t router_stream_delete; diff --git a/tools/server/server-stream.cpp b/tools/server/server-stream.cpp index d1cdfa6b1832..757c36ad257a 100644 --- a/tools/server/server-stream.cpp +++ b/tools/server/server-stream.cpp @@ -455,10 +455,8 @@ server_http_context::handler_t make_stream_get_handler() { server_http_context::handler_t make_streams_lookup_handler() { return [](const server_http_req & req) -> server_http_res_ptr { // POST /v1/streams/lookup with body {"conversation_ids": ["X", "Y", ...]} returns the - // matching sessions. you can only ask for ids you already know, the server never lists - // sessions it has not been asked about. for each requested id we match the exact key - // and any "::" variant, so a single lookup covers every per model session - // for that conv. used by the WebUI sidebar at mount and on visibilitychange + // matching sessions, only for ids the caller already knows. each id matches the exact key + // and any "::" variant, so one lookup covers every per model session for a conv std::vector requested; try { json body = json::parse(req.body); diff --git a/tools/server/server-stream.h b/tools/server/server-stream.h index e782d772bd4d..ff363bb4cd7f 100644 --- a/tools/server/server-stream.h +++ b/tools/server/server-stream.h @@ -20,11 +20,9 @@ enum class stream_read_status { OFFSET_LOST, }; -// streaming buffer for one generation, survives HTTP disconnect. -// the producer side pushes raw SSE bytes via append. HTTP readers drain from -// any offset via read_from. read_from blocks until new bytes arrive or the -// session is finalized. identity of the session is the conversation_id, no -// extra opaque token: one conv = at most one live session at a time +// streaming buffer for one generation, survives HTTP disconnect. the producer appends raw SSE +// bytes, readers drain from any offset via read_from and block until more bytes or finalize. +// keyed by conversation_id: one conv = at most one live session struct stream_session { std::string conversation_id; int64_t started_ts; // unix seconds at construction, used by /v1/streams listing @@ -48,18 +46,15 @@ struct stream_session { const std::function & should_stop); bool is_done() const; - bool is_cancelled() const; // true when cancel() has been invoked + bool is_cancelled() const; size_t total_size() const; // bytes that ever entered the session size_t dropped_prefix() const; // bytes evicted from the front due to cap int64_t completed_at() const; // 0 while alive, unix seconds after finalize - // attach a producer side stop hook, the drain sets this on startup so we can cancel its - // underlying reader. pass an empty function to detach (drain must clear before destroying - // its reader) + // attach the producer stop hook used to cancel its reader, pass an empty function to detach void set_stop_producer(std::function fn); - // invoke the stop hook if attached, signals the producer to abort its inference asap, - // idempotent + // signal the producer to abort its inference asap via the stop hook, idempotent void cancel(); private: @@ -103,14 +98,11 @@ struct stream_pipe_producer : stream_pipe { // append raw bytes to the session's ring buffer, returns false if already finalized bool write(const char * data, size_t len); - // record that the producer reached its natural end on the wire, so a later close() turns into - // a no-op. the http drain calls this right before it closes the stream cleanly + // mark the natural end on the wire so a later close() is a no-op void done(); - // close the producer end. when the peer dropped before the producer finished, pump the - // response next() into the ring buffer until it reports done. runs on the http worker, from - // on_complete. no-op once done() has fired or the session is cancelled, only a DELETE flips - // is_cancelled and cuts the drain short + // on a peer drop, pump the response next() into the ring buffer until done. runs on the http + // worker from on_complete, no-op after done() or cancel void close(); // disarm the stop hook and drop the alive guard, must run while the response the hook @@ -186,33 +178,26 @@ class stream_session_manager { std::condition_variable gc_wake_cv; }; -// the process wide stream session manager. defined in server-stream.cpp so the symbol -// resolves through the server-context static lib, both llama-server and llama-cli link it. -// start_gc() and stop_gc() are called explicitly from llama-server main(), llama-cli never -// touches it and leaves it idle. the destructor calls stop_gc() unconditionally so the -// process exit path is safe whether or not the GC thread was started +// process wide manager, linked by both llama-server and llama-cli. llama-server main() drives +// start_gc/stop_gc, llama-cli leaves it idle. the dtor calls stop_gc() unconditionally so exit +// is safe whether or not the GC thread ran extern stream_session_manager g_stream_sessions; -// route handler factories. each builds a server_http_context::handler_t that operates -// directly on g_stream_sessions, server.cpp wires them under /v1/stream/* without going -// through server-context's server_routes. keeps the resumable stream surface confined to -// server-stream and server-http +// route handler factories operating on g_stream_sessions, wired under /v1/stream/* by server.cpp. +// keeps the resumable stream surface confined to server-stream server_http_context::handler_t make_stream_get_handler(); server_http_context::handler_t make_streams_lookup_handler(); server_http_context::handler_t make_stream_delete_handler(); -// extract the X-Conversation-Id header value (case-insensitive), empty when absent. exposed -// so the router can read the conv id off a forwarded POST to track which child serves it +// extract the X-Conversation-Id header value (case-insensitive), empty when absent. exposed so +// the router can track which child serves a forwarded POST std::string stream_conv_id_from_headers(const std::map & headers); -// inspect request headers for X-Conversation-Id and, when present, create or replace a -// session on the global manager then attach a producer pipe to res. the pipe's stop_fn -// calls res.stop() (overridden by server_res_generator to stop its reader). no-op when -// the header is absent. server-context calls this from the server_res_generator constructor. +// on an X-Conversation-Id header, create or replace the session and attach a producer pipe to +// res. no-op when absent, called from the server_res_generator constructor void stream_session_attach_pipe(server_http_res & res, const std::map & headers); -// build a should_stop closure that suppresses peer-disconnect when a pipe is attached. -// when spipe is set, only an explicit cancel (DELETE /v1/stream/) stops the -// producer; peer disconnect is ignored so generation continues into the ring buffer. -// without a pipe the closure delegates to fallback, preserving the legacy non-resumable flow. +// should_stop closure that ignores peer disconnect when a pipe is attached, so only an explicit +// DELETE stops the producer and generation keeps flowing into the ring buffer. without a pipe it +// delegates to fallback, the legacy non-resumable flow std::function stream_aware_should_stop(server_http_res * res, std::function fallback); diff --git a/tools/server/server.cpp b/tools/server/server.cpp index 1e1c121ffbdf..3089085cd2f1 100644 --- a/tools/server/server.cpp +++ b/tools/server/server.cpp @@ -245,8 +245,8 @@ int llama_server(int argc, char ** argv) { // resumable streaming, the conversation_id is the session identity end to end. router and // child wire different handlers under the same paths: a child binds the local g_stream_sessions - // backed factories, the router binds proxies that route via the optional ::model suffix - // (direct) or fall back to loopback probe and fan out (suffixless conv ids) + // backed factories, the router binds proxies that resolve the owning child through the + // conv_id -> model map server_http_context::handler_t stream_get_h; server_http_context::handler_t streams_lookup_h; server_http_context::handler_t stream_delete_h; @@ -316,8 +316,7 @@ int llama_server(int argc, char ** argv) { clean_up = [&models_routes]() { SRV_INF("%s: cleaning up before exit...\n", __func__); - // stop the session GC first, this finalizes every live session and wakes any - // pending HTTP reader, the detached drains can then exit cleanly + // stop the session GC first, it finalizes live sessions and wakes pending readers g_stream_sessions.stop_gc(); if (models_routes.has_value()) { models_routes->stopping.store(true); // maybe redundant, but just to be safe @@ -345,8 +344,7 @@ int llama_server(int argc, char ** argv) { // setup clean up function, to be called before exit clean_up = [&ctx_http, &ctx_server]() { SRV_INF("%s: cleaning up before exit...\n", __func__); - // stop the session GC first, this finalizes every live session and wakes any - // pending HTTP reader, the detached drains can then exit cleanly + // stop the session GC first, it finalizes live sessions and wakes pending readers g_stream_sessions.stop_gc(); ctx_http.stop(); ctx_server.terminate(); From d346f8cb9faf33cd73e7a8372f00405575af298d Mon Sep 17 00:00:00 2001 From: Pascal Date: Thu, 25 Jun 2026 16:30:13 +0200 Subject: [PATCH 39/39] ui: balance and clean up stream comments dedup repeated rationale (frozen conv::model identity, the lookup privacy note, the abort patterns) down to one canonical spot, tighten the verbose blocks, and keep the concurrency and resume-offset reasoning. fix stale comments in stream-identity.ts and chat.service.ts that still described the old loopback probe and fan out routing, now the conv_id -> model map. --- tools/ui/src/lib/services/chat.service.ts | 9 +++---- tools/ui/src/lib/stores/chat.svelte.ts | 32 ++++++++--------------- tools/ui/src/lib/utils/abort.ts | 4 +-- tools/ui/src/lib/utils/stream-identity.ts | 6 ++--- 4 files changed, 18 insertions(+), 33 deletions(-) diff --git a/tools/ui/src/lib/services/chat.service.ts b/tools/ui/src/lib/services/chat.service.ts index 704aea3d4913..7dfee377311e 100644 --- a/tools/ui/src/lib/services/chat.service.ts +++ b/tools/ui/src/lib/services/chat.service.ts @@ -335,7 +335,7 @@ export class ChatService { const headers: Record = { ...getJsonHeaders() }; // tag streaming requests with the conversation id, this single header is the opt in for the // server side replay buffer and powers discoverActiveStream on tab reopen. with an explicit - // model the ::model suffix lets the router skip the loopback probe + // model the ::model suffix keeps the per model session distinct if (stream && conversationId) { headers['X-Conversation-Id'] = streamIdentity(conversationId, options.model); } @@ -936,10 +936,9 @@ export class ChatService { onConnectionState?.(StreamConnectionState.RESUMING); madeProgress = false; - // the server resends starting at bytesParsed, discard any partial line we held - // it will be retransmitted from a clean line boundary. reuse the model the POST was - // originally tagged with, the dropdown may have changed since but the server side - // identity is frozen at POST time + // the server resends starting at bytesParsed, discard any partial line we held, it + // will be retransmitted from a clean line boundary. reuse the frozen model, not the + // live dropdown const resumeResp = await ChatService.resumeStream( conversationId, abortSignal, diff --git a/tools/ui/src/lib/stores/chat.svelte.ts b/tools/ui/src/lib/stores/chat.svelte.ts index 83af31886d4d..faaaa9755e6b 100644 --- a/tools/ui/src/lib/stores/chat.svelte.ts +++ b/tools/ui/src/lib/stores/chat.svelte.ts @@ -91,9 +91,7 @@ class ChatStore { // off when one conv finishes while another is still streaming. mirrors chatLoadingStates // in scope but tracks the attach + tee replay path specifically private attachingConvs = new SvelteSet(); - // in-flight discoverActiveStream guard, keyed by conv id. prevents a fast remount + visibility - // race from launching two concurrent attaches on the same conv (which would dup chunks into - // the same DB message) + // in-flight discoverActiveStream guard, keyed by conv id private discoveringConvs = new SvelteSet(); private abortControllers = new SvelteMap(); private preEncodeAbortController: AbortController | null = null; @@ -200,8 +198,7 @@ class ChatStore { if (!convId) return null; let listResp: Response; try { - // POST the one conv id we are probing, the server only returns a match if it owns it, - // never lists ids the caller did not already provide + // POST the one conv id we are probing listResp = await fetch(`./v1/streams/lookup`, { method: 'POST', headers: { ...getAuthHeaders(), 'Content-Type': 'application/json' }, @@ -353,9 +350,8 @@ class ChatStore { writeActive({ content: '', reasoningContent: undefined }); } - // extract the model suffix from the server side identity, the resume calls inside - // handleStreamResponse must reuse the model the session was originally tagged with, - // not the current dropdown selection which may have changed since + // extract the model suffix, the resume calls in handleStreamResponse must reuse the model + // the session was tagged with, not the live dropdown const sepIdx = id.indexOf('::'); const attachedModel: string | null = sepIdx === -1 ? null : id.slice(sepIdx + 2); this.setChatStreaming(convId, existingContent, targetMessageId, attachedModel); @@ -647,18 +643,15 @@ class ChatStore { console.warn('syncRemoteRunningStreams DB read failed:', e); return; } - // only ask about conv ids the user already owns. the server never lists ids the caller did - // not provide, so a random foreign UUID stays unguessable + // only ask about conv ids the user already owns if (ids.length === 0) { for (const id of Array.from(this.remoteRunningConvs)) { this.remoteRunningConvs.delete(id); } return; } - // the lookup is keyed by the identity frozen at POST time (conv::model when the stream - // carried an explicit model), rebuild it per conv from the persisted state so a running - // session started with a model still matches. a single model conv stays a bare id, and - // the server response is mapped back to the bare id below for the sidebar set + // rebuild the frozen conv::model identity per conv so a session started with a model still + // matches. the server response is mapped back to the bare id below for the sidebar set const lookupIds = ids.map((id) => ChatService.resumeStreamIdentity(id, ChatService.getStreamState(id), null) ); @@ -680,9 +673,7 @@ class ChatStore { const running = new SvelteSet(); for (const s of sessions) { if (s && !s.is_done && typeof s.conversation_id === 'string' && s.conversation_id) { - // strip the optional ::model suffix, the sidebar lookup is keyed by the bare conv id - // straight from the DB. without this the sidebar spinner never matches and stays off - // when the running session was started with an explicit model + // strip the optional ::model suffix, the sidebar set is keyed by the bare conv id const sepIdx = s.conversation_id.indexOf('::'); const bareId = sepIdx === -1 ? s.conversation_id : s.conversation_id.slice(0, sepIdx); running.add(bareId); @@ -1410,10 +1401,9 @@ class ChatStore { async stopGenerationForChat(convId: string): Promise { await this.savePartialResponseIfNeeded(convId); this.setStreamingActive(false); - // tell the server to stop the generation, not just to drop the HTTP socket. without this - // the detached drain keeps producing tokens until eos or max_tokens. use the model captured - // when the session started rather than the current dropdown, the dropdown may have changed - // since and the server side identity (conv id plus ::model suffix) is frozen at POST time + // tell the server to stop the generation, not just drop the HTTP socket. without this the + // detached drain keeps producing tokens until eos or max_tokens. use the frozen identity + // captured when the session started, not the live dropdown const streamStateForStop = this.chatStreamingStates.get(convId); const modelForStop = streamStateForStop?.model ?? selectedModelName(); void ChatService.cancelServerStream(convId, modelForStop); diff --git a/tools/ui/src/lib/utils/abort.ts b/tools/ui/src/lib/utils/abort.ts index 67246b22f6fc..135ef087a0a0 100644 --- a/tools/ui/src/lib/utils/abort.ts +++ b/tools/ui/src/lib/utils/abort.ts @@ -66,9 +66,7 @@ export function isAbortError(error: unknown): boolean { if (error.name === ABORT_ERROR_NAME) { return true; } - // browser specific patterns emitted when a fetch reader is interrupted by page - // unload, navigation, or transient network drop. these are functionally aborts, - // not actionable application errors, so they should not surface as red console logs + // these patterns are functionally aborts, keep them out of the red console if (error instanceof TypeError) { const msg = error.message ?? ''; if (ABORT_LIKE_MESSAGE_PATTERNS.some((re) => re.test(msg))) return true; diff --git a/tools/ui/src/lib/utils/stream-identity.ts b/tools/ui/src/lib/utils/stream-identity.ts index 8c900a5b11c0..ce88df007443 100644 --- a/tools/ui/src/lib/utils/stream-identity.ts +++ b/tools/ui/src/lib/utils/stream-identity.ts @@ -3,10 +3,8 @@ * * The server identifies a stream session by a conversation id sent in the * X-Conversation-Id header. When the user has explicitly picked a model the - * client appends ::modelName, which lets the router fan out direct to that - * child for resume and stop without probing every other one. Without the - * suffix the router falls back to a loopback probe and a DELETE fan out, both - * still correct, just slower at the lookup step. + * client appends ::modelName, so a per model session stays distinct and the + * router resolves the owning child through its conv_id -> model map. */ export function streamIdentity(conversationId: string, model?: string | null): string { if (!conversationId) return '';