From 22056e2e0c073b8538ca22e0dccd3fb6058cc500 Mon Sep 17 00:00:00 2001 From: MagicYang1573 <1328657938@qq.com> Date: Mon, 29 Jun 2026 16:51:52 +0800 Subject: [PATCH] [Store] Add Mooncake KV Event Publisher for Dynamo-compatible KV events Mooncake now emits Dynamo-compatible BlockStored/BlockRemoved KV cache events so Dynamo's router can build its prefix index from blocks stored in Mooncake. - Input interface: extend ReplicateConfig with group-level kv_event_metadata (KvBlockEventMetadata / KvBlockComponentSpec) carrying logical-block semantics from the engine; Python bindings included. - Group manifest in MasterService coalesces physical objects into one logical block, publishing exactly one BlockStored on completion and one BlockRemoved on removal, with per-worker dedup. - segment->worker_id registry so each event's worker_id reflects the physical worker (te_endpoint), with optional explicit override. - Output interface: a single ZMQ PUB stream (ZmqKvEventPublisher) emitting the SGLang/vLLM-compatible 3-frame msgpack message; worker_id is carried per event so one centralized stream serves all workers. - Tests for manifest lifecycle, publishing/dedup, worker_id resolution, and a real ZMQ pub/sub roundtrip. Co-authored-by: Cursor --- .../mooncake_kv_event_publisher.md | 408 +++++++++++ .../mooncake_kv_event_publisher.zh.md | 364 ++++++++++ mooncake-integration/store/store_py.cpp | 32 + mooncake-store/include/kv_event.h | 115 +++ .../include/kv_event_zmq_publisher.h | 72 ++ mooncake-store/include/master_service.h | 175 ++++- mooncake-store/include/replica.h | 115 +++ mooncake-store/src/CMakeLists.txt | 33 + mooncake-store/src/kv_event.cpp | 259 +++++++ mooncake-store/src/kv_event_zmq_publisher.cpp | 108 +++ mooncake-store/src/master_service.cpp | 441 ++++++++++++ mooncake-store/src/rpc_service.cpp | 32 +- mooncake-store/tests/master_service_test.cpp | 657 ++++++++++++++++++ 13 files changed, 2808 insertions(+), 3 deletions(-) create mode 100644 docs/source/design/kv-event-dynamo/mooncake_kv_event_publisher.md create mode 100644 docs/source/design/kv-event-dynamo/mooncake_kv_event_publisher.zh.md create mode 100644 mooncake-store/include/kv_event.h create mode 100644 mooncake-store/include/kv_event_zmq_publisher.h create mode 100644 mooncake-store/src/kv_event.cpp create mode 100644 mooncake-store/src/kv_event_zmq_publisher.cpp diff --git a/docs/source/design/kv-event-dynamo/mooncake_kv_event_publisher.md b/docs/source/design/kv-event-dynamo/mooncake_kv_event_publisher.md new file mode 100644 index 00000000000..811a426d89f --- /dev/null +++ b/docs/source/design/kv-event-dynamo/mooncake_kv_event_publisher.md @@ -0,0 +1,408 @@ +# Mooncake KV Event Publisher + +This document describes the Mooncake KV Event Publisher: the feature that lets +Mooncake emit Dynamo-compatible KV cache events so Dynamo's router can build its +prefix index from blocks stored in Mooncake. It covers both interfaces of the +feature — how the inference engine (SGLang) feeds semantic information *into* +Mooncake, and how Mooncake publishes events *out* over ZMQ — plus a summary of +the internal implementation. It is the design/reference companion for the +upcoming PR. + +## 1. Overview + +Mooncake is a *centralized* KV cache manager: the master knows the full state of +every logical KV block. Dynamo's router consumes KV events to maintain its +prefix index, and its reference engines (SGLang/vLLM) emit one ZMQ stream per +worker. Because Mooncake owns all state centrally, it publishes **a single ZMQ +PUB stream for all workers** and identifies the physical owner of each block via +a per-event `worker_id` field. A Mooncake-specific adapter on the Dynamo side +reads that field. (See `mooncake开发方案.md` §7.3.) + +End-to-end pipeline: + +```text +SGLang (knows semantics) Mooncake master (centralized state) Dynamo + put(key, value, ReplicateConfig{ - groups physical objects into a ZMQ SUB + group_ids, kv_event_metadata }) → logical block (manifest) → + adapter + - when complete -> BlockStored → prefix + - on removal -> BlockRemoved index + - publishes one ZMQ PUB stream +``` + +Two interfaces matter: + +- **Input — SGLang → Mooncake (§2):** writes carry `kv_event_metadata` describing + the logical block's semantics (token ids, hashes, layout). This is the only + way Mooncake learns information it cannot derive itself. +- **Output — Mooncake → Dynamo (§3):** a single ZMQ PUB stream of msgpack events. + +When no `kv_event_metadata` is attached and no publisher is installed, Mooncake +behavior is completely unchanged. + +## 2. Input Interface: How SGLang Passes Semantics to Mooncake + +Mooncake stores opaque bytes; it does not know token ids, sequence hashes, or how +many physical objects make up one logical KV block. Only the engine knows these. +SGLang therefore attaches this semantic information to each write through an +extension of `ReplicateConfig`. + +### 2.1 Logical block vs. physical objects + +A single logical KV block (e.g. one SGLang prefix page) may be stored as several +physical Mooncake objects — for example the `K` and `V` objects of an MHA page, +or several per-head component objects. The model is: + +- `ReplicateConfig::group_ids` — one `group_id` per physical key, tying the + physical objects of one logical block together (grouped keys already share + metadata routing, coalesced lease refresh, and eviction behavior). +- `ReplicateConfig::kv_event_metadata` — group-level, **not** per-key: one + `KvBlockEventMetadata` item per `group_id` carrying the block's semantics and + the expected physical layout, so the master knows when the logical block is + fully stored. + +### 2.2 Extended `ReplicateConfig` (`replica.h`) + +```cpp +struct ReplicateConfig { + // ... existing fields ... + std::optional> group_ids{}; + // Optional group-level KV event metadata. NOT per-key: each item describes + // one logical Dynamo KV block keyed by group_id. Absent => behavior + // unchanged. + std::optional> kv_event_metadata{}; +}; +``` + +`KvBlockComponentSpec` — one physical object of a logical block: + +| Field | Type | Meaning | +|-------|------|---------| +| `object_key` | str | Physical Mooncake object key. | +| `component_role` | str | e.g. `"k"`, `"v"`, `"mla_k"`, `"head_3_k"`. | +| `component_index` | u32 | Stable ordering index within the block. | + +`KvBlockEventMetadata` — one logical block (one `group_id`): + +| Field | Type | Meaning | +|-------|------|---------| +| `schema_version` | u32 | Metadata schema version (currently `1`). | +| `group_id` | str | Logical block id; must match an entry in `group_ids`. | +| `block_hash` | u64 | Dynamo external sequence block hash. | +| `parent_block_hash` | u64? | Parent sequence hash; unset for the root block. | +| `token_ids` | `[u32]` | Token ids in the block (Dynamo needs them for its local token hash). | +| `block_size` | u32 | Tokens per block; must match the consumer's `kv_block_size`. | +| `dp_rank` | u32? | Data-parallel rank (routing namespace). | +| `model_name` | str | Model name (routing namespace). | +| `lora_name` | str | LoRA adapter name (routing namespace). | +| `additional_salt` | str | Extra namespace salt. | +| `expected_object_count` | u32 | How many physical objects make the block complete. | +| `expected_components` | `[KvBlockComponentSpec]` | The expected physical objects (preferred; `expected_object_count` is derived from it when set). | +| `emit_stored_event` | bool | Whether to publish `BlockStored` (default true). | +| `emit_removed_event` | bool | Whether to publish `BlockRemoved` (default true). | + +Completeness is decided by `expected_components` when provided; otherwise by +`expected_object_count`. Repeated writes for the same `group_id` are accepted as +idempotent retries when semantically identical (`SameSemanticsAs`) and rejected +as conflicts otherwise. + +### 2.3 What SGLang provides vs. what Mooncake derives + +| Information | Provided by | +|-------------|-------------| +| `token_ids`, `block_hash`, `parent_block_hash`, `block_size` | **SGLang** (semantics it alone knows) | +| `model_name`, `lora_name`, `dp_rank`, `additional_salt` | **SGLang** (routing namespace) | +| physical layout (`group_ids`, `expected_components`) | **SGLang** | +| `worker_id` (physical owner) | **Mooncake** (from replica placement; see §4.2) | +| `event_id`, `source`, `tenant_id`, `medium` | **Mooncake** | +| completeness / dedup / stored↔removed lifecycle | **Mooncake** (manifest) | + +### 2.4 Python bindings (`store_py.cpp`) + +`KvBlockComponentSpec`, `KvBlockEventMetadata`, and +`ReplicateConfig.kv_event_metadata` are exposed to Python. Example: one logical +block stored as two physical objects (K and V), publishing exactly one +`BlockStored` once both are present. + +```python +from mooncake.store import ( + ReplicateConfig, KvBlockEventMetadata, KvBlockComponentSpec, +) + +key_k, key_v = "blk-42:k", "blk-42:v" + +c_k = KvBlockComponentSpec(); c_k.object_key = key_k; c_k.component_role = "k"; c_k.component_index = 0 +c_v = KvBlockComponentSpec(); c_v.object_key = key_v; c_v.component_role = "v"; c_v.component_index = 1 + +meta = KvBlockEventMetadata() +meta.group_id = "blk-42" +meta.block_hash = 0x8f3a_0000_0000_0001 +meta.parent_block_hash = 0x8f3a_0000_0000_0000 # leave unset for the root block +meta.token_ids = [101, 102, 103, 104] +meta.block_size = 64 +meta.model_name = "llama-3" +meta.dp_rank = 0 +meta.expected_components = [c_k, c_v] # or: meta.expected_object_count = 2 + +config = ReplicateConfig() +config.replica_num = 1 +config.group_ids = ["blk-42", "blk-42"] # one entry per physical key +config.kv_event_metadata = [meta] + +store.put(key_k, value_k, config) +store.put(key_v, value_v, config) # block now complete -> one BlockStored event +``` + +## 3. Output Interface: The ZMQ Event Stream + +### 3.1 Transport model + +**Design decision — one ZMQ stream + per-event `worker_id`.** Dynamo's ZMQ +interface was designed for *one ZMQ connection per worker*: each engine worker +runs its own PUB socket and Dynamo identifies the source by which socket the +event arrived on. Mooncake, however, is a centralized KV cache manager — the +master alone knows the state of every block across all workers, so a single ZMQ +stream is sufficient to carry everything. We deliberately keep **one ZMQ stream** +and instead add an extra **`worker_id` field to each published event** to encode +which physical storage node actually holds the KV block. On the Dynamo side a +Mooncake-specific adapter then parses this `worker_id` to attribute each event to +the right worker. This avoids standing up (and managing) one socket per worker +on a centralized component while preserving Dynamo's per-worker routing +semantics. (See `mooncake开发方案.md` §7.3.) + +- **One PUB socket** owned by the master. Subscribers (the Dynamo adapter) + connect with a SUB socket. +- Each `Publish()` sends **one single-event batch**. Mooncake does not buffer or + coalesce events at this layer; ordering and de-duplication are handled + upstream in the manifest logic. +- `worker_id` inside each event tells the consumer which physical node holds the + block. The batch-level `dp_rank` trailer is also set (from the event), but the + adapter should rely on the per-event fields. + +### 3.2 Wire format (3 frames) + +Byte-compatible with the SGLang/vLLM ZMQ relay format Dynamo already supports +(`dynamo_kv_event_rules.md` §"ZMQ Relay Wire Format"): + +| Frame | Content | Notes | +|-------|---------|-------| +| 1 | `topic` | Matched against the subscriber's `zmq_topic` filter. Default is the empty string (matches all). The listener requires exactly 3 frames. | +| 2 | 8-byte **big-endian** sequence | Monotonic per-publisher counter. Used by Dynamo for trace/log only; it is **not** the internal `event_id`. | +| 3 | msgpack payload | `[timestamp (f64), [events], dp_rank (i32 or nil)]`. | + +Payload structure: + +```text +[ + timestamp, # f64, seconds since epoch (engine batch timestamp) + [ ], # array of raw KV events (always length 1 here) + dp_rank # i32, or nil when unknown +] +``` + +### 3.3 Map event fields + +Each event is a msgpack **map** (Dynamo parses by field name and ignores unknown +keys). `BlockStored` carries all fields; `BlockRemoved` carries only the +identity + `block_hashes` + `medium`. + +Common fields (both types): + +| Field | Type | Meaning | +|-------|------|---------| +| `type` | str | `"BlockStored"` or `"BlockRemoved"`. | +| `block_hashes` | `[u64]` | External sequence block hashes (prefix-cumulative). | +| `medium` | str | Storage tier, e.g. `"EXTERNAL"`. | +| `event_id` | str | Unique id assigned by the master (`mooncake::::`). | +| `source` | str | Always `"mooncake"`. | +| `tenant_id` | str | Mooncake tenant. | +| `group_id` | str | Logical KV block group id. | +| `worker_id` | str | Physical worker holding the block (see §4.2). | + +`BlockStored`-only fields: `parent_block_hash` (u64 or nil), `token_ids` +(`[u32]`), `block_size` (u32), `lora_name` (str), `model_name` (str), and +`dp_rank` (u32, present only when set on the event). + +> `source`, `tenant_id`, `group_id`, `worker_id`, `event_id` are Mooncake +> extensions on top of the base Dynamo schema. A vanilla Dynamo decoder ignores +> unknown fields; the Mooncake adapter is the component that interprets +> `worker_id`. + +### 3.4 Publisher C++ API (`kv_event_zmq_publisher.h`) + +```cpp +struct ZmqKvEventPublisherConfig { + std::string endpoint{"tcp://0.0.0.0:5557"}; // "tcp://host:*" picks a free port + std::string topic{}; // frame-1 topic; empty matches all + bool bind{true}; // true: bind (master is server) + int linger_ms{0}; // close linger; -1 = block, 0 = drop + int send_hwm{0}; // send high-water-mark; 0 = unlimited +}; + +class ZmqKvEventPublisher : public KvEventPublisher { + public: + explicit ZmqKvEventPublisher(const ZmqKvEventPublisherConfig& config); + void Publish(const KvEvent& event) override; + const std::string& endpoint() const; // resolved endpoint (wildcard ports too) + uint64_t published_count() const; // diagnostics +}; +``` + +Installing it on the master (any `KvEventPublisher` works; `nullptr` disables +publishing): + +```cpp +auto publisher = std::make_shared( + ZmqKvEventPublisherConfig{.endpoint = "tcp://0.0.0.0:5557"}); +master_service.SetKvEventPublisher(publisher); +``` + +## 4. Internal Implementation + +### 4.1 Group manifest & event lifecycle (`master_service.{h,cpp}`) + +- Each `group_id` that carries `kv_event_metadata` gets a **manifest** stored in + the same tenant shard as its members (so it is covered by the existing shard + lock). It tracks the expected object keys, the keys currently present, and the + per-worker dedup bookkeeping. +- `PutEnd` marks an object complete; when all expected components are present the + master publishes exactly one `BlockStored` (deduplicated via + `stored_event_published_workers`). +- Removal/eviction marks the logical block incomplete and publishes one + `BlockRemoved` (deduplicated via `removed_event_published_workers`); the + manifest is erased when the last group member is removed. +- Validation rejects conflicting redefinitions of a `group_id` and malformed + metadata (empty/duplicate group ids, missing completeness expectation). + +### 4.2 segment→worker_id registry (internal) + +`worker_id` must identify the **physical worker** that stores the block, not the +logical segment name used for allocation. An internal registry resolves it, in +priority order: + +1. **Explicit override** — `MasterService::RegisterSegmentWorkerId(segment, id)` + (deployment config / control plane). Wins over everything. +2. **Mount-time endpoint** — captured automatically when a segment is mounted, + from `Segment::te_endpoint` (the node's ip:port). Falls back to the segment + name when the endpoint is empty. +3. **Fallback** — the segment name itself, when the segment is unknown. + +Implementation notes: + +- Two maps (`segment_worker_explicit_`, `segment_worker_auto_`) under a dedicated + `shared_mutex`, so they can be read while a tenant shard lock is held. +- `MountSegment` auto-registers `segment.name → te_endpoint` after the mount + succeeds and the segment lock is released. +- `PutEnd` does `segment = DeriveSegmentNameFromMetadata(...)` → + `worker_id = ResolveWorkerId(segment)` → stamp the event. (`GetSegmentWorkerId` + exposes the mapping for diagnostics/tests.) +- The registry is in-memory and rebuilt on remount; it is **not** part of the HA + snapshot yet (see §7). + +### 4.3 Event model & ZMQ publisher + +- `KvEvent` / `KvEventType` plus the msgpack encode/decode functions + (`EncodeKvEventMap`, `EncodeKvEventBatchPayload`, `DecodeKvEventBatchPayload`) + live in `kv_event.{h,cpp}` and are shared by the ZMQ publisher and the + in-memory mock used in tests. +- `ZmqKvEventPublisher` uses the **pimpl** idiom so `zmq.hpp` never leaks into a + public header; the `Impl` owns a `zmq::context_t` and a PUB `zmq::socket_t`, + resolves the concrete endpoint via `ZMQ_LAST_ENDPOINT`, and on `Publish()` + encodes a single-event batch, prepends the monotonic big-endian sequence, and + sends the 3 frames under a mutex. ZMQ send errors are logged and swallowed + (publishing is best-effort and must not break the data path). + +## 5. Build Integration + +`mooncake-store/src/CMakeLists.txt` auto-detects ZeroMQ (cppzmq header +`zmq.hpp` + `libzmq`, with `$CONDA_PREFIX` hints): + +- When found: compiles `kv_event_zmq_publisher.cpp` and propagates + `MOONCAKE_STORE_WITH_ZMQ`, the include dir, and the link library as **PUBLIC** + usage requirements of `mooncake_store` (so tests/consumers inherit them). +- When absent: the manifest/encoder still build; only the live ZMQ publisher is + omitted. Guard ZMQ-specific code with `#ifdef MOONCAKE_STORE_WITH_ZMQ`. + +Dependencies (e.g. in a conda env): `cppzmq` + `zeromq`. + +## 6. Testing + +`mooncake-store/tests/master_service_test.cpp`: + +- Manifest lifecycle: creation/completion, sharing across group members, + conflict rejection, erase on group removal, config validation. +- Publishing: single `BlockStored` on completion, no duplicates on repeated + `PutEnd`, single `BlockRemoved` on delete, no-op without a publisher. +- worker_id: `SegmentWorkerRegistryCapturesEndpointAtMount`, + `KvEventWorkerIdReflectsSegmentEndpoint`, `KvEventWorkerIdExplicitOverrideWins`. +- Transport: `KvEventEncoderRoundTrip`, and `ZmqKvEventPublisherRoundTrip` + (guarded by `MOONCAKE_STORE_WITH_ZMQ`) where a real SUB socket receives the 3 + frames over TCP and decodes them back into the original `KvEvent`. + +## 7. Limitations / Future Work + +### 7.1 Limitations in this change + +- **HA snapshot**: neither the KV group manifests nor the segment→worker_id + registry are persisted in the master HA snapshot yet. After a failover the + registry is rebuilt from remounts; manifest dedup state still needs snapshot + support for fully correct cross-failover behavior. +- **Batching**: one event per ZMQ message today. Batching multiple events per + payload is possible later (the encoder already supports it) if throughput + requires it. + +The two pieces below complete the end-to-end path but live in their own repos +(SGLang / Dynamo) and are out of scope for the Mooncake PR; they are documented +here so the full data flow is clear. + +### 7.2 SGLang → Mooncake wiring (engine side, separate repo) + +The Mooncake side of the input interface (§2) is complete, but SGLang does not +yet populate it. Follow-up work in the SGLang repo: + +- At KV-cache write time, build one `KvBlockEventMetadata` per logical block and + attach it (with the matching `group_ids`) to the `ReplicateConfig` passed to + `store.put` / batch put. +- Map SGLang's block manager state onto the schema: external **sequence** block + hash → `block_hash` / `parent_block_hash` (cumulative prefix hash, not the + per-block local hash), the block's `token_ids`, and `block_size` (must equal + Dynamo's `kv_block_size`). +- Enumerate the physical objects of each block as `expected_components` + (`object_key` + `component_role` + `component_index`), e.g. the K and V objects + of an MHA page, or per-head components. Set `model_name` / `lora_name` / + `dp_rank` / `additional_salt` for the routing namespace. +- Decide `emit_stored_event` / `emit_removed_event` per deployment, and ensure + the same `group_id` is reused (idempotent retries) rather than redefined. + +### 7.3 Dynamo Mooncake adapter (router side, separate repo) + +A single ZMQ stream carrying a centralized publisher's events needs a +Mooncake-specific adapter on the Dynamo side (the counterpart to the §3.1 design +decision). Follow-up work in the Dynamo repo: + +- Subscribe to Mooncake's single PUB endpoint with a SUB socket and the matching + `zmq_topic`, decode the 3-frame message and the `[timestamp, [events], + dp_rank]` payload. +- **Parse the per-event `worker_id`** and attribute each `BlockStored` / + `BlockRemoved` to that physical worker, instead of inferring the worker from + the socket/connection (the assumption baked into the default one-socket-per- + worker path). Use the per-event `worker_id` (and `dp_rank`) rather than the + batch trailer. +- Map the Mooncake extension fields (`source`, `tenant_id`, `group_id`, + `event_id`) into Dynamo's internal `RouterEvent` model, and translate `medium` + → storage tier. Honor Dynamo's own `block_size == kv_block_size` and + re-numbering/dedup rules. +- Treat the frame-2 sequence as trace/log only; rely on Mooncake's `event_id` + and Dynamo's own monotonic ids for ordering. + +## 8. Changed / Added Files + +| File | Change | +|------|--------| +| `include/replica.h` | `KvBlockComponentSpec`, `KvBlockEventMetadata`, and `ReplicateConfig::kv_event_metadata`. | +| `mooncake-integration/store/store_py.cpp` | Python bindings for the metadata types + `kv_event_metadata`. | +| `include/kv_event.h`, `src/kv_event.cpp` | Event model + msgpack encode/decode. | +| `include/kv_event_zmq_publisher.h`, `src/kv_event_zmq_publisher.cpp` | **New.** ZMQ PUB transport (pimpl). | +| `include/master_service.h`, `src/master_service.cpp` | Group manifest, event publishing/dedup, segment→worker_id registry, `SetKvEventPublisher` / `Register`/`GetSegmentWorkerId`. | +| `src/rpc_service.cpp` | Config-level `kv_event_metadata` validation at the RPC entry. | +| `src/CMakeLists.txt` | ZeroMQ detection; conditional source + PUBLIC include/lib/`MOONCAKE_STORE_WITH_ZMQ`. | +| `tests/master_service_test.cpp` | Manifest, publisher, worker_id, and real ZMQ roundtrip tests. | diff --git a/docs/source/design/kv-event-dynamo/mooncake_kv_event_publisher.zh.md b/docs/source/design/kv-event-dynamo/mooncake_kv_event_publisher.zh.md new file mode 100644 index 00000000000..84d40f9cdfc --- /dev/null +++ b/docs/source/design/kv-event-dynamo/mooncake_kv_event_publisher.zh.md @@ -0,0 +1,364 @@ +# Mooncake KV Event Publisher + +本文介绍 Mooncake KV Event Publisher:使 Mooncake 能够发布 Dynamo 兼容的 KV cache +event,从而让 Dynamo 的 router 基于存储在 Mooncake 中的 block 构建 prefix index。 +文档覆盖该特性的两个接口——推理引擎(SGLang)如何把语义信息**输入**到 Mooncake, +以及 Mooncake 如何通过 ZMQ 把事件**输出**出去——并概述内部实现。本文作为后续 PR +的设计/参考说明。 + +## 1. 概述 + +Mooncake 是**中心化**的 KV cache 管理者:master 掌握每个逻辑 KV block 的完整状态。 +Dynamo 的 router 通过消费 KV event 维护其 prefix index,其参考引擎(SGLang/vLLM) +是**每个 worker 一条 ZMQ 流**。由于 Mooncake 状态集中管理,我们用**一条 ZMQ PUB 流 +承载所有 worker 的事件**,并通过每条事件中的 `worker_id` 字段标识 block 的物理归属; +Dynamo 侧由一个 Mooncake 专用 adapter 解析该字段。(详见 `mooncake开发方案.md` §7.3。) + +端到端流程: + +```text +SGLang(掌握语义) Mooncake master(中心化状态) Dynamo + put(key, value, ReplicateConfig{ - 将多个物理对象聚合为 ZMQ SUB + group_ids, kv_event_metadata }) → 一个逻辑 block(manifest) → + adapter + - 完整时 -> BlockStored → prefix + - 移除时 -> BlockRemoved index + - 通过一条 ZMQ PUB 流发布 +``` + +两个关键接口: + +- **输入 —— SGLang → Mooncake(§2):** 写入时携带 `kv_event_metadata`,描述逻辑 + block 的语义(token id、各类 hash、物理布局)。这是 Mooncake 获取其自身无法推导 + 的信息的唯一途径。 +- **输出 —— Mooncake → Dynamo(§3):** 一条 ZMQ PUB 流,承载 msgpack 事件。 + +当写入未携带 `kv_event_metadata` 且未安装 publisher 时,Mooncake 行为完全不变。 + +## 2. 输入接口:SGLang 如何向 Mooncake 传递语义 + +Mooncake 存储的是不透明字节,它不知道 token id、sequence hash,也不知道一个逻辑 +KV block 由几个物理对象组成。只有引擎知道这些。因此 SGLang 通过扩展 `ReplicateConfig` +把语义信息附加到每次写入上。 + +### 2.1 逻辑 block 与物理对象 + +一个逻辑 KV block(例如一个 SGLang prefix page)可能被存为多个物理 Mooncake 对象—— +比如一个 MHA page 的 `K` 与 `V` 对象,或若干个 per-head 的 component 对象。模型如下: + +- `ReplicateConfig::group_ids` —— 每个物理 key 对应一个 `group_id`,把同一逻辑 block + 的物理对象绑在一起(分组的 key 本就共享元数据路由、合并的 lease 刷新与淘汰行为)。 +- `ReplicateConfig::kv_event_metadata` —— group 级别,**非** per-key:每个 `group_id` + 对应一个 `KvBlockEventMetadata`,携带该 block 的语义与期望的物理布局,使 master + 知道逻辑 block 何时被完整存储。 + +### 2.2 扩展后的 `ReplicateConfig`(`replica.h`) + +```cpp +struct ReplicateConfig { + // ... 既有字段 ... + std::optional> group_ids{}; + // 可选的 group 级 KV event 元数据。非 per-key:每个 item 描述一个由 group_id + // 标识的逻辑 Dynamo KV block。缺省 => 行为不变。 + std::optional> kv_event_metadata{}; +}; +``` + +`KvBlockComponentSpec` —— 逻辑 block 的一个物理对象: + +| 字段 | 类型 | 含义 | +|------|------|------| +| `object_key` | str | 物理 Mooncake 对象 key。 | +| `component_role` | str | 例如 `"k"`、`"v"`、`"mla_k"`、`"head_3_k"`。 | +| `component_index` | u32 | block 内稳定的排序索引。 | + +`KvBlockEventMetadata` —— 一个逻辑 block(一个 `group_id`): + +| 字段 | 类型 | 含义 | +|------|------|------| +| `schema_version` | u32 | 元数据 schema 版本(当前为 `1`)。 | +| `group_id` | str | 逻辑 block id;须与 `group_ids` 中的某项一致。 | +| `block_hash` | u64 | Dynamo 外部 sequence block hash。 | +| `parent_block_hash` | u64? | 父 sequence hash;根 block 不设置。 | +| `token_ids` | `[u32]` | block 内 token id(Dynamo 据此计算 local token hash)。 | +| `block_size` | u32 | 每个 block 的 token 数;须与消费者的 `kv_block_size` 一致。 | +| `dp_rank` | u32? | data-parallel rank(路由命名空间)。 | +| `model_name` | str | 模型名(路由命名空间)。 | +| `lora_name` | str | LoRA adapter 名(路由命名空间)。 | +| `additional_salt` | str | 额外命名空间 salt。 | +| `expected_object_count` | u32 | 完整该 block 需要多少个物理对象。 | +| `expected_components` | `[KvBlockComponentSpec]` | 期望的物理对象(优先;设置后 `expected_object_count` 由其推导)。 | +| `emit_stored_event` | bool | 是否发布 `BlockStored`(默认 true)。 | +| `emit_removed_event` | bool | 是否发布 `BlockRemoved`(默认 true)。 | + +完整性判定:提供 `expected_components` 时以其为准,否则用 `expected_object_count`。 +对同一 `group_id` 的重复写入,若语义相同(`SameSemanticsAs`)则视为幂等重试接受, +否则视为冲突拒绝。 + +### 2.3 SGLang 提供 vs. Mooncake 推导 + +| 信息 | 提供方 | +|------|--------| +| `token_ids`、`block_hash`、`parent_block_hash`、`block_size` | **SGLang**(仅其掌握的语义) | +| `model_name`、`lora_name`、`dp_rank`、`additional_salt` | **SGLang**(路由命名空间) | +| 物理布局(`group_ids`、`expected_components`) | **SGLang** | +| `worker_id`(物理归属) | **Mooncake**(由副本放置推导;见 §4.2) | +| `event_id`、`source`、`tenant_id`、`medium` | **Mooncake** | +| 完整性 / 去重 / stored↔removed 生命周期 | **Mooncake**(manifest) | + +### 2.4 Python 绑定(`store_py.cpp`) + +`KvBlockComponentSpec`、`KvBlockEventMetadata` 与 +`ReplicateConfig.kv_event_metadata` 均已暴露给 Python。示例:一个逻辑 block 存为两个 +物理对象(K 与 V),在两者都到位后恰好发布一条 `BlockStored`。 + +```python +from mooncake.store import ( + ReplicateConfig, KvBlockEventMetadata, KvBlockComponentSpec, +) + +key_k, key_v = "blk-42:k", "blk-42:v" + +c_k = KvBlockComponentSpec(); c_k.object_key = key_k; c_k.component_role = "k"; c_k.component_index = 0 +c_v = KvBlockComponentSpec(); c_v.object_key = key_v; c_v.component_role = "v"; c_v.component_index = 1 + +meta = KvBlockEventMetadata() +meta.group_id = "blk-42" +meta.block_hash = 0x8f3a_0000_0000_0001 +meta.parent_block_hash = 0x8f3a_0000_0000_0000 # 根 block 则不设置 +meta.token_ids = [101, 102, 103, 104] +meta.block_size = 64 +meta.model_name = "llama-3" +meta.dp_rank = 0 +meta.expected_components = [c_k, c_v] # 或:meta.expected_object_count = 2 + +config = ReplicateConfig() +config.replica_num = 1 +config.group_ids = ["blk-42", "blk-42"] # 每个物理 key 一项 +config.kv_event_metadata = [meta] + +store.put(key_k, value_k, config) +store.put(key_v, value_v, config) # block 此时完整 -> 一条 BlockStored 事件 +``` + +## 3. 输出接口:ZMQ 事件流 + +### 3.1 传输模型 + +**设计决策 —— 单条 ZMQ 流 + 每条事件携带 `worker_id`。** Dynamo 设计的 ZMQ 接口是 +*一个 worker 连一个 ZMQ*:每个引擎 worker 自己跑一个 PUB socket,Dynamo 通过事件从 +哪个 socket 到达来识别来源。但 Mooncake 是中心化的 KV cache 管理——只有 master 掌握 +所有 worker 上每个 block 的状态,因此**一个 ZMQ 就足以把所有信息发过去**。我们有意 +**保持用一个 ZMQ 流**,转而在发布的 event 中**额外增加 `worker_id` 字段**,用来表示 +KV 实际上在哪个物理存储节点;Dynamo 那边之后再写一个 Mooncake 专用的 adapter 来解析 +这个 `worker_id`,从而把每条事件归属到正确的 worker。这样既避免了在一个中心化组件上 +为每个 worker 各起(并管理)一个 socket,又保留了 Dynamo 的 per-worker 路由语义。 +(详见 `mooncake开发方案.md` §7.3。) + +- master 持有**一个 PUB socket**;订阅方(Dynamo adapter)用 SUB socket 连接。 +- 每次 `Publish()` 发送**一个单事件 batch**。该层不做缓冲/合并;排序与去重由上层 + manifest 逻辑负责。 +- 每条事件内的 `worker_id` 告诉消费者该 block 在哪个物理节点。batch 层的 `dp_rank` + 也会被设置(取自事件),但 adapter 应以每条事件的字段为准。 + +### 3.2 线格式(3 帧) + +与 Dynamo 已支持的 SGLang/vLLM ZMQ relay 格式字节兼容 +(`dynamo_kv_event_rules.md` 的 "ZMQ Relay Wire Format" 一节): + +| 帧 | 内容 | 说明 | +|----|------|------| +| 1 | `topic` | 与订阅方的 `zmq_topic` 过滤器匹配;默认空字符串(匹配全部)。listener 要求总帧数恰好为 3。 | +| 2 | 8 字节**大端**序列号 | 每个 publisher 单调递增。Dynamo 仅用于 trace/log,**不**作为内部 `event_id`。 | +| 3 | msgpack payload | `[timestamp (f64), [events], dp_rank (i32 或 nil)]`。 | + +payload 结构: + +```text +[ + timestamp, # f64,自 epoch 起的秒数(engine batch 时间戳) + [ ], # 原始 KV event 数组(此处长度恒为 1) + dp_rank # i32;未知时为 nil +] +``` + +### 3.3 map event 字段 + +每条事件是一个 msgpack **map**(Dynamo 按字段名解析,忽略未知字段)。 +`BlockStored` 携带全部字段;`BlockRemoved` 仅携带身份字段 + `block_hashes` + `medium`。 + +公共字段(两种类型都有): + +| 字段 | 类型 | 含义 | +|------|------|------| +| `type` | str | `"BlockStored"` 或 `"BlockRemoved"`。 | +| `block_hashes` | `[u64]` | 外部 sequence block hash(前缀累计)。 | +| `medium` | str | 存储层级,例如 `"EXTERNAL"`。 | +| `event_id` | str | master 分配的唯一 id(`mooncake::::`)。 | +| `source` | str | 恒为 `"mooncake"`。 | +| `tenant_id` | str | Mooncake 租户。 | +| `group_id` | str | 逻辑 KV block group id。 | +| `worker_id` | str | 持有该 block 的物理 worker(见 §4.2)。 | + +`BlockStored` 专有字段:`parent_block_hash`(u64 或 nil)、`token_ids`(`[u32]`)、 +`block_size`(u32)、`lora_name`(str)、`model_name`(str),以及 `dp_rank`(u32, +仅在事件设置了该值时出现)。 + +> `source`、`tenant_id`、`group_id`、`worker_id`、`event_id` 是在 Dynamo 基础 schema +> 之上的 Mooncake 扩展字段。原生 Dynamo 解码器会忽略未知字段;负责解析 `worker_id` +> 的是 Mooncake 专用 adapter。 + +### 3.4 Publisher C++ API(`kv_event_zmq_publisher.h`) + +```cpp +struct ZmqKvEventPublisherConfig { + std::string endpoint{"tcp://0.0.0.0:5557"}; // "tcp://host:*" 自动选空闲端口 + std::string topic{}; // 帧 1 topic;空表示匹配全部 + bool bind{true}; // true:bind(master 作服务端) + int linger_ms{0}; // 关闭 linger;-1 阻塞,0 丢弃 + int send_hwm{0}; // 发送高水位;0 表示不限 +}; + +class ZmqKvEventPublisher : public KvEventPublisher { + public: + explicit ZmqKvEventPublisher(const ZmqKvEventPublisherConfig& config); + void Publish(const KvEvent& event) override; + const std::string& endpoint() const; // 解析后的端点(含通配端口) + uint64_t published_count() const; // 诊断用 +}; +``` + +在 master 上安装(任何 `KvEventPublisher` 实现均可;传 `nullptr` 关闭发布): + +```cpp +auto publisher = std::make_shared( + ZmqKvEventPublisherConfig{.endpoint = "tcp://0.0.0.0:5557"}); +master_service.SetKvEventPublisher(publisher); +``` + +## 4. 内部实现 + +### 4.1 Group manifest 与事件生命周期(`master_service.{h,cpp}`) + +- 每个携带 `kv_event_metadata` 的 `group_id` 会得到一个 **manifest**,与其成员存放在 + 同一 tenant shard(从而受既有 shard 锁保护)。它跟踪期望的对象 key、当前已到位的 + key,以及 per-worker 的去重记录。 +- `PutEnd` 标记某对象完整;当所有期望 component 到位时,master 恰好发布一条 + `BlockStored`(通过 `stored_event_published_workers` 去重)。 +- 移除/淘汰会使逻辑 block 变为不完整,并发布一条 `BlockRemoved`(通过 + `removed_event_published_workers` 去重);当最后一个 group 成员被移除时擦除 manifest。 +- 校验会拒绝对某 `group_id` 的冲突性重定义,以及非法元数据(空/重复 group id、缺少 + 完整性期望)。 + +### 4.2 segment→worker_id 注册表(内部) + +`worker_id` 必须标识**存储该 block 的物理 worker**,而非用于分配的逻辑 segment 名。 +内部注册表按以下优先级解析: + +1. **显式覆盖** —— `MasterService::RegisterSegmentWorkerId(segment, id)`(部署配置 / + 控制面),优先级最高。 +2. **挂载时端点** —— segment 挂载时自动捕获,取 `Segment::te_endpoint`(节点 ip:port); + 端点为空时回退到 segment 名。 +3. **兜底** —— 注册表中无此 segment 时,用 segment 名本身。 + +实现要点: + +- 两个 map(`segment_worker_explicit_`、`segment_worker_auto_`)由独立的 `shared_mutex` + 保护,从而可在持有 tenant shard 锁时读取。 +- `MountSegment` 在挂载成功并释放 segment 锁后,自动注册 `segment.name → te_endpoint`。 +- `PutEnd` 流程:`segment = DeriveSegmentNameFromMetadata(...)` → + `worker_id = ResolveWorkerId(segment)` → 打入事件。(`GetSegmentWorkerId` 暴露映射 + 供诊断/测试使用。) +- 注册表为内存态,remount 时重建;**尚未**纳入 HA snapshot(见 §7)。 + +### 4.3 事件模型与 ZMQ publisher + +- `KvEvent` / `KvEventType` 以及 msgpack 编解码函数(`EncodeKvEventMap`、 + `EncodeKvEventBatchPayload`、`DecodeKvEventBatchPayload`)位于 `kv_event.{h,cpp}`, + 由 ZMQ publisher 与测试中的内存 mock 共用。 +- `ZmqKvEventPublisher` 采用 **pimpl** 手法,避免 `zmq.hpp` 泄漏进公共头;`Impl` 持有 + `zmq::context_t` 与 PUB `zmq::socket_t`,通过 `ZMQ_LAST_ENDPOINT` 解析具体端点, + 并在 `Publish()` 中构造单事件 batch、前置单调大端序列号,在 mutex 保护下发送 3 帧。 + ZMQ 发送错误仅记录日志并吞掉(发布是 best-effort,绝不能影响数据路径)。 + +## 5. 构建集成 + +`mooncake-store/src/CMakeLists.txt` 自动探测 ZeroMQ(cppzmq 头 `zmq.hpp` + `libzmq`, +带 `$CONDA_PREFIX` 提示路径): + +- 找到时:编译 `kv_event_zmq_publisher.cpp`,并把 `MOONCAKE_STORE_WITH_ZMQ`、include + 目录、链接库作为 `mooncake_store` 的 **PUBLIC** 使用要求传播(测试 / 消费方可继承)。 +- 未找到时:manifest/编码器仍可构建,仅省略实时 ZMQ publisher。ZMQ 相关代码请用 + `#ifdef MOONCAKE_STORE_WITH_ZMQ` 包裹。 + +依赖(例如 conda 环境):`cppzmq` + `zeromq`。 + +## 6. 测试 + +`mooncake-store/tests/master_service_test.cpp`: + +- manifest 生命周期:创建/完整、跨 group 成员共享、冲突拒绝、group 移除时擦除、 + config 校验。 +- 发布:完整时一条 `BlockStored`、重复 `PutEnd` 不重复发布、删除时一条 `BlockRemoved`、 + 无 publisher 时为空操作。 +- worker_id:`SegmentWorkerRegistryCapturesEndpointAtMount`、 + `KvEventWorkerIdReflectsSegmentEndpoint`、`KvEventWorkerIdExplicitOverrideWins`。 +- 传输:`KvEventEncoderRoundTrip`,以及 `ZmqKvEventPublisherRoundTrip` + (由 `MOONCAKE_STORE_WITH_ZMQ` 保护)——真实 SUB socket 经 TCP 收到 3 帧并解码还原为 + 原始 `KvEvent`。 + +## 7. 局限与后续工作 + +### 7.1 本次改动的局限 + +- **HA snapshot**:KV group manifest 与 segment→worker_id 注册表均尚未纳入 master HA + snapshot。故障切换后注册表可由 remount 重建;但 manifest 去重状态仍需 snapshot 支持 + 才能在跨切换时完全正确。 +- **批处理**:当前一条 ZMQ 消息一个事件。若吞吐需要,后续可在单 payload 内批量多事件 + (编码器已支持)。 + +下面两块用于打通端到端链路,但位于各自仓库(SGLang / Dynamo),不在 Mooncake 本次 PR +范围内;在此记录是为了让完整数据流清晰。 + +### 7.2 SGLang → Mooncake 接入(引擎侧,独立仓库) + +输入接口(§2)的 Mooncake 侧已完成,但 SGLang 尚未填充。SGLang 仓库的后续工作: + +- 在 KV-cache 写入时,为每个逻辑 block 构造一个 `KvBlockEventMetadata`,连同匹配的 + `group_ids` 一起附加到传给 `store.put` / batch put 的 `ReplicateConfig` 上。 +- 将 SGLang block manager 的状态映射到 schema:外部 **sequence** block hash → + `block_hash` / `parent_block_hash`(前缀累计 hash,而非单 block 的 local hash), + block 的 `token_ids`,以及 `block_size`(须等于 Dynamo 的 `kv_block_size`)。 +- 把每个 block 的物理对象枚举为 `expected_components`(`object_key` + + `component_role` + `component_index`),例如一个 MHA page 的 K 与 V 对象,或 per-head + component。为路由命名空间设置 `model_name` / `lora_name` / `dp_rank` / + `additional_salt`。 +- 按部署决定 `emit_stored_event` / `emit_removed_event`,并确保对同一 `group_id` 复用 + (幂等重试)而非重定义。 + +### 7.3 Dynamo Mooncake adapter(router 侧,独立仓库) + +由中心化 publisher 用一条 ZMQ 流承载事件,需要 Dynamo 侧的 Mooncake 专用 adapter +(对应 §3.1 的设计决策)。Dynamo 仓库的后续工作: + +- 用 SUB socket 与匹配的 `zmq_topic` 订阅 Mooncake 的单一 PUB 端点,解码 3 帧消息与 + `[timestamp, [events], dp_rank]` payload。 +- **解析每条事件的 `worker_id`**,据此把每条 `BlockStored` / `BlockRemoved` 归属到对应 + 物理 worker,而不是从 socket/连接推断 worker(默认 one-socket-per-worker 路径所内置的 + 假设)。应以每条事件的 `worker_id`(与 `dp_rank`)为准,而非 batch 末尾的字段。 +- 将 Mooncake 扩展字段(`source`、`tenant_id`、`group_id`、`event_id`)映射进 Dynamo + 内部的 `RouterEvent` 模型,并把 `medium` → storage tier。遵守 Dynamo 自身的 + `block_size == kv_block_size` 以及重新编号/去重规则。 +- 帧 2 的序列号仅作 trace/log;排序依赖 Mooncake 的 `event_id` 与 Dynamo 自身的单调 id。 + +## 8. 改动 / 新增文件 + +| 文件 | 改动 | +|------|------| +| `include/replica.h` | `KvBlockComponentSpec`、`KvBlockEventMetadata`、`ReplicateConfig::kv_event_metadata`。 | +| `mooncake-integration/store/store_py.cpp` | 元数据类型 + `kv_event_metadata` 的 Python 绑定。 | +| `include/kv_event.h`、`src/kv_event.cpp` | 事件模型 + msgpack 编解码。 | +| `include/kv_event_zmq_publisher.h`、`src/kv_event_zmq_publisher.cpp` | **新增。** ZMQ PUB 传输(pimpl)。 | +| `include/master_service.h`、`src/master_service.cpp` | group manifest、事件发布/去重、segment→worker_id 注册表、`SetKvEventPublisher` / `Register`/`GetSegmentWorkerId`。 | +| `src/rpc_service.cpp` | RPC 入口处的 `kv_event_metadata` config 级校验。 | +| `src/CMakeLists.txt` | ZeroMQ 探测;条件源文件 + PUBLIC include/lib/`MOONCAKE_STORE_WITH_ZMQ`。 | +| `tests/master_service_test.cpp` | manifest、publisher、worker_id、真实 ZMQ 往返测试。 | diff --git a/mooncake-integration/store/store_py.cpp b/mooncake-integration/store/store_py.cpp index aab9ed35570..935fd28e401 100644 --- a/mooncake-integration/store/store_py.cpp +++ b/mooncake-integration/store/store_py.cpp @@ -1786,6 +1786,36 @@ PYBIND11_MODULE(store, m) { .value("GENERAL", ObjectDataType::GENERAL) .export_values(); + // KV event metadata types attached to writes via ReplicateConfig. + py::class_(m, "KvBlockComponentSpec") + .def(py::init<>()) + .def_readwrite("object_key", &KvBlockComponentSpec::object_key) + .def_readwrite("component_role", &KvBlockComponentSpec::component_role) + .def_readwrite("component_index", &KvBlockComponentSpec::component_index); + + py::class_(m, "KvBlockEventMetadata") + .def(py::init<>()) + .def_readwrite("schema_version", &KvBlockEventMetadata::schema_version) + .def_readwrite("group_id", &KvBlockEventMetadata::group_id) + .def_readwrite("block_hash", &KvBlockEventMetadata::block_hash) + .def_readwrite("parent_block_hash", + &KvBlockEventMetadata::parent_block_hash) + .def_readwrite("token_ids", &KvBlockEventMetadata::token_ids) + .def_readwrite("block_size", &KvBlockEventMetadata::block_size) + .def_readwrite("dp_rank", &KvBlockEventMetadata::dp_rank) + .def_readwrite("model_name", &KvBlockEventMetadata::model_name) + .def_readwrite("lora_name", &KvBlockEventMetadata::lora_name) + .def_readwrite("additional_salt", + &KvBlockEventMetadata::additional_salt) + .def_readwrite("expected_object_count", + &KvBlockEventMetadata::expected_object_count) + .def_readwrite("expected_components", + &KvBlockEventMetadata::expected_components) + .def_readwrite("emit_stored_event", + &KvBlockEventMetadata::emit_stored_event) + .def_readwrite("emit_removed_event", + &KvBlockEventMetadata::emit_removed_event); + // Define the ReplicateConfig class py::class_(m, "ReplicateConfig") .def(py::init<>()) @@ -1802,6 +1832,8 @@ PYBIND11_MODULE(store, m) { &ReplicateConfig::prefer_alloc_in_same_node) .def_readwrite("data_type", &ReplicateConfig::data_type) .def_readwrite("group_ids", &ReplicateConfig::group_ids) + .def_readwrite("kv_event_metadata", + &ReplicateConfig::kv_event_metadata) .def("__str__", [](const ReplicateConfig &config) { std::ostringstream oss; oss << config; diff --git a/mooncake-store/include/kv_event.h b/mooncake-store/include/kv_event.h new file mode 100644 index 00000000000..638b4dd14e8 --- /dev/null +++ b/mooncake-store/include/kv_event.h @@ -0,0 +1,115 @@ +#pragma once + +#include +#include +#include +#include + +namespace mooncake { + +/** + * @brief Type of a Dynamo-compatible KV cache event published by Mooncake. + */ +enum class KvEventType { + kBlockStored, + kBlockRemoved, +}; + +/** + * @brief A single Dynamo-compatible KV cache event. + * + * Field semantics follow the Dynamo ZMQ relay "map event" wire format plus the + * Mooncake extension fields (source / tenant_id / group_id / worker_id / + * event_id). The encoder in kv_event.cpp serializes this into the msgpack map + * that the Dynamo Mooncake adapter consumes. + * + * BlockStored uses all fields; BlockRemoved only needs the identity fields plus + * block_hashes and medium (token_ids/block_size/parent_block_hash are ignored). + */ +struct KvEvent { + KvEventType type{KvEventType::kBlockStored}; + + // Identity / Mooncake extension fields. + std::string event_id; + std::string source{"mooncake"}; + std::string tenant_id; + std::string group_id; + std::string worker_id; + + // Dynamo core fields. + std::vector block_hashes; + std::string medium{"EXTERNAL"}; + + // BlockStored-only fields. + std::optional parent_block_hash; + std::vector token_ids; + uint32_t block_size{0}; + std::string lora_name; + std::string model_name; + std::optional dp_rank; + + bool operator==(const KvEvent& other) const { + return type == other.type && event_id == other.event_id && + source == other.source && tenant_id == other.tenant_id && + group_id == other.group_id && worker_id == other.worker_id && + block_hashes == other.block_hashes && medium == other.medium && + parent_block_hash == other.parent_block_hash && + token_ids == other.token_ids && block_size == other.block_size && + lora_name == other.lora_name && model_name == other.model_name && + dp_rank == other.dp_rank; + } +}; + +/** + * @brief Abstract sink for Dynamo-compatible KV events. + * + * The master calls Publish() when a logical KV block transitions between + * complete/incomplete. Implementations decide the transport (e.g. a single ZMQ + * PUB stream). Tests use an in-memory mock. A null publisher means events are + * disabled and Mooncake behavior is unchanged. + */ +class KvEventPublisher { + public: + virtual ~KvEventPublisher() = default; + virtual void Publish(const KvEvent& event) = 0; +}; + +// --- Encoding / decoding (msgpack) ----------------------------------------- +// +// These produce / consume the Dynamo ZMQ relay wire format. Returning byte +// strings keeps msgpack out of widely-included headers. + +/** + * @brief Encode one event into the Dynamo "map event" msgpack representation. + */ +std::string EncodeKvEventMap(const KvEvent& event); + +/** + * @brief Encode a batch of events into the ZMQ relay payload: + * [timestamp (f64), [events], dp_rank (i32, optional)]. + */ +std::string EncodeKvEventBatchPayload(const std::vector& events, + double timestamp, + std::optional dp_rank); + +/** + * @brief Decode one msgpack "map event". Returns std::nullopt on malformed + * input or unsupported event type. Intended for tests / adapters. + */ +std::optional DecodeKvEventMap(const std::string& bytes); + +/** + * @brief Result of decoding a ZMQ relay batch payload. + */ +struct KvEventBatch { + double timestamp{0.0}; + std::vector events; + std::optional dp_rank; +}; + +/** + * @brief Decode a ZMQ relay payload produced by EncodeKvEventBatchPayload. + */ +std::optional DecodeKvEventBatchPayload(const std::string& bytes); + +} // namespace mooncake diff --git a/mooncake-store/include/kv_event_zmq_publisher.h b/mooncake-store/include/kv_event_zmq_publisher.h new file mode 100644 index 00000000000..666f5d5b1a1 --- /dev/null +++ b/mooncake-store/include/kv_event_zmq_publisher.h @@ -0,0 +1,72 @@ +#pragma once + +#include +#include +#include +#include + +#include "kv_event.h" + +namespace mooncake { + +/** + * @brief Configuration for ZmqKvEventPublisher. + */ +struct ZmqKvEventPublisherConfig { + // ZMQ endpoint to publish on, e.g. "tcp://0.0.0.0:5557". Use a "*" port + // (e.g. "tcp://127.0.0.1:*") to let ZMQ choose a free port; the chosen + // endpoint can be read back via ZmqKvEventPublisher::endpoint(). + std::string endpoint{"tcp://0.0.0.0:5557"}; + // ZMQ topic prefix sent as frame 1. Dynamo's listener matches this against + // its zmq_topic subscription filter; default is empty (matches all). + std::string topic{}; + // When true (default) the PUB socket binds the endpoint (Mooncake master is + // the stable server). When false it connects instead (for relays/proxies). + bool bind{true}; + // Linger in milliseconds applied on close so a clean shutdown can flush + // buffered events. -1 keeps the ZMQ default (block forever); 0 drops. + int linger_ms{0}; + // Send high-water-mark (max queued outbound messages); 0 means unlimited. + int send_hwm{0}; +}; + +/** + * @brief KvEventPublisher backed by a single ZMQ PUB socket. + * + * Emits the SGLang/vLLM-compatible 3-frame message Dynamo's ZMQ relay expects: + * frame 1: topic + * frame 2: 8-byte big-endian monotonic sequence number + * frame 3: msgpack payload [timestamp, [event], dp_rank] + * + * Because Mooncake's master is the single centralized owner of KV state, all + * workers' events flow through this one PUB stream; the physical owner of each + * block is carried in the per-event worker_id field (see kv_event.h). Each + * Publish() call sends a single-event batch. + * + * Thread-safety: Publish() is serialized with an internal mutex, so the + * publisher may be shared across master threads. + */ +class ZmqKvEventPublisher : public KvEventPublisher { + public: + explicit ZmqKvEventPublisher(const ZmqKvEventPublisherConfig& config); + ~ZmqKvEventPublisher() override; + + ZmqKvEventPublisher(const ZmqKvEventPublisher&) = delete; + ZmqKvEventPublisher& operator=(const ZmqKvEventPublisher&) = delete; + + void Publish(const KvEvent& event) override; + + // The actual bound/connected endpoint. When the config used a wildcard + // port (e.g. "tcp://127.0.0.1:*"), this returns the concrete endpoint ZMQ + // selected (e.g. "tcp://127.0.0.1:54321"). + const std::string& endpoint() const; + + // Number of messages successfully handed to ZMQ so far (for diagnostics). + uint64_t published_count() const; + + private: + struct Impl; + std::unique_ptr impl_; +}; + +} // namespace mooncake diff --git a/mooncake-store/include/master_service.h b/mooncake-store/include/master_service.h index 9043d592862..6728325904d 100644 --- a/mooncake-store/include/master_service.h +++ b/mooncake-store/include/master_service.h @@ -32,6 +32,7 @@ #include "master_config.h" #include "rpc_types.h" #include "replica.h" +#include "kv_event.h" #include "ha/ha_types.h" #include "ha/snapshot/object/snapshot_object_store.h" #include "task_manager.h" @@ -49,6 +50,18 @@ class EvictionStrategy; class HttpMetadataServer; struct MetadataStoragePlugin; +// Public read-only snapshot of a KV group manifest, returned by +// MasterService::GetKvGroupManifest. Sets are exposed as sorted vectors so +// callers (and tests) get a deterministic view without touching internal state. +struct KvGroupManifestView { + std::string tenant_id; + std::string group_id; + KvBlockEventMetadata metadata; + std::vector expected_object_keys; // sorted + std::vector current_object_keys; // completed, sorted + uint64_t generation{0}; +}; + // Forward declarations for test classes namespace test { class MasterServiceSnapshotTestBase; @@ -324,6 +337,65 @@ class MasterService { BatchGetReplicaList(const std::vector& keys, const std::string& tenant_id); + /** + * @brief Look up the KV event group manifest for a logical block group. + * + * Returns std::nullopt when no manifest exists for the (tenant, group) + * pair, e.g. when the writes did not carry kv_event_metadata. Intended for + * KV event bookkeeping and tests; it does not publish any event. + */ + std::optional GetKvGroupManifest( + const std::string& tenant_id, const std::string& group_id) const; + + /** + * @brief Validate the KV event metadata carried by a full (non per-key) + * ReplicateConfig. + * + * Checks that every metadata item has a non-empty, unique group_id that is + * present in config.group_ids (when group_ids is set), and that each item + * can express completeness (expected_components or expected_object_count). + * Returns INVALID_PARAMS otherwise. A no-op (OK) when no metadata is set. + * + * This is a config-consistency check meant for the batch/RPC entry layer, + * which still sees the full group_ids list; per-key manifest validation is + * handled separately inside PutStart. + */ + static tl::expected ValidateKvEventMetadataConfig( + const ReplicateConfig& config); + + /** + * @brief Install the sink for Dynamo-compatible KV events. + * + * Pass nullptr (the default) to disable publishing, which keeps Mooncake + * behavior unchanged. Intended to be set once before serving traffic (e.g. + * at startup or in tests with a mock publisher). + */ + void SetKvEventPublisher(std::shared_ptr publisher); + + /** + * @brief Register / override the worker_id reported for a segment in KV + * events. + * + * This is the deployment-config / control-plane entry point for the + * segment->worker_id registry. An explicit registration takes precedence + * over the transport endpoint captured automatically at mount time, so a + * deployment can map a segment to a logical worker identity that differs + * from its te_endpoint. Passing an empty worker_id clears the explicit + * override (falling back to the mount-time value). Thread-safe. + */ + void RegisterSegmentWorkerId(const std::string& segment_name, + const std::string& worker_id); + + /** + * @brief Look up the worker_id currently associated with a segment name. + * + * Returns the explicit override when present, otherwise the worker_id + * captured at mount time (te_endpoint), or std::nullopt when the segment is + * unknown to the registry. Intended for diagnostics and tests. + */ + std::optional GetSegmentWorkerId( + const std::string& segment_name) const; + /** * @brief Start a put operation for an object * @param[out] replica_list Vector to store replica information for the @@ -1197,6 +1269,35 @@ class MasterService { static constexpr size_t kNumShards = 1024; // Number of metadata shards + // Master-internal state tracking one logical Dynamo KV block group that + // opted into KV events. Lives in the same shard as the group's members + // (grouped keys are sharded by group_id), so it is protected by that + // shard's mutex. No event is published in this phase; this only models the + // group's completeness state. + struct KvGroupManifest { + std::string tenant_id; + std::string group_id; + KvBlockEventMetadata metadata; + + // Expected physical object keys derived from metadata when + // expected_components is provided; may be empty if only + // expected_object_count is set. + std::unordered_set expected_object_keys; + // Physical object keys whose write has completed (PutEnd). + std::unordered_set current_object_keys; + + // Physical placement: object_key -> owning worker / storage node id. + // Populated in a later phase (segment -> worker reverse lookup). + std::unordered_map object_worker_ids; + + // Per-worker event publish bookkeeping, reserved for the publisher + // phase. Unused while events are not emitted. + std::unordered_set stored_event_published_workers; + std::unordered_set removed_event_published_workers; + + uint64_t generation{0}; + }; + struct TenantState { std::unordered_map metadata; std::unordered_set processing_keys; @@ -1208,10 +1309,14 @@ class MasterService { std::unordered_map> group_members; // group_id → set of keys + std::unordered_map + kv_group_manifests; // group_id → manifest + bool Empty() const { return metadata.empty() && processing_keys.empty() && replication_tasks.empty() && offloading_tasks.empty() && - promotion_tasks.empty() && group_members.empty(); + promotion_tasks.empty() && group_members.empty() && + kv_group_manifests.empty(); } }; @@ -1377,6 +1482,74 @@ class MasterService { const std::string& tenant_id, const std::string& key, const std::string& group_id); + + // KV event manifest helpers. All operate on the group's shard tenant_state + // (caller must hold that shard's write lock). + // + // Finds the KvBlockEventMetadata entry in config whose group_id matches the + // given group. Returns nullptr when config carries no metadata for it. + static const KvBlockEventMetadata* FindKvEventMetadataForGroup( + const ReplicateConfig& config, const std::string& group_id); + // Validates the metadata for a key being put into group_id. Returns an + // error for structurally invalid metadata or a conflicting redefinition of + // an existing group manifest. A no-op (OK) when no metadata applies. + tl::expected ValidateKvManifestForKey( + const TenantState& tenant_state, const std::string& tenant_id, + const std::string& group_id, const std::string& key, + const ReplicateConfig& config) const; + // Creates the group manifest if it does not exist yet. Assumes validation + // already passed (idempotent for repeated identical metadata). + void UpsertKvManifestForKey(TenantState& tenant_state, + const std::string& tenant_id, + const std::string& group_id, + const std::string& key, + const ReplicateConfig& config); + // Marks a physical object key as completed in its group manifest, if any, + // records its owning worker, and publishes a BlockStored event if the + // logical block just became complete. + void MarkKvManifestObjectComplete(TenantState& tenant_state, + const std::string& group_id, + const std::string& key, + const std::string& worker_id); + // Handles removal/eviction of a physical object: publishes a BlockRemoved + // event if the logical block was previously stored and is now affected. + void OnKvManifestObjectRemoved(TenantState& tenant_state, + const std::string& group_id, + const std::string& key); + // Whether all expected objects of a group are completed. + static bool IsKvManifestComplete(const KvGroupManifest& manifest); + // Publishes BlockStored / BlockRemoved with dedup, mutating the manifest's + // published-worker bookkeeping. No-ops when no publisher is installed. + void MaybePublishKvBlockStored(KvGroupManifest& manifest, + const std::string& worker_id); + void MaybePublishKvBlockRemoved(KvGroupManifest& manifest); + // Derives the segment name that owns an object from its replica placement. + // Returns empty when placement is unknown. + static std::string DeriveSegmentNameFromMetadata( + const ObjectMetadata& metadata); + // Resolves a segment name to the worker_id reported in KV events using the + // segment->worker_id registry, falling back to the segment name itself when + // the segment is unregistered. + std::string ResolveWorkerId(const std::string& segment_name) const; + // Records the worker_id captured for a segment at mount time. Does not + // overwrite an explicit registration. No-op for empty inputs. + void AutoRegisterSegmentWorkerId(const std::string& segment_name, + const std::string& worker_id); + // Builds a unique event_id string for a manifest event. + std::string MakeKvEventId(const std::string& tenant_id, + const std::string& group_id, const char* kind); + + std::shared_ptr kv_event_publisher_; + std::atomic kv_event_seq_{0}; + // segment->worker_id registry, decoupled from KV-event manifest sharding so + // it can be consulted while holding a tenant shard lock. The explicit map + // is populated by RegisterSegmentWorkerId (deployment config / control + // plane) and wins over the auto map, which is populated at mount time with + // the segment's transport endpoint. Repopulated on remount; not part of the + // HA snapshot. + mutable std::shared_mutex segment_worker_registry_mutex_; + std::unordered_map segment_worker_explicit_; + std::unordered_map segment_worker_auto_; std::unordered_map::iterator EraseMetadata( TenantState& tenant_state, std::unordered_map::iterator it, diff --git a/mooncake-store/include/replica.h b/mooncake-store/include/replica.h index 677e77b3333..308b3ccc32d 100644 --- a/mooncake-store/include/replica.h +++ b/mooncake-store/include/replica.h @@ -75,6 +75,98 @@ inline std::ostream& operator<<(std::ostream& os, return os; } +/** + * @brief Describes one physical Mooncake object that belongs to a logical + * Dynamo KV block group. + * + * A single logical KV block (e.g. one SGLang prefix page) may be stored as + * multiple physical objects, such as the K and V objects of an MHA page, or + * several per-head component objects. Each such object is described by one + * KvBlockComponentSpec. + */ +struct KvBlockComponentSpec { + std::string object_key; // Physical Mooncake object key. + std::string component_role; // e.g. "k", "v", "mla_k", "head_3_k". + uint32_t component_index{0}; + + bool operator==(const KvBlockComponentSpec& other) const { + return object_key == other.object_key && + component_role == other.component_role && + component_index == other.component_index; + } + bool operator!=(const KvBlockComponentSpec& other) const { + return !(*this == other); + } +}; + +/** + * @brief Group-level KV event metadata for a single logical Dynamo KV block. + * + * This is attached to a write (Put/BatchPut/Upsert) via ReplicateConfig and + * carries the semantic information that only the inference engine (SGLang) + * knows but Mooncake needs in order to publish Dynamo-compatible KV events. + * + * One KvBlockEventMetadata item corresponds to exactly one group_id. That + * group_id may appear for multiple physical object keys (see + * ReplicateConfig::group_ids). + */ +struct KvBlockEventMetadata { + uint32_t schema_version{1}; + + // Logical group identity. Must match an entry in ReplicateConfig::group_ids. + std::string group_id; + + // Dynamo logical block identity. + uint64_t block_hash{0}; + std::optional parent_block_hash; + + // Token semantic information required by Dynamo. + std::vector token_ids; + uint32_t block_size{0}; + + // Optional routing namespace fields. + std::optional dp_rank; + std::string model_name; + std::string lora_name; + std::string additional_salt; + + // Physical layout expectation: how many / which objects must be present + // before this logical block is considered fully stored. + uint32_t expected_object_count{0}; + std::vector expected_components; + + // Event behavior. + bool emit_stored_event{true}; + bool emit_removed_event{true}; + + // Returns the set of expected physical object keys. When + // expected_components is non-empty it is derived from there; otherwise it + // is empty and completeness has to rely on expected_object_count. + std::vector ExpectedObjectKeys() const { + std::vector keys; + keys.reserve(expected_components.size()); + for (const auto& component : expected_components) { + keys.push_back(component.object_key); + } + return keys; + } + + // Semantic equality used to decide whether a repeated write for the same + // group_id is an idempotent retry or a conflicting redefinition. Event + // behavior flags are intentionally ignored here. + bool SameSemanticsAs(const KvBlockEventMetadata& other) const { + return schema_version == other.schema_version && + group_id == other.group_id && block_hash == other.block_hash && + parent_block_hash == other.parent_block_hash && + token_ids == other.token_ids && block_size == other.block_size && + dp_rank == other.dp_rank && model_name == other.model_name && + lora_name == other.lora_name && + additional_salt == other.additional_salt && + expected_object_count == other.expected_object_count && + expected_components == other.expected_components; + } +}; + /** * @brief Configuration for replica management */ @@ -96,12 +188,21 @@ struct ReplicateConfig { // and memory eviction behavior. std::optional> group_ids{}; + // Optional group-level KV event metadata. This is NOT per-key: each item + // describes one logical Dynamo KV block keyed by its group_id, and that + // group_id may be shared by multiple physical object keys in group_ids. + // When absent, Mooncake behavior is completely unchanged. + std::optional> kv_event_metadata{}; + ReplicateConfig ForSingleKey(size_t key_index) const { ReplicateConfig key_config = *this; if (group_ids.has_value()) { key_config.group_ids = std::vector{group_ids->at(key_index)}; } + // kv_event_metadata is group-level, not per-key, so it is intentionally + // carried through unchanged; the per-key PutStart looks up the entry + // whose group_id matches the key's group. return key_config; } @@ -138,6 +239,20 @@ struct ReplicateConfig { } os << "]"; } + if (config.kv_event_metadata.has_value()) { + os << ", kv_event_metadata: ["; + for (size_t i = 0; i < config.kv_event_metadata->size(); ++i) { + const auto& meta = config.kv_event_metadata->at(i); + os << "{group_id: " << meta.group_id + << ", block_hash: " << meta.block_hash + << ", block_size: " << meta.block_size + << ", expected_object_count: " << meta.expected_object_count + << ", expected_components: " << meta.expected_components.size() + << "}"; + if (i + 1 < config.kv_event_metadata->size()) os << ", "; + } + os << "]"; + } os << " }"; return os; } diff --git a/mooncake-store/src/CMakeLists.txt b/mooncake-store/src/CMakeLists.txt index e2c2830c38d..a00eb9aa01b 100644 --- a/mooncake-store/src/CMakeLists.txt +++ b/mooncake-store/src/CMakeLists.txt @@ -8,6 +8,7 @@ set(MOONCAKE_STORE_SOURCES client_metric.cpp types.cpp master_client.cpp + kv_event.cpp utils.cpp mmap_arena.cpp master_metric_manager.cpp @@ -116,6 +117,30 @@ else() ) endif() +# ZeroMQ (optional): enables the real ZMQ transport for Dynamo-compatible KV +# events (kv_event_zmq_publisher.cpp). cppzmq (zmq.hpp) is a header-only C++ +# binding over libzmq. When absent, the KV-event manifest/encoder still build; +# only the live ZMQ publisher is omitted. +find_path( + ZMQ_INCLUDE_DIR + NAMES zmq.hpp + HINTS $ENV{CONDA_PREFIX}/include + PATHS /usr/include /usr/local/include) +find_library( + ZMQ_LIBRARY + NAMES zmq libzmq + HINTS $ENV{CONDA_PREFIX}/lib + PATHS /usr/lib /usr/local/lib /usr/lib64) +if(ZMQ_INCLUDE_DIR AND ZMQ_LIBRARY) + message(STATUS "Found ZeroMQ: include=${ZMQ_INCLUDE_DIR} lib=${ZMQ_LIBRARY}") + set(MOONCAKE_STORE_HAS_ZMQ TRUE) + list(APPEND MOONCAKE_STORE_SOURCES kv_event_zmq_publisher.cpp) +else() + message( + STATUS "ZeroMQ not found, real ZMQ KV-event transport will be disabled") + set(MOONCAKE_STORE_HAS_ZMQ FALSE) +endif() + if(USE_3FS) add_subdirectory(hf3fs) list(APPEND MOONCAKE_STORE_SOURCES ${HF3FS_SOURCES}) @@ -272,6 +297,14 @@ if(URING_LIB AND URING_INCLUDE) target_include_directories(mooncake_store PRIVATE ${URING_INCLUDE}) endif() +# Propagate ZeroMQ to mooncake_store (and, via PUBLIC, to tests/consumers that +# need to build against the real ZMQ KV-event publisher). +if(MOONCAKE_STORE_HAS_ZMQ) + target_compile_definitions(mooncake_store PUBLIC MOONCAKE_STORE_WITH_ZMQ) + target_include_directories(mooncake_store PUBLIC ${ZMQ_INCLUDE_DIR}) + target_link_libraries(mooncake_store PUBLIC ${ZMQ_LIBRARY}) +endif() + if(BUILD_SHARED_LIBS) install(TARGETS mooncake_store DESTINATION lib) endif() diff --git a/mooncake-store/src/kv_event.cpp b/mooncake-store/src/kv_event.cpp new file mode 100644 index 00000000000..079f8f80fd7 --- /dev/null +++ b/mooncake-store/src/kv_event.cpp @@ -0,0 +1,259 @@ +#include "kv_event.h" + +#include +#include + +#include +#include + +namespace mooncake { + +namespace { + +constexpr const char* kBlockStoredTag = "BlockStored"; +constexpr const char* kBlockRemovedTag = "BlockRemoved"; + +// Packs a single event as a Dynamo "map event" into the given packer. Only the +// fields relevant to the event type are emitted. +template +void PackEventMap(Packer& packer, const KvEvent& event) { + const bool is_stored = event.type == KvEventType::kBlockStored; + + // Common fields shared by both event types. + // type, block_hashes, medium, event_id, source, tenant_id, group_id, + // worker_id => 8 fields. Stored adds parent_block_hash, token_ids, + // block_size, lora_name, model_name, and optionally dp_rank. + uint32_t field_count = 8; + if (is_stored) { + field_count += 5; // parent_block_hash, token_ids, block_size, + // lora_name, model_name + if (event.dp_rank.has_value()) { + field_count += 1; + } + } + + packer.pack_map(field_count); + + packer.pack(std::string("type")); + packer.pack(std::string(is_stored ? kBlockStoredTag : kBlockRemovedTag)); + + packer.pack(std::string("block_hashes")); + packer.pack(event.block_hashes); + + packer.pack(std::string("medium")); + packer.pack(event.medium); + + packer.pack(std::string("event_id")); + packer.pack(event.event_id); + + packer.pack(std::string("source")); + packer.pack(event.source); + + packer.pack(std::string("tenant_id")); + packer.pack(event.tenant_id); + + packer.pack(std::string("group_id")); + packer.pack(event.group_id); + + packer.pack(std::string("worker_id")); + packer.pack(event.worker_id); + + if (is_stored) { + packer.pack(std::string("parent_block_hash")); + if (event.parent_block_hash.has_value()) { + packer.pack(event.parent_block_hash.value()); + } else { + packer.pack_nil(); + } + + packer.pack(std::string("token_ids")); + packer.pack(event.token_ids); + + packer.pack(std::string("block_size")); + packer.pack(event.block_size); + + packer.pack(std::string("lora_name")); + packer.pack(event.lora_name); + + packer.pack(std::string("model_name")); + packer.pack(event.model_name); + + if (event.dp_rank.has_value()) { + packer.pack(std::string("dp_rank")); + packer.pack(event.dp_rank.value()); + } + } +} + +bool ObjectToString(const msgpack::object& obj, std::string& out) { + if (obj.type != msgpack::type::STR) { + return false; + } + out.assign(obj.via.str.ptr, obj.via.str.size); + return true; +} + +// Parses a single msgpack map object into a KvEvent. Unknown keys are ignored, +// mirroring the Dynamo decoder's tolerant behavior. +bool DecodeEventObject(const msgpack::object& obj, KvEvent& out) { + if (obj.type != msgpack::type::MAP) { + return false; + } + + std::string type_tag; + bool has_type = false; + KvEvent event; + + for (uint32_t i = 0; i < obj.via.map.size; ++i) { + const msgpack::object& key_obj = obj.via.map.ptr[i].key; + const msgpack::object& val_obj = obj.via.map.ptr[i].val; + std::string key; + if (!ObjectToString(key_obj, key)) { + continue; + } + + try { + if (key == "type") { + has_type = ObjectToString(val_obj, type_tag); + } else if (key == "block_hashes") { + if (val_obj.type == msgpack::type::ARRAY) { + val_obj.convert(event.block_hashes); + } + } else if (key == "medium") { + ObjectToString(val_obj, event.medium); + } else if (key == "event_id") { + ObjectToString(val_obj, event.event_id); + } else if (key == "source") { + ObjectToString(val_obj, event.source); + } else if (key == "tenant_id") { + ObjectToString(val_obj, event.tenant_id); + } else if (key == "group_id") { + ObjectToString(val_obj, event.group_id); + } else if (key == "worker_id") { + ObjectToString(val_obj, event.worker_id); + } else if (key == "parent_block_hash") { + if (val_obj.type == msgpack::type::NIL) { + event.parent_block_hash = std::nullopt; + } else { + event.parent_block_hash = val_obj.as(); + } + } else if (key == "token_ids") { + if (val_obj.type == msgpack::type::ARRAY) { + val_obj.convert(event.token_ids); + } + } else if (key == "block_size") { + event.block_size = val_obj.as(); + } else if (key == "lora_name") { + ObjectToString(val_obj, event.lora_name); + } else if (key == "model_name") { + ObjectToString(val_obj, event.model_name); + } else if (key == "dp_rank") { + if (val_obj.type != msgpack::type::NIL) { + event.dp_rank = val_obj.as(); + } + } + } catch (const std::exception& e) { + LOG(ERROR) << "Failed to decode kv event field '" << key + << "': " << e.what(); + return false; + } + } + + if (!has_type) { + return false; + } + if (type_tag == kBlockStoredTag) { + event.type = KvEventType::kBlockStored; + } else if (type_tag == kBlockRemovedTag) { + event.type = KvEventType::kBlockRemoved; + } else { + return false; + } + + out = std::move(event); + return true; +} + +} // namespace + +std::string EncodeKvEventMap(const KvEvent& event) { + msgpack::sbuffer sbuf; + msgpack::packer packer(&sbuf); + PackEventMap(packer, event); + return std::string(sbuf.data(), sbuf.size()); +} + +std::string EncodeKvEventBatchPayload(const std::vector& events, + double timestamp, + std::optional dp_rank) { + msgpack::sbuffer sbuf; + msgpack::packer packer(&sbuf); + + packer.pack_array(3); + packer.pack(timestamp); + + packer.pack_array(static_cast(events.size())); + for (const auto& event : events) { + PackEventMap(packer, event); + } + + if (dp_rank.has_value()) { + packer.pack(dp_rank.value()); + } else { + packer.pack_nil(); + } + + return std::string(sbuf.data(), sbuf.size()); +} + +std::optional DecodeKvEventMap(const std::string& bytes) { + try { + msgpack::object_handle oh = msgpack::unpack(bytes.data(), bytes.size()); + KvEvent event; + if (!DecodeEventObject(oh.get(), event)) { + return std::nullopt; + } + return event; + } catch (const std::exception& e) { + LOG(ERROR) << "Failed to unpack kv event map: " << e.what(); + return std::nullopt; + } +} + +std::optional DecodeKvEventBatchPayload(const std::string& bytes) { + try { + msgpack::object_handle oh = msgpack::unpack(bytes.data(), bytes.size()); + const msgpack::object& obj = oh.get(); + if (obj.type != msgpack::type::ARRAY || obj.via.array.size != 3) { + return std::nullopt; + } + + KvEventBatch batch; + const msgpack::object& ts_obj = obj.via.array.ptr[0]; + batch.timestamp = ts_obj.as(); + + const msgpack::object& events_obj = obj.via.array.ptr[1]; + if (events_obj.type != msgpack::type::ARRAY) { + return std::nullopt; + } + for (uint32_t i = 0; i < events_obj.via.array.size; ++i) { + KvEvent event; + if (!DecodeEventObject(events_obj.via.array.ptr[i], event)) { + return std::nullopt; + } + batch.events.push_back(std::move(event)); + } + + const msgpack::object& dp_obj = obj.via.array.ptr[2]; + if (dp_obj.type != msgpack::type::NIL) { + batch.dp_rank = dp_obj.as(); + } + + return batch; + } catch (const std::exception& e) { + LOG(ERROR) << "Failed to unpack kv event batch: " << e.what(); + return std::nullopt; + } +} + +} // namespace mooncake diff --git a/mooncake-store/src/kv_event_zmq_publisher.cpp b/mooncake-store/src/kv_event_zmq_publisher.cpp new file mode 100644 index 00000000000..b084cbd1bb2 --- /dev/null +++ b/mooncake-store/src/kv_event_zmq_publisher.cpp @@ -0,0 +1,108 @@ +#include "kv_event_zmq_publisher.h" + +#include +#include + +#include +#include +#include +#include + +namespace mooncake { + +namespace { + +// Current wall-clock time in seconds since the epoch, matching the engine-side +// batch timestamp semantics Dynamo expects in payload frame 3. +double NowSeconds() { + const auto now = std::chrono::system_clock::now().time_since_epoch(); + return std::chrono::duration(now).count(); +} + +// Encode a 64-bit value as 8 big-endian bytes (frame 2 of the wire format). +std::array EncodeBigEndian64(uint64_t value) { + std::array out{}; + for (int i = 7; i >= 0; --i) { + out[i] = static_cast(value & 0xFFu); + value >>= 8; + } + return out; +} + +} // namespace + +struct ZmqKvEventPublisher::Impl { + explicit Impl(const ZmqKvEventPublisherConfig& config) + : topic(config.topic), + context(1), + socket(context, zmq::socket_type::pub) { + socket.set(zmq::sockopt::linger, config.linger_ms); + if (config.send_hwm > 0) { + socket.set(zmq::sockopt::sndhwm, config.send_hwm); + } + if (config.bind) { + socket.bind(config.endpoint); + } else { + socket.connect(config.endpoint); + } + // Resolve the concrete endpoint (handles wildcard ports on bind). + endpoint = socket.get(zmq::sockopt::last_endpoint); + if (endpoint.empty()) { + endpoint = config.endpoint; + } + } + + std::string topic; + zmq::context_t context; + zmq::socket_t socket; + std::string endpoint; + std::mutex send_mutex; + std::atomic seq{0}; + std::atomic published{0}; +}; + +ZmqKvEventPublisher::ZmqKvEventPublisher( + const ZmqKvEventPublisherConfig& config) + : impl_(std::make_unique(config)) { + LOG(INFO) << "ZmqKvEventPublisher started, endpoint=" << impl_->endpoint + << ", topic='" << impl_->topic << "'"; +} + +ZmqKvEventPublisher::~ZmqKvEventPublisher() = default; + +void ZmqKvEventPublisher::Publish(const KvEvent& event) { + // Single-event batch: Mooncake centralizes KV state, so one PUB stream + // carries every worker's events with worker_id encoded per event. + std::optional dp_rank; + if (event.dp_rank.has_value()) { + dp_rank = static_cast(event.dp_rank.value()); + } + const std::string payload = + EncodeKvEventBatchPayload({event}, NowSeconds(), dp_rank); + + const uint64_t seq = impl_->seq.fetch_add(1); + const std::array seq_frame = EncodeBigEndian64(seq); + + std::lock_guard lock(impl_->send_mutex); + try { + impl_->socket.send(zmq::buffer(impl_->topic), + zmq::send_flags::sndmore); + impl_->socket.send(zmq::buffer(seq_frame.data(), seq_frame.size()), + zmq::send_flags::sndmore); + impl_->socket.send(zmq::buffer(payload), zmq::send_flags::none); + impl_->published.fetch_add(1); + } catch (const zmq::error_t& e) { + LOG(ERROR) << "ZmqKvEventPublisher failed to publish event_id=" + << event.event_id << ", error=" << e.what(); + } +} + +const std::string& ZmqKvEventPublisher::endpoint() const { + return impl_->endpoint; +} + +uint64_t ZmqKvEventPublisher::published_count() const { + return impl_->published.load(); +} + +} // namespace mooncake diff --git a/mooncake-store/src/master_service.cpp b/mooncake-store/src/master_service.cpp index 922e919a4de..6aab120f605 100644 --- a/mooncake-store/src/master_service.cpp +++ b/mooncake-store/src/master_service.cpp @@ -1,5 +1,6 @@ #include "master_service.h" +#include #include #include #include @@ -721,6 +722,12 @@ auto MasterService::MountSegment(const Segment& segment, const UUID& client_id) return tl::make_unexpected(err); } } + // Record this segment's physical worker identity for KV events. The + // transport endpoint (ip:port) identifies the node holding the data; fall + // back to the segment name when it is unset. + AutoRegisterSegmentWorkerId(segment.name, segment.te_endpoint.empty() + ? segment.name + : segment.te_endpoint); RecomputeTenantEffectiveQuotas(); return {}; } @@ -1289,6 +1296,15 @@ void MasterService::UnregisterGroupMember(TenantState& tenant_state, group_empty = true; } } + // Keep the KV event manifest in sync with group membership. + auto manifest_it = tenant_state.kv_group_manifests.find(group_id); + if (manifest_it != tenant_state.kv_group_manifests.end()) { + manifest_it->second.current_object_keys.erase(key); + manifest_it->second.object_worker_ids.erase(key); + if (group_empty) { + tenant_state.kv_group_manifests.erase(manifest_it); + } + } std::unique_lock lock(group_routing_mutex_); auto route_it = object_group_ids_.find(MakeTenantScopedKey(normalized_tenant, key)); @@ -1301,6 +1317,408 @@ void MasterService::UnregisterGroupMember(TenantState& tenant_state, } } +tl::expected MasterService::ValidateKvEventMetadataConfig( + const ReplicateConfig& config) { + if (!config.kv_event_metadata.has_value()) { + return {}; + } + + std::unordered_set declared_group_ids; + if (config.group_ids.has_value()) { + for (const auto& gid : config.group_ids.value()) { + if (!gid.empty()) { + declared_group_ids.insert(gid); + } + } + } + + std::unordered_set seen_metadata_groups; + for (const auto& meta : config.kv_event_metadata.value()) { + if (meta.group_id.empty()) { + LOG(ERROR) << "error=kv_event_metadata_empty_group_id"; + return tl::make_unexpected(ErrorCode::INVALID_PARAMS); + } + if (!seen_metadata_groups.insert(meta.group_id).second) { + LOG(ERROR) << "group_id=" << meta.group_id + << ", error=kv_event_metadata_duplicate_group"; + return tl::make_unexpected(ErrorCode::INVALID_PARAMS); + } + if (meta.expected_components.empty() && + meta.expected_object_count == 0) { + LOG(ERROR) << "group_id=" << meta.group_id + << ", error=kv_event_metadata_missing_expectation"; + return tl::make_unexpected(ErrorCode::INVALID_PARAMS); + } + for (const auto& component : meta.expected_components) { + if (component.object_key.empty()) { + LOG(ERROR) << "group_id=" << meta.group_id + << ", error=kv_event_metadata_empty_component_key"; + return tl::make_unexpected(ErrorCode::INVALID_PARAMS); + } + } + if (config.group_ids.has_value() && + declared_group_ids.find(meta.group_id) == + declared_group_ids.end()) { + LOG(ERROR) << "group_id=" << meta.group_id + << ", error=kv_event_metadata_group_not_in_group_ids"; + return tl::make_unexpected(ErrorCode::INVALID_PARAMS); + } + } + return {}; +} + +const KvBlockEventMetadata* MasterService::FindKvEventMetadataForGroup( + const ReplicateConfig& config, const std::string& group_id) { + if (!config.kv_event_metadata.has_value() || group_id.empty()) { + return nullptr; + } + for (const auto& meta : config.kv_event_metadata.value()) { + if (meta.group_id == group_id) { + return &meta; + } + } + return nullptr; +} + +tl::expected MasterService::ValidateKvManifestForKey( + const TenantState& tenant_state, const std::string& tenant_id, + const std::string& group_id, const std::string& key, + const ReplicateConfig& config) const { + (void)tenant_id; + (void)key; + const KvBlockEventMetadata* meta = + FindKvEventMetadataForGroup(config, group_id); + if (meta == nullptr) { + // No KV event metadata applies to this key's group: unchanged behavior. + return {}; + } + + // Structural validation: we must be able to decide completeness, which + // requires either an explicit expected component list or a positive + // expected object count. + if (meta->expected_components.empty() && meta->expected_object_count == 0) { + LOG(ERROR) << "group_id=" << group_id + << ", error=kv_event_metadata_missing_expectation"; + return tl::make_unexpected(ErrorCode::INVALID_PARAMS); + } + for (const auto& component : meta->expected_components) { + if (component.object_key.empty()) { + LOG(ERROR) << "group_id=" << group_id + << ", error=kv_event_metadata_empty_component_key"; + return tl::make_unexpected(ErrorCode::INVALID_PARAMS); + } + } + + // Conflict detection against an already-recorded manifest: identical + // metadata is treated as an idempotent retry, divergent metadata is + // rejected. + auto manifest_it = tenant_state.kv_group_manifests.find(group_id); + if (manifest_it != tenant_state.kv_group_manifests.end()) { + if (!manifest_it->second.metadata.SameSemanticsAs(*meta)) { + LOG(ERROR) << "group_id=" << group_id + << ", error=kv_event_metadata_conflict"; + return tl::make_unexpected(ErrorCode::INVALID_PARAMS); + } + } + return {}; +} + +void MasterService::UpsertKvManifestForKey(TenantState& tenant_state, + const std::string& tenant_id, + const std::string& group_id, + const std::string& key, + const ReplicateConfig& config) { + (void)key; + const KvBlockEventMetadata* meta = + FindKvEventMetadataForGroup(config, group_id); + if (meta == nullptr) { + return; + } + auto& manifest = tenant_state.kv_group_manifests[group_id]; + if (manifest.group_id.empty()) { + // Newly created manifest. + manifest.tenant_id = NormalizeTenantId(tenant_id); + manifest.group_id = group_id; + manifest.metadata = *meta; + for (const auto& component : meta->expected_components) { + manifest.expected_object_keys.insert(component.object_key); + } + } + // Idempotent for repeated identical metadata; conflicting metadata is + // rejected earlier by ValidateKvManifestForKey. +} + +void MasterService::SetKvEventPublisher( + std::shared_ptr publisher) { + kv_event_publisher_ = std::move(publisher); +} + +std::string MasterService::DeriveSegmentNameFromMetadata( + const ObjectMetadata& metadata) { + // Prefer a completed memory/nof replica's segment as the owning worker. + for (const auto& replica : metadata.GetAllReplicas()) { + if (!replica.is_completed()) { + continue; + } + for (const auto& name : replica.get_segment_names()) { + if (name.has_value() && !name->empty()) { + return name.value(); + } + } + } + for (const auto& replica : metadata.GetAllReplicas()) { + for (const auto& name : replica.get_segment_names()) { + if (name.has_value() && !name->empty()) { + return name.value(); + } + } + } + return std::string(); +} + +std::string MasterService::ResolveWorkerId( + const std::string& segment_name) const { + if (segment_name.empty()) { + return segment_name; + } + std::shared_lock lock(segment_worker_registry_mutex_); + auto explicit_it = segment_worker_explicit_.find(segment_name); + if (explicit_it != segment_worker_explicit_.end()) { + return explicit_it->second; + } + auto auto_it = segment_worker_auto_.find(segment_name); + if (auto_it != segment_worker_auto_.end()) { + return auto_it->second; + } + // Unregistered segment: fall back to the segment name so the event still + // carries a stable, if coarse, worker identity. + return segment_name; +} + +void MasterService::AutoRegisterSegmentWorkerId( + const std::string& segment_name, const std::string& worker_id) { + if (segment_name.empty() || worker_id.empty()) { + return; + } + std::unique_lock lock(segment_worker_registry_mutex_); + segment_worker_auto_[segment_name] = worker_id; +} + +void MasterService::RegisterSegmentWorkerId(const std::string& segment_name, + const std::string& worker_id) { + if (segment_name.empty()) { + return; + } + std::unique_lock lock(segment_worker_registry_mutex_); + if (worker_id.empty()) { + segment_worker_explicit_.erase(segment_name); + } else { + segment_worker_explicit_[segment_name] = worker_id; + } +} + +std::optional MasterService::GetSegmentWorkerId( + const std::string& segment_name) const { + std::shared_lock lock(segment_worker_registry_mutex_); + auto explicit_it = segment_worker_explicit_.find(segment_name); + if (explicit_it != segment_worker_explicit_.end()) { + return explicit_it->second; + } + auto auto_it = segment_worker_auto_.find(segment_name); + if (auto_it != segment_worker_auto_.end()) { + return auto_it->second; + } + return std::nullopt; +} + +std::string MasterService::MakeKvEventId(const std::string& tenant_id, + const std::string& group_id, + const char* kind) { + const uint64_t seq = kv_event_seq_.fetch_add(1) + 1; + return "mooncake:" + tenant_id + ":" + group_id + ":" + kind + ":" + + std::to_string(seq); +} + +bool MasterService::IsKvManifestComplete(const KvGroupManifest& manifest) { + if (!manifest.expected_object_keys.empty()) { + for (const auto& expected : manifest.expected_object_keys) { + if (manifest.current_object_keys.find(expected) == + manifest.current_object_keys.end()) { + return false; + } + } + return true; + } + // Weaker count-only completeness when no explicit component list was given. + return manifest.metadata.expected_object_count > 0 && + manifest.current_object_keys.size() >= + manifest.metadata.expected_object_count; +} + +void MasterService::MaybePublishKvBlockStored(KvGroupManifest& manifest, + const std::string& worker_id) { + if (!kv_event_publisher_) { + return; + } + if (!manifest.metadata.emit_stored_event) { + return; + } + if (!IsKvManifestComplete(manifest)) { + return; + } + // v1: a single representative worker per logical block. The worker set is + // reserved for full multi-worker semantics; a non-empty set means the + // current placement has already been announced as stored. + if (!manifest.stored_event_published_workers.empty()) { + return; + } + + std::string representative = worker_id; + if (representative.empty()) { + for (const auto& [object_key, wid] : manifest.object_worker_ids) { + (void)object_key; + if (!wid.empty()) { + representative = wid; + break; + } + } + } + + const auto& metadata = manifest.metadata; + KvEvent event; + event.type = KvEventType::kBlockStored; + event.event_id = + MakeKvEventId(manifest.tenant_id, manifest.group_id, "stored"); + event.tenant_id = manifest.tenant_id; + event.group_id = manifest.group_id; + event.worker_id = representative; + event.medium = "EXTERNAL"; + event.block_hashes = {metadata.block_hash}; + event.parent_block_hash = metadata.parent_block_hash; + event.token_ids = metadata.token_ids; + event.block_size = metadata.block_size; + event.lora_name = metadata.lora_name; + event.model_name = metadata.model_name; + event.dp_rank = metadata.dp_rank; + + kv_event_publisher_->Publish(event); + + manifest.stored_event_published_workers.insert(representative); + manifest.removed_event_published_workers.clear(); +} + +void MasterService::MaybePublishKvBlockRemoved(KvGroupManifest& manifest) { + if (!kv_event_publisher_) { + return; + } + if (!manifest.metadata.emit_removed_event) { + return; + } + // Only meaningful if the block was previously announced as stored and not + // yet announced as removed for the current placement. + if (manifest.stored_event_published_workers.empty()) { + return; + } + + const std::string representative = + *manifest.stored_event_published_workers.begin(); + + KvEvent event; + event.type = KvEventType::kBlockRemoved; + event.event_id = + MakeKvEventId(manifest.tenant_id, manifest.group_id, "removed"); + event.tenant_id = manifest.tenant_id; + event.group_id = manifest.group_id; + event.worker_id = representative; + event.medium = "EXTERNAL"; + event.block_hashes = {manifest.metadata.block_hash}; + + kv_event_publisher_->Publish(event); + + manifest.removed_event_published_workers.insert(representative); + manifest.stored_event_published_workers.clear(); +} + +void MasterService::MarkKvManifestObjectComplete(TenantState& tenant_state, + const std::string& group_id, + const std::string& key, + const std::string& worker_id) { + if (group_id.empty()) { + return; + } + auto manifest_it = tenant_state.kv_group_manifests.find(group_id); + if (manifest_it == tenant_state.kv_group_manifests.end()) { + return; + } + auto& manifest = manifest_it->second; + manifest.current_object_keys.insert(key); + if (!worker_id.empty()) { + manifest.object_worker_ids[key] = worker_id; + } + MaybePublishKvBlockStored(manifest, worker_id); +} + +void MasterService::OnKvManifestObjectRemoved(TenantState& tenant_state, + const std::string& group_id, + const std::string& key) { + if (group_id.empty()) { + return; + } + auto manifest_it = tenant_state.kv_group_manifests.find(group_id); + if (manifest_it == tenant_state.kv_group_manifests.end()) { + return; + } + auto& manifest = manifest_it->second; + // Removal of an expected object (or any member when only a count is known) + // makes the logical block incomplete. + const bool affects_block = + manifest.expected_object_keys.empty() || + manifest.expected_object_keys.find(key) != + manifest.expected_object_keys.end(); + if (!affects_block) { + return; + } + MaybePublishKvBlockRemoved(manifest); +} + +std::optional MasterService::GetKvGroupManifest( + const std::string& tenant_id, const std::string& group_id) const { + if (group_id.empty()) { + return std::nullopt; + } + const auto normalized_tenant = NormalizeTenantId(tenant_id); + // Grouped keys (and therefore the manifest) live in the shard selected by + // the group_id. + const size_t shard_idx = getShardIndex(group_id); + std::shared_lock shared_lock(snapshot_mutex_); + MetadataShardAccessorRO shard(this, shard_idx); + auto tenant_it = shard->tenants.find(normalized_tenant); + if (tenant_it == shard->tenants.end()) { + return std::nullopt; + } + const auto& manifests = tenant_it->second.kv_group_manifests; + auto manifest_it = manifests.find(group_id); + if (manifest_it == manifests.end()) { + return std::nullopt; + } + const auto& manifest = manifest_it->second; + + KvGroupManifestView view; + view.tenant_id = manifest.tenant_id; + view.group_id = manifest.group_id; + view.metadata = manifest.metadata; + view.generation = manifest.generation; + view.expected_object_keys.assign(manifest.expected_object_keys.begin(), + manifest.expected_object_keys.end()); + view.current_object_keys.assign(manifest.current_object_keys.begin(), + manifest.current_object_keys.end()); + std::sort(view.expected_object_keys.begin(), + view.expected_object_keys.end()); + std::sort(view.current_object_keys.begin(), + view.current_object_keys.end()); + return view; +} + bool MasterService::HasCompletedMemoryCacheReplica( const ObjectMetadata& metadata) { return metadata.HasReplica([](const Replica& replica) { @@ -1435,6 +1853,9 @@ MasterService::EraseMetadata( if (had_completed_disk && shard) { shard->OnDiskReplicaRemoved(had_completed_disk); } + // Publish BlockRemoved before the group manifest is torn down by + // UnregisterGroupMember (which erases the manifest once the group empties). + OnKvManifestObjectRemoved(tenant_state, group_id, key); UnregisterGroupMember(tenant_state, tenant_id, key, group_id); return next; } @@ -2361,6 +2782,14 @@ auto MasterService::AllocateAndInsertMetadata( return tl::make_unexpected(ErrorCode::OBJECT_ALREADY_EXISTS); } + // Reject structurally invalid or conflicting KV event metadata before + // doing any allocation, so callers fail fast and no partial state leaks. + if (auto validation = ValidateKvManifestForKey(tenant_state, tenant_id, + group_id, key, config); + !validation) { + return tl::make_unexpected(validation.error()); + } + const uint64_t reserved_quota_charge = RequestedMemoryQuotaCharge(value_length, config); auto quota_result = ReserveTenantQuota(tenant_id, reserved_quota_charge); @@ -2523,6 +2952,7 @@ auto MasterService::AllocateAndInsertMetadata( } it->second.reserved_quota_charge_bytes = reserved_quota_charge; RegisterGroupMember(tenant_state, tenant_id, key, group_id); + UpsertKvManifestForKey(tenant_state, tenant_id, group_id, key, config); tenant_state.processing_keys.insert(key); return replica_list; @@ -2770,6 +3200,17 @@ auto MasterService::PutEnd(const UUID& client_id, const std::string& key, accessor.EraseFromProcessing(); } + // Track completion in the KV event group manifest and publish BlockStored + // once the logical block is complete. + if (metadata.IsGrouped() && + metadata.AllReplicas(&Replica::fn_is_completed)) { + const std::string segment_name = + DeriveSegmentNameFromMetadata(metadata); + const std::string worker_id = ResolveWorkerId(segment_name); + MarkKvManifestObjectComplete(accessor.GetTenantState(), + metadata.group_id, key, worker_id); + } + SyncCacheTotalAccounting(metadata); // TODO: add inc_nof_cache_nums() (ranhaojia) // 1. Set lease timeout to now, indicating that the object has no lease diff --git a/mooncake-store/src/rpc_service.cpp b/mooncake-store/src/rpc_service.cpp index a85b93490dd..c6eccb553d5 100644 --- a/mooncake-store/src/rpc_service.cpp +++ b/mooncake-store/src/rpc_service.cpp @@ -237,7 +237,12 @@ WrappedMasterService::PutStart(const UUID& client_id, const std::string& key, const std::string& tenant_id) { return execute_rpc( "PutStart", - [&] { + [&]() -> tl::expected, ErrorCode> { + if (auto validation = + MasterService::ValidateKvEventMetadataConfig(config); + !validation) { + return tl::make_unexpected(validation.error()); + } return master_service_.PutStart(client_id, key, tenant_id, slice_length, config); }, @@ -310,6 +315,12 @@ WrappedMasterService::BatchPutStart(const UUID& client_id, << " != keys.size()=" << keys.size(); results.assign(keys.size(), tl::make_unexpected(ErrorCode::INVALID_PARAMS)); + } else if (auto kv_validation = + MasterService::ValidateKvEventMetadataConfig(config); + !kv_validation) { + LOG(ERROR) << "BatchPutStart: invalid kv_event_metadata"; + results.assign(keys.size(), + tl::make_unexpected(kv_validation.error())); } else if (config.prefer_alloc_in_same_node) { ReplicateConfig new_config = config; for (size_t i = 0; i < keys.size(); ++i) { @@ -465,7 +476,12 @@ WrappedMasterService::UpsertStart(const UUID& client_id, const std::string& key, const std::string& tenant_id) { return execute_rpc( "UpsertStart", - [&] { + [&]() -> tl::expected, ErrorCode> { + if (auto validation = + MasterService::ValidateKvEventMetadataConfig(config); + !validation) { + return tl::make_unexpected(validation.error()); + } return master_service_.UpsertStart(client_id, key, tenant_id, slice_length, config); }, @@ -521,6 +537,18 @@ WrappedMasterService::BatchUpsertStart( timer.LogRequest("client_id=", client_id, ", keys_count=", total_keys); MasterMetricManager::instance().inc_batch_put_start_requests(total_keys); + if (auto kv_validation = + MasterService::ValidateKvEventMetadataConfig(config); + !kv_validation) { + LOG(ERROR) << "BatchUpsertStart: invalid kv_event_metadata"; + std::vector, ErrorCode>> + results(keys.size(), + tl::make_unexpected(kv_validation.error())); + MasterMetricManager::instance().inc_batch_put_start_failures( + keys.size()); + return results; + } + auto results = master_service_.BatchUpsertStart(client_id, keys, tenant_id, slice_lengths, config); diff --git a/mooncake-store/tests/master_service_test.cpp b/mooncake-store/tests/master_service_test.cpp index ad1c55e0685..054048feb5c 100644 --- a/mooncake-store/tests/master_service_test.cpp +++ b/mooncake-store/tests/master_service_test.cpp @@ -18,6 +18,12 @@ #include "types.h" +#ifdef MOONCAKE_STORE_WITH_ZMQ +#include + +#include "kv_event_zmq_publisher.h" +#endif + namespace mooncake::test { class MasterServiceTest : public ::testing::Test { @@ -645,6 +651,657 @@ TEST_F(MasterServiceTest, PutStartGroupIdsValidation) { EXPECT_TRUE(exists.value()); } +// --------------------------------------------------------------------------- +// KV event metadata / group manifest tests (Phase 1: interface + manifest). +// No KV event is published in this phase; these only verify the state model. +// --------------------------------------------------------------------------- + +namespace { + +KvBlockEventMetadata MakeKvMetadata( + const std::string& group_id, uint64_t block_hash, + const std::vector& components) { + KvBlockEventMetadata meta; + meta.group_id = group_id; + meta.block_hash = block_hash; + meta.parent_block_hash = block_hash > 0 ? block_hash - 1 : 0; + meta.token_ids = {1, 2, 3, 4}; + meta.block_size = 64; + meta.expected_object_count = static_cast(components.size()); + meta.expected_components = components; + return meta; +} + +} // namespace + +TEST_F(MasterServiceTest, KvEventMetadataAbsentLeavesNoManifest) { + std::unique_ptr service_(new MasterService()); + [[maybe_unused]] const auto context = PrepareSimpleSegment(*service_); + UUID client_id = generate_uuid(); + + const std::string key = "kv_no_meta_key"; + const std::string group_id = FindGroupIdOnDifferentShard(key); + ReplicateConfig config; + config.replica_num = 1; + config.group_ids = std::vector{group_id}; + + PutCompletedObject(*service_, client_id, key, config); + + // Grouped object exists, but with no kv_event_metadata there is no manifest. + EXPECT_FALSE(service_->GetKvGroupManifest("default", group_id).has_value()); + auto exists = service_->ExistKey(key, "default"); + ASSERT_TRUE(exists.has_value()); + EXPECT_TRUE(exists.value()); +} + +TEST_F(MasterServiceTest, KvEventManifestCreatedAndCompleted) { + std::unique_ptr service_(new MasterService()); + [[maybe_unused]] const auto context = PrepareSimpleSegment(*service_); + UUID client_id = generate_uuid(); + + const std::string key = "kv_single_key"; + const std::string group_id = FindGroupIdOnDifferentShard(key); + + ReplicateConfig config; + config.replica_num = 1; + config.group_ids = std::vector{group_id}; + config.kv_event_metadata = std::vector{MakeKvMetadata( + group_id, 10001, {KvBlockComponentSpec{key, "k", 0}})}; + + ASSERT_TRUE( + service_->PutStart(client_id, key, "default", 1024, config).has_value()); + + { + auto manifest = service_->GetKvGroupManifest("default", group_id); + ASSERT_TRUE(manifest.has_value()); + EXPECT_EQ(manifest->group_id, group_id); + EXPECT_EQ(manifest->metadata.block_hash, 10001u); + EXPECT_EQ(manifest->metadata.block_size, 64u); + ASSERT_EQ(manifest->expected_object_keys.size(), 1u); + EXPECT_EQ(manifest->expected_object_keys[0], key); + // Not completed until PutEnd. + EXPECT_TRUE(manifest->current_object_keys.empty()); + } + + ASSERT_TRUE(service_->PutEnd(client_id, key, "default", ReplicaType::MEMORY) + .has_value()); + + { + auto manifest = service_->GetKvGroupManifest("default", group_id); + ASSERT_TRUE(manifest.has_value()); + ASSERT_EQ(manifest->current_object_keys.size(), 1u); + EXPECT_EQ(manifest->current_object_keys[0], key); + } +} + +TEST_F(MasterServiceTest, KvEventManifestSharedAcrossGroupMembers) { + std::unique_ptr service_(new MasterService()); + [[maybe_unused]] const auto context = PrepareSimpleSegment(*service_); + UUID client_id = generate_uuid(); + + const std::string group_id = "kv_mha_group"; + const std::string key_k = "kv_mha_10001_k"; + const std::string key_v = "kv_mha_10001_v"; + + auto meta = MakeKvMetadata( + group_id, 10001, + {KvBlockComponentSpec{key_k, "k", 0}, KvBlockComponentSpec{key_v, "v", + 1}}); + ReplicateConfig config; + config.replica_num = 1; + config.group_ids = std::vector{group_id}; + config.kv_event_metadata = std::vector{meta}; + + // Two physical objects of the same logical block. Each PutStart is per-key + // but carries the same group-level metadata (idempotent retry). + ASSERT_TRUE(service_->PutStart(client_id, key_k, "default", 1024, config) + .has_value()); + ASSERT_TRUE(service_->PutStart(client_id, key_v, "default", 1024, config) + .has_value()); + + { + auto manifest = service_->GetKvGroupManifest("default", group_id); + ASSERT_TRUE(manifest.has_value()); + // A single manifest with both expected components. + ASSERT_EQ(manifest->expected_object_keys.size(), 2u); + EXPECT_EQ(manifest->expected_object_keys[0], key_k); + EXPECT_EQ(manifest->expected_object_keys[1], key_v); + EXPECT_TRUE(manifest->current_object_keys.empty()); + } + + ASSERT_TRUE(service_->PutEnd(client_id, key_k, "default", ReplicaType::MEMORY) + .has_value()); + { + auto manifest = service_->GetKvGroupManifest("default", group_id); + ASSERT_TRUE(manifest.has_value()); + EXPECT_EQ(manifest->current_object_keys.size(), 1u); + } + + ASSERT_TRUE(service_->PutEnd(client_id, key_v, "default", ReplicaType::MEMORY) + .has_value()); + { + auto manifest = service_->GetKvGroupManifest("default", group_id); + ASSERT_TRUE(manifest.has_value()); + ASSERT_EQ(manifest->current_object_keys.size(), 2u); + EXPECT_EQ(manifest->current_object_keys[0], key_k); + EXPECT_EQ(manifest->current_object_keys[1], key_v); + } +} + +TEST_F(MasterServiceTest, KvEventManifestConflictingMetadataRejected) { + std::unique_ptr service_(new MasterService()); + [[maybe_unused]] const auto context = PrepareSimpleSegment(*service_); + UUID client_id = generate_uuid(); + + const std::string group_id = "kv_conflict_group"; + const std::string key1 = "kv_conf_a"; + const std::string key2 = "kv_conf_b"; + const std::vector components{ + KvBlockComponentSpec{key1, "k", 0}, + KvBlockComponentSpec{key2, "v", 1}}; + + ReplicateConfig config1; + config1.replica_num = 1; + config1.group_ids = std::vector{group_id}; + config1.kv_event_metadata = + std::vector{MakeKvMetadata(group_id, 10001, + components)}; + ASSERT_TRUE(service_->PutStart(client_id, key1, "default", 1024, config1) + .has_value()); + + // Same group, different block_hash -> conflicting redefinition. + ReplicateConfig config2; + config2.replica_num = 1; + config2.group_ids = std::vector{group_id}; + config2.kv_event_metadata = + std::vector{MakeKvMetadata(group_id, 20002, + components)}; + auto conflict = service_->PutStart(client_id, key2, "default", 1024, + config2); + EXPECT_FALSE(conflict.has_value()); + EXPECT_EQ(ErrorCode::INVALID_PARAMS, conflict.error()); + + // Original manifest is unchanged. + auto manifest = service_->GetKvGroupManifest("default", group_id); + ASSERT_TRUE(manifest.has_value()); + EXPECT_EQ(manifest->metadata.block_hash, 10001u); +} + +TEST_F(MasterServiceTest, KvEventManifestErasedWhenGroupRemoved) { + std::unique_ptr service_(new MasterService()); + [[maybe_unused]] const auto context = PrepareSimpleSegment(*service_); + UUID client_id = generate_uuid(); + + const std::string key = "kv_remove_key"; + const std::string group_id = FindGroupIdOnDifferentShard(key); + ReplicateConfig config; + config.replica_num = 1; + config.group_ids = std::vector{group_id}; + config.kv_event_metadata = std::vector{MakeKvMetadata( + group_id, 10001, {KvBlockComponentSpec{key, "k", 0}})}; + + PutCompletedObject(*service_, client_id, key, config); + ASSERT_TRUE(service_->GetKvGroupManifest("default", group_id).has_value()); + + ASSERT_TRUE( + service_->Remove(key, "default", /*force=*/true).has_value()); + + // Group is now empty, so its manifest is gone too. + EXPECT_FALSE(service_->GetKvGroupManifest("default", group_id).has_value()); +} + +TEST_F(MasterServiceTest, KvEventMetadataConfigValidation) { + auto make_valid = [](const std::string& g) { + return MakeKvMetadata(g, 1, {KvBlockComponentSpec{g + "_k", "k", 0}}); + }; + + // Valid config passes. + { + ReplicateConfig c; + c.group_ids = std::vector{"g0"}; + c.kv_event_metadata = + std::vector{make_valid("g0")}; + EXPECT_TRUE( + MasterService::ValidateKvEventMetadataConfig(c).has_value()); + } + + // No metadata is a no-op (valid). + { + ReplicateConfig c; + c.group_ids = std::vector{"g0"}; + EXPECT_TRUE( + MasterService::ValidateKvEventMetadataConfig(c).has_value()); + } + + // Metadata group_id not present in group_ids. + { + ReplicateConfig c; + c.group_ids = std::vector{"g0"}; + c.kv_event_metadata = + std::vector{make_valid("other")}; + auto r = MasterService::ValidateKvEventMetadataConfig(c); + EXPECT_FALSE(r.has_value()); + EXPECT_EQ(ErrorCode::INVALID_PARAMS, r.error()); + } + + // Empty metadata group_id. + { + ReplicateConfig c; + c.group_ids = std::vector{""}; + c.kv_event_metadata = + std::vector{make_valid("")}; + EXPECT_FALSE( + MasterService::ValidateKvEventMetadataConfig(c).has_value()); + } + + // Duplicate metadata for the same group. + { + ReplicateConfig c; + c.group_ids = std::vector{"g0", "g0"}; + c.kv_event_metadata = std::vector{ + make_valid("g0"), make_valid("g0")}; + EXPECT_FALSE( + MasterService::ValidateKvEventMetadataConfig(c).has_value()); + } + + // Missing completeness expectation (no components and zero count). + { + ReplicateConfig c; + c.group_ids = std::vector{"g0"}; + KvBlockEventMetadata meta; + meta.group_id = "g0"; + meta.block_size = 64; + c.kv_event_metadata = std::vector{meta}; + EXPECT_FALSE( + MasterService::ValidateKvEventMetadataConfig(c).has_value()); + } +} + +// --------------------------------------------------------------------------- +// KV event publisher tests (Phase 2). A mock publisher captures the events the +// master would otherwise send over its single ZMQ stream. +// --------------------------------------------------------------------------- + +namespace { + +class MockKvEventPublisher : public KvEventPublisher { + public: + void Publish(const KvEvent& event) override { + std::lock_guard lock(mutex_); + events_.push_back(event); + } + + std::vector Events() const { + std::lock_guard lock(mutex_); + return events_; + } + + size_t CountOfType(KvEventType type) const { + std::lock_guard lock(mutex_); + size_t count = 0; + for (const auto& event : events_) { + if (event.type == type) { + ++count; + } + } + return count; + } + + void Clear() { + std::lock_guard lock(mutex_); + events_.clear(); + } + + private: + mutable std::mutex mutex_; + std::vector events_; +}; + +} // namespace + +TEST_F(MasterServiceTest, KvEventEncoderRoundTrip) { + KvEvent stored; + stored.type = KvEventType::kBlockStored; + stored.event_id = "mooncake:default:g1:stored:1"; + stored.tenant_id = "tenant-a"; + stored.group_id = "g1"; + stored.worker_id = "mooncake-node-3"; + stored.medium = "EXTERNAL"; + stored.block_hashes = {10001}; + stored.parent_block_hash = 9999; + stored.token_ids = {101, 102, 103}; + stored.block_size = 64; + stored.lora_name = "adapter"; + stored.model_name = "llama"; + stored.dp_rank = 2; + + auto decoded = DecodeKvEventMap(EncodeKvEventMap(stored)); + ASSERT_TRUE(decoded.has_value()); + EXPECT_EQ(stored, decoded.value()); + + KvEvent removed; + removed.type = KvEventType::kBlockRemoved; + removed.event_id = "mooncake:default:g1:removed:2"; + removed.tenant_id = "tenant-a"; + removed.group_id = "g1"; + removed.worker_id = "mooncake-node-3"; + removed.medium = "EXTERNAL"; + removed.block_hashes = {10001}; + + auto decoded_removed = DecodeKvEventMap(EncodeKvEventMap(removed)); + ASSERT_TRUE(decoded_removed.has_value()); + EXPECT_EQ(removed, decoded_removed.value()); + + // Batch payload [timestamp, [events], dp_rank]. + auto payload = EncodeKvEventBatchPayload({stored, removed}, 1234.5, 0); + auto batch = DecodeKvEventBatchPayload(payload); + ASSERT_TRUE(batch.has_value()); + EXPECT_DOUBLE_EQ(batch->timestamp, 1234.5); + ASSERT_EQ(batch->events.size(), 2u); + EXPECT_EQ(batch->events[0], stored); + EXPECT_EQ(batch->events[1], removed); + ASSERT_TRUE(batch->dp_rank.has_value()); + EXPECT_EQ(batch->dp_rank.value(), 0); +} + +TEST_F(MasterServiceTest, KvEventPublishesSingleBlockStoredWhenComplete) { + std::unique_ptr service_(new MasterService()); + [[maybe_unused]] const auto context = PrepareSimpleSegment(*service_); + auto publisher = std::make_shared(); + service_->SetKvEventPublisher(publisher); + UUID client_id = generate_uuid(); + + const std::string group_id = "kv_pub_mha_group"; + const std::string key_k = "kv_pub_10001_k"; + const std::string key_v = "kv_pub_10001_v"; + auto meta = MakeKvMetadata( + group_id, 10001, + {KvBlockComponentSpec{key_k, "k", 0}, + KvBlockComponentSpec{key_v, "v", 1}}); + ReplicateConfig config; + config.replica_num = 1; + config.group_ids = std::vector{group_id}; + config.kv_event_metadata = std::vector{meta}; + + // Write + complete only K: not enough to publish. + ASSERT_TRUE(service_->PutStart(client_id, key_k, "default", 1024, config) + .has_value()); + ASSERT_TRUE(service_->PutEnd(client_id, key_k, "default", ReplicaType::MEMORY) + .has_value()); + EXPECT_EQ(publisher->CountOfType(KvEventType::kBlockStored), 0u); + + // Complete V: logical block is now complete -> one BlockStored. + ASSERT_TRUE(service_->PutStart(client_id, key_v, "default", 1024, config) + .has_value()); + ASSERT_TRUE(service_->PutEnd(client_id, key_v, "default", ReplicaType::MEMORY) + .has_value()); + + auto events = publisher->Events(); + ASSERT_EQ(events.size(), 1u); + const KvEvent& ev = events[0]; + EXPECT_EQ(ev.type, KvEventType::kBlockStored); + EXPECT_EQ(ev.source, "mooncake"); + EXPECT_EQ(ev.tenant_id, "default"); + EXPECT_EQ(ev.group_id, group_id); + EXPECT_EQ(ev.medium, "EXTERNAL"); + EXPECT_EQ(ev.worker_id, "test_segment"); + ASSERT_EQ(ev.block_hashes.size(), 1u); + EXPECT_EQ(ev.block_hashes[0], 10001u); + ASSERT_TRUE(ev.parent_block_hash.has_value()); + EXPECT_EQ(ev.parent_block_hash.value(), 10000u); + EXPECT_EQ(ev.token_ids, meta.token_ids); + EXPECT_EQ(ev.block_size, 64u); + EXPECT_FALSE(ev.event_id.empty()); +} + +TEST_F(MasterServiceTest, KvEventNoDuplicateBlockStoredOnRepeatedPutEnd) { + std::unique_ptr service_(new MasterService()); + [[maybe_unused]] const auto context = PrepareSimpleSegment(*service_); + auto publisher = std::make_shared(); + service_->SetKvEventPublisher(publisher); + UUID client_id = generate_uuid(); + + const std::string group_id = FindGroupIdOnDifferentShard("kv_dup_key"); + const std::string key = "kv_dup_key"; + ReplicateConfig config; + config.replica_num = 1; + config.group_ids = std::vector{group_id}; + config.kv_event_metadata = std::vector{MakeKvMetadata( + group_id, 555, {KvBlockComponentSpec{key, "k", 0}})}; + + ASSERT_TRUE( + service_->PutStart(client_id, key, "default", 1024, config).has_value()); + ASSERT_TRUE(service_->PutEnd(client_id, key, "default", ReplicaType::MEMORY) + .has_value()); + EXPECT_EQ(publisher->CountOfType(KvEventType::kBlockStored), 1u); + + // Repeated PutEnd must not re-publish. + ASSERT_TRUE(service_->PutEnd(client_id, key, "default", ReplicaType::MEMORY) + .has_value()); + EXPECT_EQ(publisher->CountOfType(KvEventType::kBlockStored), 1u); +} + +TEST_F(MasterServiceTest, KvEventPublishesSingleBlockRemovedOnDelete) { + std::unique_ptr service_(new MasterService()); + [[maybe_unused]] const auto context = PrepareSimpleSegment(*service_); + auto publisher = std::make_shared(); + service_->SetKvEventPublisher(publisher); + UUID client_id = generate_uuid(); + + const std::string group_id = "kv_rm_mha_group"; + const std::string key_k = "kv_rm_10001_k"; + const std::string key_v = "kv_rm_10001_v"; + auto meta = MakeKvMetadata( + group_id, 10001, + {KvBlockComponentSpec{key_k, "k", 0}, + KvBlockComponentSpec{key_v, "v", 1}}); + ReplicateConfig config; + config.replica_num = 1; + config.group_ids = std::vector{group_id}; + config.kv_event_metadata = std::vector{meta}; + + PutCompletedObject(*service_, client_id, key_k, config); + PutCompletedObject(*service_, client_id, key_v, config); + ASSERT_EQ(publisher->CountOfType(KvEventType::kBlockStored), 1u); + + publisher->Clear(); + + // Removing one component makes the logical block unavailable -> one removed. + ASSERT_TRUE( + service_->Remove(key_k, "default", /*force=*/true).has_value()); + auto removed_events = publisher->Events(); + ASSERT_EQ(removed_events.size(), 1u); + EXPECT_EQ(removed_events[0].type, KvEventType::kBlockRemoved); + EXPECT_EQ(removed_events[0].group_id, group_id); + EXPECT_EQ(removed_events[0].worker_id, "test_segment"); + ASSERT_EQ(removed_events[0].block_hashes.size(), 1u); + EXPECT_EQ(removed_events[0].block_hashes[0], 10001u); + EXPECT_TRUE(removed_events[0].token_ids.empty()); + + // Removing the second component must not publish another BlockRemoved. + ASSERT_TRUE( + service_->Remove(key_v, "default", /*force=*/true).has_value()); + EXPECT_EQ(publisher->CountOfType(KvEventType::kBlockRemoved), 1u); +} + +TEST_F(MasterServiceTest, KvEventNotPublishedWithoutPublisher) { + std::unique_ptr service_(new MasterService()); + [[maybe_unused]] const auto context = PrepareSimpleSegment(*service_); + UUID client_id = generate_uuid(); + + const std::string key = "kv_nopub_key"; + const std::string group_id = FindGroupIdOnDifferentShard(key); + ReplicateConfig config; + config.replica_num = 1; + config.group_ids = std::vector{group_id}; + config.kv_event_metadata = std::vector{MakeKvMetadata( + group_id, 777, {KvBlockComponentSpec{key, "k", 0}})}; + + // No publisher installed: must behave normally and not crash. + PutCompletedObject(*service_, client_id, key, config); + auto manifest = service_->GetKvGroupManifest("default", group_id); + ASSERT_TRUE(manifest.has_value()); + EXPECT_EQ(manifest->current_object_keys.size(), 1u); + ASSERT_TRUE( + service_->Remove(key, "default", /*force=*/true).has_value()); +} + +// --------------------------------------------------------------------------- +// segment -> worker_id registry tests. worker_id in published events must +// reflect the physical worker (transport endpoint) rather than the logical +// segment name, with deployment-config overrides taking precedence. +// --------------------------------------------------------------------------- + +TEST_F(MasterServiceTest, SegmentWorkerRegistryCapturesEndpointAtMount) { + std::unique_ptr service_(new MasterService()); + + // Unknown segment: no mapping yet. + EXPECT_FALSE(service_->GetSegmentWorkerId("unmounted_seg").has_value()); + + Segment segment = MakeSegment("reg_logical_seg"); + segment.te_endpoint = "10.0.0.7:4400"; + const UUID client_id = generate_uuid(); + ASSERT_TRUE(service_->MountSegment(segment, client_id).has_value()); + + auto worker_id = service_->GetSegmentWorkerId("reg_logical_seg"); + ASSERT_TRUE(worker_id.has_value()); + EXPECT_EQ(worker_id.value(), "10.0.0.7:4400"); +} + +TEST_F(MasterServiceTest, KvEventWorkerIdReflectsSegmentEndpoint) { + std::unique_ptr service_(new MasterService()); + auto publisher = std::make_shared(); + service_->SetKvEventPublisher(publisher); + + // Segment whose physical endpoint differs from its logical name. + Segment segment = MakeSegment("worker_seg_logical"); + segment.te_endpoint = "10.0.0.7:4400"; + const UUID client_id = generate_uuid(); + ASSERT_TRUE(service_->MountSegment(segment, client_id).has_value()); + + const std::string group_id = "kv_worker_seg_group"; + const std::string key = "kv_worker_seg_key"; + ReplicateConfig config; + config.replica_num = 1; + config.preferred_segment = "worker_seg_logical"; + config.group_ids = std::vector{group_id}; + config.kv_event_metadata = std::vector{ + MakeKvMetadata(group_id, 4242, {KvBlockComponentSpec{key, "k", 0}})}; + + PutCompletedObject(*service_, client_id, key, config); + + auto events = publisher->Events(); + ASSERT_EQ(events.size(), 1u); + EXPECT_EQ(events[0].type, KvEventType::kBlockStored); + // worker_id is the transport endpoint, not the logical segment name. + EXPECT_EQ(events[0].worker_id, "10.0.0.7:4400"); +} + +TEST_F(MasterServiceTest, KvEventWorkerIdExplicitOverrideWins) { + std::unique_ptr service_(new MasterService()); + auto publisher = std::make_shared(); + service_->SetKvEventPublisher(publisher); + + Segment segment = MakeSegment("override_seg"); + segment.te_endpoint = "10.0.0.9:5500"; + const UUID client_id = generate_uuid(); + ASSERT_TRUE(service_->MountSegment(segment, client_id).has_value()); + + // Explicit registration (deployment config / control plane) wins over the + // mount-time endpoint. + service_->RegisterSegmentWorkerId("override_seg", "logical-worker-42"); + auto wid = service_->GetSegmentWorkerId("override_seg"); + ASSERT_TRUE(wid.has_value()); + EXPECT_EQ(wid.value(), "logical-worker-42"); + + const std::string group_id = "kv_override_group"; + const std::string key = "kv_override_key"; + ReplicateConfig config; + config.replica_num = 1; + config.preferred_segment = "override_seg"; + config.group_ids = std::vector{group_id}; + config.kv_event_metadata = std::vector{ + MakeKvMetadata(group_id, 99, {KvBlockComponentSpec{key, "k", 0}})}; + PutCompletedObject(*service_, client_id, key, config); + + auto events = publisher->Events(); + ASSERT_EQ(events.size(), 1u); + EXPECT_EQ(events[0].worker_id, "logical-worker-42"); + + // Clearing the override falls back to the mount-time endpoint. + service_->RegisterSegmentWorkerId("override_seg", ""); + auto fallback = service_->GetSegmentWorkerId("override_seg"); + ASSERT_TRUE(fallback.has_value()); + EXPECT_EQ(fallback.value(), "10.0.0.9:5500"); +} + +#ifdef MOONCAKE_STORE_WITH_ZMQ +// --------------------------------------------------------------------------- +// Real ZMQ transport: a SUB socket must receive and decode the 3-frame +// SGLang/vLLM-compatible message produced by ZmqKvEventPublisher. +// --------------------------------------------------------------------------- +TEST_F(MasterServiceTest, ZmqKvEventPublisherRoundTrip) { + ZmqKvEventPublisherConfig config; + config.endpoint = "tcp://127.0.0.1:*"; // let ZMQ pick a free port + config.topic = ""; + ZmqKvEventPublisher pub(config); + const std::string endpoint = pub.endpoint(); + ASSERT_FALSE(endpoint.empty()); + + zmq::context_t ctx(1); + zmq::socket_t sub(ctx, zmq::socket_type::sub); + sub.set(zmq::sockopt::subscribe, ""); + sub.set(zmq::sockopt::rcvtimeo, 200); // ms per recv + sub.connect(endpoint); + + KvEvent event; + event.type = KvEventType::kBlockStored; + event.event_id = "mooncake:default:zmqg:stored:1"; + event.tenant_id = "default"; + event.group_id = "zmqg"; + event.worker_id = "10.1.2.3:7000"; + event.medium = "EXTERNAL"; + event.block_hashes = {424242}; + event.parent_block_hash = 424241; + event.token_ids = {7, 8, 9}; + event.block_size = 16; + event.dp_rank = 1; + + // PUB/SUB has a slow-joiner window; republish until the SUB receives a + // full 3-frame message. + std::optional received; + for (int attempt = 0; attempt < 100 && !received.has_value(); ++attempt) { + pub.Publish(event); + zmq::message_t topic_frame; + auto topic_res = sub.recv(topic_frame, zmq::recv_flags::none); + if (!topic_res.has_value()) { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + continue; + } + ASSERT_TRUE(topic_frame.more()); + + zmq::message_t seq_frame; + ASSERT_TRUE(sub.recv(seq_frame, zmq::recv_flags::none).has_value()); + EXPECT_EQ(seq_frame.size(), 8u); + ASSERT_TRUE(seq_frame.more()); + + zmq::message_t payload_frame; + ASSERT_TRUE(sub.recv(payload_frame, zmq::recv_flags::none).has_value()); + EXPECT_FALSE(payload_frame.more()); + received = DecodeKvEventBatchPayload( + std::string(static_cast(payload_frame.data()), + payload_frame.size())); + } + + ASSERT_TRUE(received.has_value()); + ASSERT_EQ(received->events.size(), 1u); + EXPECT_EQ(received->events[0], event); + ASSERT_TRUE(received->dp_rank.has_value()); + EXPECT_EQ(received->dp_rank.value(), 1); + EXPECT_GT(pub.published_count(), 0u); +} +#endif // MOONCAKE_STORE_WITH_ZMQ + TEST_F(MasterServiceTest, GroupedObjectRoutesKeyLevelLookupAndRemove) { std::unique_ptr service_(new MasterService()); [[maybe_unused]] const auto context = PrepareSimpleSegment(*service_);