diff --git a/csrc/dist/distributed_radix_tree.cpp b/csrc/dist/distributed_radix_tree.cpp index 1831896861..29249f332f 100644 --- a/csrc/dist/distributed_radix_tree.cpp +++ b/csrc/dist/distributed_radix_tree.cpp @@ -571,6 +571,7 @@ std::shared_ptr RefRadixTree::match_prefix( auto last_ready_node = root; auto prefix_blocks_num = 0; auto ready_prefix_blocks_num = 0; + bool ready_prefix_open = true; auto last_node_matched_length = 0; auto physical_blocks_tensor = torch::empty({num_blocks}, torch::dtype(torch::kInt64)); auto *pb_out = physical_blocks_tensor.data_ptr(); @@ -643,9 +644,11 @@ std::shared_ptr RefRadixTree::match_prefix( ni_out[ni_write++] = (*bnis)[i]; } - if (current_node->is_ready()) { + if (ready_prefix_open && current_node->is_ready()) { last_ready_node = current_node; ready_prefix_blocks_num += matched; + } else if (matched > 0) { + ready_prefix_open = false; } prefix_blocks_num += matched; @@ -674,7 +677,7 @@ std::shared_ptr RefRadixTree::match_prefix( auto physical_blocks = physical_blocks_tensor.narrow(0, 0, pb_write); auto node_ids = node_ids_tensor.narrow(0, 0, ni_write); - return std::make_shared(prefix_blocks_num, prefix_blocks_num, last_node_matched_length, + return std::make_shared(ready_prefix_blocks_num, prefix_blocks_num, last_node_matched_length, last_ready_node, current_node, physical_blocks, node_ids); } @@ -1041,4 +1044,3 @@ void attach_and_merge_root_child(CRadixTreeIndex* temp_tree, CRadixNode* root_ch } // namespace flexkv - diff --git a/csrc/dist/local_radix_tree.cpp b/csrc/dist/local_radix_tree.cpp index bc9f81eaa7..50ff38d857 100644 --- a/csrc/dist/local_radix_tree.cpp +++ b/csrc/dist/local_radix_tree.cpp @@ -117,13 +117,19 @@ CRadixNode *LocalRadixTree::insert(torch::Tensor &physical_block_ids, } assert(last_node != nullptr); - assert(last_node_matched_length != 0 || is_root(last_node)); assert(physical_block_ids.size() == num_insert_blocks - num_matched_blocks); if (num_matched_blocks >= num_insert_blocks) { return nullptr; } + // Keep stale-match handling identical to the base radix tree. This check is + // deliberately before split/LeaseMeta allocation/current_block_count changes; + // a rejected insert leaves both the tree and caller-owned physical blocks + // untouched. + validate_insert_target(last_node, block_hashes, num_blocks, + num_matched_blocks, last_node_matched_length); + // Handle split if the last node only partially matched if (last_node_matched_length < last_node->size()) { CRadixNode *new_parent = last_node->split(last_node_matched_length); diff --git a/csrc/radix_tree.cpp b/csrc/radix_tree.cpp index 12ac707c32..d44a2a05c6 100644 --- a/csrc/radix_tree.cpp +++ b/csrc/radix_tree.cpp @@ -257,6 +257,62 @@ std::deque *CRadixNode::shrink_simple(int length) { return shrink_blocks; } +void CRadixTreeIndex::validate_insert_target( + CRadixNode *last_node, torch::Tensor &block_hashes, int num_blocks, + int num_matched_blocks, int last_node_matched_length) { + static constexpr const char *stale_match_error = + "radix insert conflict: stale match result; rematch before retrying"; + + if (last_node == nullptr || last_node->get_index() != this || + num_matched_blocks < 0 || + num_matched_blocks >= num_blocks) { + throw std::runtime_error(stale_match_error); + } + + const int node_size = last_node->size(); + auto *block_hashes_ptr = block_hashes.data_ptr(); + if (is_root(last_node)) { + if (last_node_matched_length != 0 || num_matched_blocks != 0) { + throw std::runtime_error(stale_match_error); + } + } else { + CRadixNode *parent = last_node->get_parent(); + const HashType head_hash = last_node->get_head_hash(); + if (parent == nullptr || !parent->lookup_child(head_hash) || + parent->get_child(head_hash) != last_node || + last_node_matched_length <= 0 || + last_node_matched_length > node_size) { + throw std::runtime_error(stale_match_error); + } + + const int matched_start = + num_matched_blocks - last_node_matched_length; + if (matched_start < 0) { + throw std::runtime_error(stale_match_error); + } + for (int i = 0; i < last_node_matched_length; ++i) { + if (last_node->get_hash(i) != + HashType(block_hashes_ptr[matched_start + i])) { + throw std::runtime_error(stale_match_error); + } + } + } + + const HashType target_hash = + HashType(block_hashes_ptr[num_matched_blocks]); + if (last_node_matched_length < node_size) { + // A valid partial match diverges at the split point. Equality means the + // supplied match result no longer describes this node. + if (last_node->get_hash(last_node_matched_length) == target_hash) { + throw std::runtime_error(stale_match_error); + } + } else if (last_node->lookup_child(target_hash)) { + throw std::runtime_error( + "radix insert conflict: target child already exists; " + "rematch before retrying"); + } +} + CRadixNode *CRadixTreeIndex::insert(torch::Tensor &physical_block_ids, torch::Tensor &block_hashes, int num_blocks, int num_insert_blocks, bool ready, @@ -278,13 +334,17 @@ CRadixNode *CRadixTreeIndex::insert(torch::Tensor &physical_block_ids, } assert(last_node != nullptr); - assert(last_node_matched_length != 0 || is_root(last_node)); assert(physical_block_ids.size() == num_insert_blocks - num_matched_blocks); if (num_matched_blocks >= num_insert_blocks) { return nullptr; } + // This must run before allocating/splitting/removing leaves: on conflict the + // tree is unchanged and the caller retains every supplied physical block. + validate_insert_target(last_node, block_hashes, num_blocks, + num_matched_blocks, last_node_matched_length); + auto new_node = new CRadixNode(this, ready, 0); auto &new_block_hashes = new_node->get_block_hashes(); auto &new_physical_blocks = new_node->get_physical_blocks(); @@ -574,6 +634,9 @@ CRadixTreeIndex::match_prefix(torch::Tensor &block_hashes, int num_blocks, auto last_ready_node = root; auto prefix_blocks_num = 0; auto ready_prefix_blocks_num = 0; + // Readiness is a contiguous-prefix property. Once a matched node is unready, + // a ready descendant must not be counted until that ancestor becomes ready. + bool ready_prefix_open = true; auto last_node_matched_length = 0; // SWA (node-mount): deepest fully-matched ready node carrying a live SWA // slot. @@ -613,7 +676,7 @@ CRadixTreeIndex::match_prefix(torch::Tensor &block_hashes, int num_blocks, break; } } - if (current_node->is_ready()) { + if (ready_prefix_open && current_node->is_ready()) { last_ready_node = current_node; ready_prefix_blocks_num += matched_length; // SWA hit only when the WHOLE node matched (its trailing page is @@ -622,6 +685,8 @@ CRadixTreeIndex::match_prefix(torch::Tensor &block_hashes, int num_blocks, last_swa_node = current_node; swa_hit_blocks = ready_prefix_blocks_num; } + } else if (matched_length > 0) { + ready_prefix_open = false; } last_node_matched_length = matched_length; prefix_blocks_num += matched_length; @@ -637,7 +702,7 @@ CRadixTreeIndex::match_prefix(torch::Tensor &block_hashes, int num_blocks, if (child_hash_opt.has_value() && current_node->lookup_child(child_hash_opt.value())) { child_hash = child_hash_opt.value(); - if (current_node->is_ready()) { + if (ready_prefix_open && current_node->is_ready()) { last_ready_node = current_node; ready_prefix_blocks_num += current_node->size(); // Whole node matched (we are descending into a child): expose its SWA. @@ -645,6 +710,8 @@ CRadixTreeIndex::match_prefix(torch::Tensor &block_hashes, int num_blocks, last_swa_node = current_node; swa_hit_blocks = ready_prefix_blocks_num; } + } else if (current_node->size() > 0) { + ready_prefix_open = false; } prefix_blocks_num += current_node->size(); auto &pbs = current_node->get_physical_blocks(); @@ -683,7 +750,7 @@ CRadixTreeIndex::match_prefix(torch::Tensor &block_hashes, int num_blocks, matched_length = 0; } - if (current_node->is_ready()) { + if (ready_prefix_open && current_node->is_ready()) { last_ready_node = current_node; ready_prefix_blocks_num += matched_length; // Only a full node match exposes the trailing page's SWA. @@ -691,6 +758,8 @@ CRadixTreeIndex::match_prefix(torch::Tensor &block_hashes, int num_blocks, last_swa_node = current_node; swa_hit_blocks = ready_prefix_blocks_num; } + } else if (matched_length > 0) { + ready_prefix_open = false; } last_node_matched_length = matched_length; diff --git a/csrc/radix_tree.h b/csrc/radix_tree.h index 64ce400229..db2dc867d9 100644 --- a/csrc/radix_tree.h +++ b/csrc/radix_tree.h @@ -340,6 +340,15 @@ class CRadixTreeIndex { std::vector &out_blocks, std::vector *out_hashes); + // Validate a caller-supplied match anchor before any split, allocation, or + // child-map mutation. A conflict raises std::runtime_error; ownership of the + // supplied physical blocks remains with the caller so it can rematch/retry or + // recycle them. LocalRadixTree reuses the same guard in its insert override. + void validate_insert_target(CRadixNode *last_node, + torch::Tensor &block_hashes, int num_blocks, + int num_matched_blocks, + int last_node_matched_length); + public: CRadixTreeIndex(int tokens_per_block, int max_num_blocks = 1000000, int hit_reward_seconds = 0, diff --git a/flexkv/cache/cache_engine.py b/flexkv/cache/cache_engine.py index 49a58adfe2..8bb6e29b31 100644 --- a/flexkv/cache/cache_engine.py +++ b/flexkv/cache/cache_engine.py @@ -16,7 +16,7 @@ import logging import threading import time -from functools import partial +from functools import partial, wraps from queue import Queue from typing import List, Tuple, Optional, Dict, Callable from dataclasses import dataclass, field @@ -34,6 +34,8 @@ from flexkv.common.block import SequenceMeta, format_block_hash from flexkv.common.config import CacheConfig, ModelConfig, GLOBAL_CONFIG_FROM_ENV, SWAPoolConfig from flexkv.common.transfer import ( + CompletedOp, + CompletionAwareCallback, DeviceType, TransferOpGraph, TransferOp, @@ -53,6 +55,58 @@ _VALID_EVICTION_POLICIES = {'lru', 'lfu', 'slru', 'fifo', 'mru', 'filo'} +def _synchronized_cache_tree(method: Callable) -> Callable: + """Serialize radix-tree planning and completion-time mutations.""" + @wraps(method) + def wrapped(self, *args, **kwargs): + with self._cache_tree_lock: + return method(self, *args, **kwargs) + return wrapped + + +@dataclass +class MooncakeLoadResult: + """Per-request Mooncake outcome populated by the REMOTE2H callback.""" + + block_results: Optional[Tuple[bool, ...]] = None + + def record(self, completed_op: Optional[CompletedOp]) -> None: + self.block_results = ( + None if completed_op is None else completed_op.block_results) + + def successful_prefix(self, expected_blocks: int) -> int: + if (self.block_results is None + or len(self.block_results) != expected_blocks): + return 0 + prefix = 0 + for succeeded in self.block_results: + if not succeeded: + break + prefix += 1 + return prefix + + +@dataclass(frozen=True) +class DeferredCacheInsert: + """Detached tier blocks published by the graph-completion callback. + + ``load_result`` is present for Mooncake reads, where only the longest + successful remote prefix is valid. PUT staging has already completed all + of its graph consumers when this record is committed, so ``None`` means the + complete staged range is valid. + """ + + device_type: DeviceType + sequence_meta: SequenceMeta + physical_blocks: np.ndarray + staged_start_block: int + remote_start_block: int + requested_end_block: int + load_result: Optional[MooncakeLoadResult] = None + swa_slot: int = -1 + publish_to_peer: bool = False + + @dataclass class GetTransferPlan: transfer_graph: TransferOpGraph @@ -61,6 +115,7 @@ class GetTransferPlan: op_callback_dict: Dict[int, Callable] buffer_to_free: Dict[DeviceType, np.ndarray] num_gpu_blocks_to_transfer: int + deferred_inserts: List[DeferredCacheInsert] = field(default_factory=list) # SWA read reservation held by this plan; released by an op callback on the # normal path, or by the abort path when the plan is cancelled unlaunched. swa_reservation: Optional["SWAReadReservation"] = None @@ -86,6 +141,7 @@ class PutTransferPlan: buffer_to_free: Dict[DeviceType, np.ndarray] num_gpu_blocks_to_transfer: int skipped_gpu_blocks: int + deferred_inserts: List[DeferredCacheInsert] = field(default_factory=list) # SWA slots reserved for this put but not yet mounted (publication happens # in an op callback); the abort path must return them to the host pool. swa_slots_to_free: List[Tuple[DeviceType, int]] = field(default_factory=list) @@ -166,6 +222,11 @@ def abort(self) -> None: self._consumed = True self._abort() + @property + def keywords(self) -> Dict: + """Expose completion-partial metadata for existing callback users.""" + return getattr(self._complete, "keywords", {}) + class CacheEngineAccel: def __init__(self, @@ -782,6 +843,9 @@ class CacheStrategy: class GlobalCacheEngine: def __init__(self, cache_config: CacheConfig, model_config: ModelConfig, redis_meta: RedisMeta = None, event_collector: Optional[KVEventCollector] = None): + # pybind releases the GIL around radix match/insert/evict. Protect the + # tree across planning and callback-time rematch+insert transactions. + self._cache_tree_lock = threading.RLock() self.cache_config = cache_config self.model_config = model_config self.tokens_per_block = cache_config.tokens_per_block @@ -953,6 +1017,7 @@ def start(self) -> None: if self.remote_cache_engine and self.cache_config.enable_3rd_remote: self.remote_cache_engine.start() + @_synchronized_cache_tree def reset(self) -> None: if self.cpu_cache_engine: self.cpu_cache_engine.reset() @@ -974,6 +1039,7 @@ def _update_mempool_metrics(self) -> None: engine.mempool.num_free_blocks ) + @_synchronized_cache_tree def get(self, request_id: int, token_ids: np.ndarray, @@ -1072,10 +1138,12 @@ def get(self, callback = TransferPlanHandle( complete=partial(self._transfer_callback, node_to_unlock=plan.node_to_unlock, - buffer_to_free=plan.buffer_to_free), + buffer_to_free=plan.buffer_to_free, + deferred_inserts=plan.deferred_inserts), abort=partial(self._abort_transfer_plan, node_to_unlock=plan.node_to_unlock, buffer_to_free=plan.buffer_to_free, + deferred_inserts=plan.deferred_inserts, swa_reservation=plan.swa_reservation), ) @@ -1112,6 +1180,7 @@ def combined_callback() -> None: op_callback_dict[op_id] = combined_callback + @_synchronized_cache_tree def _publish_swa_put_slot(self, device_type: DeviceType, node, @@ -1244,6 +1313,8 @@ def _get_impl_global(self, fragment12_num_blocks = max(len(cpu_matched_blocks), len(ssd_matched_blocks)) fragment3_num_blocks = max(len(remote_matched_blocks) - fragment12_num_blocks, 0) fragment23_num_blocks = fragment2_num_blocks + fragment3_num_blocks + defer_mooncake_commit = ( + self.use_mooncake_store_backend and fragment3_num_blocks > 0) fragment123_gpu_blocks = gpu_block_ids[:fragment123_num_blocks] fragment123_cpu_blocks = cpu_matched_blocks @@ -1278,19 +1349,22 @@ def _get_impl_global(self, self._metrics_collector.record_allocation_failure("global") return self._empty_get_return(request_id) fragment123_cpu_blocks = np.concatenate([fragment123_cpu_blocks, fragment23_cpu_blocks]) - # we only insert the buffer blocks to cpu cache engine only: - # 1. the cpu cache engine satisfies prefix cache after insertion - # 2. the sequence is all ready blocks - if (cpu_matched_result.num_ready_matched_blocks >= block_mask_start and - cpu_matched_result.num_ready_matched_blocks == cpu_matched_result.num_matched_blocks): - cpu_node_to_unlock = self.cpu_cache_engine.insert(sequence_meta, - fragment23_cpu_blocks, - num_insert_blocks=fragment123_num_blocks + \ - block_mask_start, - is_ready=False, - match_result=cpu_matched_result) - else: - cpu_blocks_to_free = fragment23_cpu_blocks + # Mooncake can partially fail after match. Keep its staging blocks + # detached until graph completion; the callback will fresh-rematch + # and publish only the successfully loaded prefix. + if not defer_mooncake_commit: + if (cpu_matched_result.num_ready_matched_blocks >= block_mask_start and + cpu_matched_result.num_ready_matched_blocks == + cpu_matched_result.num_matched_blocks): + cpu_node_to_unlock = self.cpu_cache_engine.insert( + sequence_meta, + fragment23_cpu_blocks, + num_insert_blocks=fragment123_num_blocks + block_mask_start, + is_ready=False, + match_result=cpu_matched_result, + ) + else: + cpu_blocks_to_free = fragment23_cpu_blocks # Record cache hit/miss metrics after confirming successful allocation if self._metrics_collector is not None: @@ -1341,7 +1415,8 @@ def _get_impl_global(self, if (enable_ssd and op_remote2h is not None and ssd_matched_result.num_ready_matched_blocks >= block_mask_start and - ssd_matched_result.num_ready_matched_blocks == ssd_matched_result.num_matched_blocks): + ssd_matched_result.num_ready_matched_blocks == ssd_matched_result.num_matched_blocks and + ssd_matched_result.num_matched_blocks == block_mask_start + fragment12_num_blocks): # only when the above all are satisfied, we load data back from cpu to ssd write_ssd_blocks_from_remote = True fragment3_ssd_blocks = self.ssd_cache_engine.take( @@ -1363,12 +1438,14 @@ def _get_impl_global(self, transfer_graph.add_transfer_op(op_h2disk) transfer_graph.add_dependency(op_h2disk.op_id, op_remote2h.op_id) - ssd_node_to_unlock = self.ssd_cache_engine.insert(sequence_meta, - fragment3_ssd_blocks, - num_insert_blocks=fragment123_num_blocks + \ - block_mask_start, - is_ready=False, - match_result=ssd_matched_result) + if not defer_mooncake_commit: + ssd_node_to_unlock = self.ssd_cache_engine.insert( + sequence_meta, + fragment3_ssd_blocks, + num_insert_blocks=fragment123_num_blocks + block_mask_start, + is_ready=False, + match_result=ssd_matched_result, + ) if enable_gpu: op_h2d = TransferOp( graph_id = transfer_graph.graph_id, @@ -1386,15 +1463,48 @@ def _get_impl_global(self, node_to_unlock = {} if cpu_node_to_unlock is not None: - node_to_unlock[DeviceType.CPU] = (cpu_node_to_unlock, cpu_node_to_unlock.size()) + node_to_unlock[DeviceType.CPU] = ( + cpu_node_to_unlock, + 0 if defer_mooncake_commit else cpu_node_to_unlock.size(), + ) if ssd_node_to_unlock is not None: - node_to_unlock[DeviceType.SSD] = (ssd_node_to_unlock, ssd_node_to_unlock.size()) + node_to_unlock[DeviceType.SSD] = ( + ssd_node_to_unlock, + 0 if defer_mooncake_commit else ssd_node_to_unlock.size(), + ) if remote_node_to_unlock is not None: node_to_unlock[DeviceType.REMOTE] = (remote_node_to_unlock, remote_node_to_unlock.size()) buffer_to_free = {DeviceType.CPU: cpu_blocks_to_free} num_gpu_blocks_to_transfer = len(fragment123_gpu_blocks) if enable_gpu else 0 op_callback_dict = {} + deferred_inserts: List[DeferredCacheInsert] = [] + if defer_mooncake_commit: + assert op_remote2h is not None + load_result = MooncakeLoadResult() + op_callback_dict[op_remote2h.op_id] = CompletionAwareCallback( + load_result.record) + remote_start_block = block_mask_start + fragment12_num_blocks + requested_end_block = block_mask_start + fragment123_num_blocks + deferred_inserts.append(DeferredCacheInsert( + device_type=DeviceType.CPU, + sequence_meta=sequence_meta, + physical_blocks=fragment23_cpu_blocks, + staged_start_block=block_mask_start + fragment1_num_blocks, + remote_start_block=remote_start_block, + requested_end_block=requested_end_block, + load_result=load_result, + )) + if write_ssd_blocks_from_remote: + deferred_inserts.append(DeferredCacheInsert( + device_type=DeviceType.SSD, + sequence_meta=sequence_meta, + physical_blocks=fragment3_ssd_blocks, + staged_start_block=remote_start_block, + remote_start_block=remote_start_block, + requested_end_block=requested_end_block, + load_result=load_result, + )) if swa_reservation is not None: assert num_gpu_blocks_to_transfer > 0 finished_ops_ids.append(swa_reservation.h2d_id) @@ -1412,6 +1522,7 @@ def _get_impl_global(self, op_callback_dict=op_callback_dict, buffer_to_free=buffer_to_free, num_gpu_blocks_to_transfer=num_gpu_blocks_to_transfer, + deferred_inserts=deferred_inserts, swa_reservation=swa_reservation, ) @@ -1690,6 +1801,7 @@ def _get_impl_local(self, swa_reservation=swa_reservation, ) + @_synchronized_cache_tree def put(self, request_id: int, token_ids: np.ndarray, @@ -1756,10 +1868,12 @@ def put(self, complete=partial(self._transfer_callback, node_to_unlock=plan.node_to_unlock, buffer_to_free=plan.buffer_to_free, + deferred_inserts=plan.deferred_inserts, is_put=True), abort=partial(self._abort_transfer_plan, node_to_unlock=plan.node_to_unlock, buffer_to_free=plan.buffer_to_free, + deferred_inserts=plan.deferred_inserts, swa_slots_to_free=plan.swa_slots_to_free), ) @@ -1811,16 +1925,34 @@ def _put_impl_global(self, else: cpu_matched_result, ssd_matched_result, remote_matched_result = self.match_all(sequence_meta, temp_cache_strategy=temp_cache_strategy) + defer_put_commit = self.use_mooncake_store_backend + if defer_put_commit: + # PUT sources must be addresses in this process's registered host + # buffers. A hierarchical match may select a peer node, which is a + # valid GET source but cannot back local H2DISK/H2REMOTE writes. + match_cpu_local = getattr(self.cpu_cache_engine, "match_local", None) + if callable(match_cpu_local): + cpu_matched_result = match_cpu_local(sequence_meta) + if enable_ssd: + match_ssd_local = getattr(self.ssd_cache_engine, "match_local", None) + if callable(match_ssd_local): + ssd_matched_result = match_ssd_local(sequence_meta) + cpu_matched_count = ( + cpu_matched_result.num_ready_matched_blocks + if defer_put_commit else cpu_matched_result.num_matched_blocks) + ssd_matched_count = ( + ssd_matched_result.num_ready_matched_blocks + if defer_put_commit else ssd_matched_result.num_matched_blocks) cpu_matched_blocks = cpu_matched_result.physical_blocks[ - :cpu_matched_result.num_matched_blocks][block_mask_start:block_mask_end] + :cpu_matched_count][block_mask_start:block_mask_end] ssd_matched_blocks = ssd_matched_result.physical_blocks[ - :ssd_matched_result.num_matched_blocks][block_mask_start:block_mask_end] + :ssd_matched_count][block_mask_start:block_mask_end] remote_matched_blocks = remote_matched_result.physical_blocks[ :remote_matched_result.num_matched_blocks][block_mask_start:block_mask_end] num_skipped_blocks = len(cpu_matched_blocks) fragment12_num_blocks = len(gpu_block_ids) - num_skipped_blocks - if fragment12_num_blocks == 0: + if fragment12_num_blocks == 0 and not defer_put_commit: return self._empty_put_return(request_id) fragment2_num_blocks = len(gpu_block_ids) - len(ssd_matched_blocks) if not enable_ssd: @@ -1836,11 +1968,17 @@ def _put_impl_global(self, remote_put_hit_blocks = len(remote_matched_blocks) fragment3_num_blocks = len(gpu_block_ids) - len(remote_matched_blocks) + if (fragment12_num_blocks == 0 + and fragment2_num_blocks == 0 + and fragment3_num_blocks == 0): + return self._empty_put_return(request_id) + fragment12_gpu_blocks = gpu_block_ids[num_skipped_blocks:] fragment12_cpu_blocks = self.cpu_cache_engine.take( num_required_blocks=fragment12_num_blocks, - protected_node = cpu_matched_result.last_node, + protected_node=(cpu_matched_result.last_ready_node + if defer_put_commit else cpu_matched_result.last_node), strict=False ) if len(fragment12_cpu_blocks) < fragment12_num_blocks: @@ -1850,7 +1988,8 @@ def _put_impl_global(self, if enable_ssd and fragment2_num_blocks > 0: fragment2_ssd_blocks = self.ssd_cache_engine.take( num_required_blocks=fragment2_num_blocks, - protected_node = ssd_matched_result.last_node, + protected_node=(ssd_matched_result.last_ready_node + if defer_put_commit else ssd_matched_result.last_node), strict=False ) if len(fragment2_ssd_blocks) == fragment2_num_blocks: @@ -1880,10 +2019,12 @@ def _put_impl_global(self, if self.swa_op_constructor.enabled: cpu_swa_slot = self.cpu_cache_engine._alloc_swa_slot( - cpu_matched_result.last_node) + cpu_matched_result.last_ready_node + if defer_put_commit else cpu_matched_result.last_node) if cpu_swa_slot >= 0 and put_to_ssd: ssd_swa_slot = self.ssd_cache_engine._alloc_swa_slot( - ssd_matched_result.last_node) + ssd_matched_result.last_ready_node + if defer_put_commit else ssd_matched_result.last_node) if (cpu_swa_slot >= 0 and (not put_to_ssd or ssd_swa_slot >= 0) and put_to_remote): @@ -1914,27 +2055,29 @@ def _put_impl_global(self, transfer_graph = TransferOpGraph() finished_ops_ids = [] op_node_to_ready = {} + op_d2h = None + if fragment12_num_blocks > 0: + op_d2h = TransferOp( + graph_id=transfer_graph.graph_id, + transfer_type=TransferType.D2H, + src_block_ids=fragment12_gpu_blocks, + dst_block_ids=fragment12_cpu_blocks, + dp_client_id=dp_client_id, + ) + flexkv_logger.info( + "[FlexKV-SEGV-DEBUG] cache_engine create D2H op (global_put) " + f"request_id={request_id}, op_id={op_d2h.op_id}, " + f"graph_id={transfer_graph.graph_id}, dp_client_id={dp_client_id}, " + f"fragment12_num_blocks={fragment12_num_blocks}, " + f"fragment2_num_blocks={fragment2_num_blocks}, " + f"fragment3_num_blocks={fragment3_num_blocks}, " + f"{summarize_id_tensor('gpu_src', fragment12_gpu_blocks)}, " + f"{summarize_id_tensor('cpu_dst', fragment12_cpu_blocks)}" + ) + transfer_graph.add_transfer_op(op_d2h) + finished_ops_ids.append(op_d2h.op_id) - op_d2h = TransferOp( - graph_id = transfer_graph.graph_id, - transfer_type = TransferType.D2H, - src_block_ids = fragment12_gpu_blocks, - dst_block_ids = fragment12_cpu_blocks, - dp_client_id = dp_client_id, - ) - flexkv_logger.info( - "[FlexKV-SEGV-DEBUG] cache_engine create D2H op (global_put) " - f"request_id={request_id}, op_id={op_d2h.op_id}, " - f"graph_id={transfer_graph.graph_id}, dp_client_id={dp_client_id}, " - f"fragment12_num_blocks={fragment12_num_blocks}, " - f"fragment2_num_blocks={fragment2_num_blocks}, " - f"fragment3_num_blocks={fragment3_num_blocks}, " - f"{summarize_id_tensor('gpu_src', fragment12_gpu_blocks)}, " - f"{summarize_id_tensor('cpu_dst', fragment12_cpu_blocks)}" - ) - transfer_graph.add_transfer_op(op_d2h) - finished_ops_ids.append(op_d2h.op_id) - + op_h2disk = None if put_to_ssd: if len(fragment12_cpu_blocks) < fragment2_num_blocks: num_needed_from_cpu_matched = fragment2_num_blocks - len(fragment12_cpu_blocks) @@ -1951,8 +2094,10 @@ def _put_impl_global(self, ) transfer_graph.add_transfer_op(op_h2disk) - transfer_graph.add_dependency(op_h2disk.op_id, op_d2h.op_id) + if op_d2h is not None: + transfer_graph.add_dependency(op_h2disk.op_id, op_d2h.op_id) + op_h2remote = None if put_to_remote: if fragment3_num_blocks > fragment12_num_blocks: extra_num_cpu_blocks = fragment3_num_blocks - fragment12_num_blocks @@ -1975,7 +2120,15 @@ def _put_impl_global(self, mooncake_store_block_hashes = mooncake_block_hashes, ) transfer_graph.add_transfer_op(op_h2remote) - transfer_graph.add_dependency(op_h2remote.op_id, op_d2h.op_id) + if op_d2h is not None: + transfer_graph.add_dependency(op_h2remote.op_id, op_d2h.op_id) + + if op_d2h is None: + assert defer_put_commit + if op_h2disk is not None: + finished_ops_ids.append(op_h2disk.op_id) + if op_h2remote is not None: + finished_ops_ids.append(op_h2remote.op_id) if cpu_swa_slot >= 0: empty = np.array([], dtype=np.int64) @@ -2006,6 +2159,55 @@ def _put_impl_global(self, assert swa_ops.h2remote_id is not None finished_ops_ids.append(swa_ops.d2h_id) + if defer_put_commit: + deferred_inserts: List[DeferredCacheInsert] = [] + node_to_unlock = {} + requested_end = block_mask_start + len(gpu_block_ids) + if fragment12_num_blocks > 0 or cpu_swa_slot >= 0: + deferred_inserts.append(DeferredCacheInsert( + device_type=DeviceType.CPU, + sequence_meta=sequence_meta, + physical_blocks=fragment12_cpu_blocks, + staged_start_block=block_mask_start + num_skipped_blocks, + remote_start_block=block_mask_start + num_skipped_blocks, + requested_end_block=requested_end, + swa_slot=cpu_swa_slot, + publish_to_peer=self.cache_config.enable_p2p_cpu, + )) + if put_to_ssd: + deferred_inserts.append(DeferredCacheInsert( + device_type=DeviceType.SSD, + sequence_meta=sequence_meta, + physical_blocks=fragment2_ssd_blocks, + staged_start_block=block_mask_start + len(ssd_matched_blocks), + remote_start_block=block_mask_start + len(ssd_matched_blocks), + requested_end_block=requested_end, + swa_slot=ssd_swa_slot, + publish_to_peer=self.cache_config.enable_p2p_ssd, + )) + + # Existing ready CPU blocks may feed H2DISK/H2REMOTE while the graph + # is running. Pin their deepest node until every consumer finishes. + cpu_anchor = cpu_matched_result.last_ready_node + if len(cpu_matched_blocks) > 0 and cpu_anchor is not None: + node_to_unlock[DeviceType.CPU] = (cpu_anchor, 0) + ssd_anchor = ssd_matched_result.last_ready_node + if (put_to_ssd and len(ssd_matched_blocks) > 0 + and ssd_anchor is not None): + node_to_unlock[DeviceType.SSD] = (ssd_anchor, 0) + skipped_gpu_blocks = len(cpu_matched_blocks) + return PutTransferPlan( + transfer_graph=transfer_graph, + finished_ops_ids=finished_ops_ids, + node_to_unlock=node_to_unlock, + op_callback_dict={}, + buffer_to_free={}, + num_gpu_blocks_to_transfer=len(fragment12_gpu_blocks), + skipped_gpu_blocks=skipped_gpu_blocks, + deferred_inserts=deferred_inserts, + ) + + assert op_d2h is not None cpu_node_to_unlock = self.cpu_cache_engine.insert( sequence_meta, fragment12_cpu_blocks, @@ -2023,7 +2225,8 @@ def _put_impl_global(self, match_result=ssd_matched_result, ) op_node_to_ready[op_h2disk.op_id] = ( - DeviceType.SSD, ssd_node_to_unlock, ssd_node_to_unlock.size()) + DeviceType.SSD, ssd_node_to_unlock, + ssd_node_to_unlock.size()) remote_node_to_unlock = None if put_to_remote: remote_node_to_unlock = self.remote_cache_engine.insert( @@ -2039,11 +2242,14 @@ def _put_impl_global(self, ) node_to_unlock = {} if cpu_node_to_unlock is not None: - node_to_unlock[DeviceType.CPU] = (cpu_node_to_unlock, cpu_node_to_unlock.size()) + node_to_unlock[DeviceType.CPU] = ( + cpu_node_to_unlock, cpu_node_to_unlock.size()) if ssd_node_to_unlock is not None: - node_to_unlock[DeviceType.SSD] = (ssd_node_to_unlock, ssd_node_to_unlock.size()) + node_to_unlock[DeviceType.SSD] = ( + ssd_node_to_unlock, ssd_node_to_unlock.size()) if remote_node_to_unlock is not None: - node_to_unlock[DeviceType.REMOTE] = (remote_node_to_unlock, remote_node_to_unlock.size()) + node_to_unlock[DeviceType.REMOTE] = ( + remote_node_to_unlock, remote_node_to_unlock.size()) op_callback_dict = self._build_op_callback_dict(op_node_to_ready) if cpu_swa_slot >= 0: @@ -2065,7 +2271,8 @@ def _put_impl_global(self, op_callback_dict, swa_ops.h2remote_id, partial(self._publish_swa_put_slot, - DeviceType.REMOTE, remote_node_to_unlock, remote_swa_slot), + DeviceType.REMOTE, remote_node_to_unlock, + remote_swa_slot), ) skipped_gpu_blocks = len(cpu_matched_blocks) swa_slots_to_free = [(device_type, slot) for device_type, slot in @@ -2298,46 +2505,226 @@ def _put_impl_local(self, swa_slots_to_free=swa_slots_to_free, ) + @staticmethod + def _matched_boundary_node(current_match, matched_blocks: int): + """Return the matched node only when ``matched_blocks`` ends on it.""" + if int(current_match.num_matched_blocks) != matched_blocks: + return None + node = current_match.last_node + if node is None: + return None + if int(current_match.last_node_matched_length) != int(node.size()): + return None + return node + + @staticmethod + def _release_pending_swa_slot(engine, pending: DeferredCacheInsert) -> None: + if pending.swa_slot >= 0: + engine._free_swa_slot(pending.swa_slot) + + def _discard_deferred_insert( + self, engine, pending: DeferredCacheInsert, + physical_blocks: np.ndarray) -> None: + """Release staging that is still wholly owned by this request.""" + engine.recycle(physical_blocks) + self._release_pending_swa_slot(engine, pending) + + @staticmethod + def _publish_pending_swa_slot( + engine, pending: DeferredCacheInsert, node) -> None: + if pending.swa_slot < 0: + return + if node is None or node.has_swa(): + engine._free_swa_slot(pending.swa_slot) + return + engine.index.set_swa(node, int(pending.swa_slot)) + engine._drain_unmounted_swa_slots() + + def _commit_deferred_insert(self, pending: DeferredCacheInsert): + """Fresh-rematch and atomically publish one valid staging prefix.""" + engine = self.cache_engines[pending.device_type] + physical_blocks = np.asarray(pending.physical_blocks, dtype=np.int64) + staged_blocks = pending.requested_end_block - pending.staged_start_block + remote_blocks = pending.requested_end_block - pending.remote_start_block + if (staged_blocks != len(physical_blocks) + or pending.staged_start_block > pending.remote_start_block + or remote_blocks < 0): + self._discard_deferred_insert(engine, pending, physical_blocks) + flexkv_logger.error( + "Invalid deferred cache insert range: " + f"staged=[{pending.staged_start_block}, " + f"{pending.requested_end_block}), " + f"remote_start={pending.remote_start_block}, " + f"physical_blocks={len(physical_blocks)}") + return None + + if pending.load_result is None: + publish_end = pending.requested_end_block + else: + successful_remote = pending.load_result.successful_prefix(remote_blocks) + publish_end = pending.remote_start_block + successful_remote + + # Hierarchical engines must rematch their local tree. match() may choose + # a distributed peer node, which is not a valid insertion anchor here. + match_local = getattr(engine, "match_local", None) + try: + current_match = ( + match_local(pending.sequence_meta) + if callable(match_local) + else engine.match(pending.sequence_meta) + ) + except Exception: + # No radix mutation has started, so all staging is still ours. + self._discard_deferred_insert(engine, pending, physical_blocks) + raise + current_blocks = int(current_match.num_matched_blocks) + ready_blocks = int(current_match.num_ready_matched_blocks) + + # Never attach below, or mark ready through, another in-flight writer's + # unready node. The staged allocation remains ours and is safe to recycle. + if (current_blocks != ready_blocks + or current_blocks < pending.staged_start_block): + self._discard_deferred_insert(engine, pending, physical_blocks) + return None + + if current_blocks >= publish_end: + engine.recycle(physical_blocks) + boundary_node = self._matched_boundary_node( + current_match, pending.requested_end_block) + if pending.swa_slot >= 0: + if boundary_node is None or publish_end != pending.requested_end_block: + self._release_pending_swa_slot(engine, pending) + else: + self._publish_pending_swa_slot( + engine, pending, boundary_node) + return boundary_node + + successful_staged_blocks = publish_end - pending.staged_start_block + if successful_staged_blocks <= 0: + self._discard_deferred_insert(engine, pending, physical_blocks) + return None + + skipped_blocks = current_blocks - pending.staged_start_block + blocks_to_insert = physical_blocks[ + skipped_blocks:successful_staged_blocks] + + try: + node = engine.insert( + pending.sequence_meta, + blocks_to_insert, + num_insert_blocks=publish_end, + is_ready=False, + match_result=current_match, + ) + except RuntimeError as error: + # The radix guard raises before any mutation and deliberately leaves + # block ownership with the caller. Other insert failures may happen + # after mutation, so keep those fail-closed. + if not str(error).startswith("radix insert conflict:"): + self._release_pending_swa_slot(engine, pending) + raise + self._discard_deferred_insert(engine, pending, physical_blocks) + return None + except Exception: + self._release_pending_swa_slot(engine, pending) + raise + if node is None: + self._discard_deferred_insert(engine, pending, physical_blocks) + return None + + unused_blocks = np.concatenate(( + physical_blocks[:skipped_blocks], + physical_blocks[successful_staged_blocks:], + )) + if len(unused_blocks) > 0: + engine.recycle(unused_blocks) + + # Keep the inserted length explicit. set_ready supports split fragments, + # while this transaction keeps rematch, insert, and readiness atomic. + try: + engine.set_ready(node, True, len(blocks_to_insert)) + except Exception: + # Inserted blocks now belong to the unready tree node and must not + # be recycled, but the detached SWA slot is still ours. + self._release_pending_swa_slot(engine, pending) + raise + + if pending.swa_slot >= 0: + if publish_end == pending.requested_end_block: + self._publish_pending_swa_slot(engine, pending, node) + else: + self._release_pending_swa_slot(engine, pending) + + if pending.publish_to_peer: + engine.local_index.insert_and_publish(node) + return node + + @_synchronized_cache_tree def _transfer_callback(self, node_to_unlock: Dict[DeviceType, Tuple[RadixNode, int]], buffer_to_free: Optional[Dict[DeviceType, np.ndarray]] = None, + deferred_inserts: Optional[List[DeferredCacheInsert]] = None, is_put: bool = False) -> None: - if DeviceType.CPU in node_to_unlock: - assert self.cpu_cache_engine is not None - cpu_node = node_to_unlock[DeviceType.CPU][0] - self.cpu_cache_engine.unlock(cpu_node) - self.cpu_cache_engine.set_ready(cpu_node, True, node_to_unlock[DeviceType.CPU][1]) - if is_put and self.cache_config.enable_p2p_cpu: - self.cpu_cache_engine.local_index.insert_and_publish(cpu_node) - if DeviceType.SSD in node_to_unlock: - assert self.ssd_cache_engine is not None - ssd_node = node_to_unlock[DeviceType.SSD][0] - self.ssd_cache_engine.unlock(ssd_node) - self.ssd_cache_engine.set_ready(ssd_node, True, node_to_unlock[DeviceType.SSD][1]) - if is_put and self.cache_config.enable_p2p_ssd: - self.ssd_cache_engine.local_index.insert_and_publish(node_to_unlock[DeviceType.SSD][0]) - if DeviceType.REMOTE in node_to_unlock: - assert self.remote_cache_engine is not None - self.remote_cache_engine.unlock(node_to_unlock[DeviceType.REMOTE][0]) - self.remote_cache_engine.set_ready( - node_to_unlock[DeviceType.REMOTE][0], True, node_to_unlock[DeviceType.REMOTE][1] - ) - if is_put and self.enable_kv_sharing: - self.remote_cache_engine.insert_and_publish(node_to_unlock[DeviceType.REMOTE][0]) - if buffer_to_free is not None: - if DeviceType.CPU in buffer_to_free: + try: + for pending in deferred_inserts or []: + try: + self._commit_deferred_insert(pending) + except Exception: + # Never strand the request's pre-existing radix-node locks. + # The commit helper recycles blocks on every known pre-insert + # rejection; an unexpected post-insert exception has uncertain + # ownership and must not return those blocks to the mempool. + flexkv_logger.error( + "Deferred cache publication failed: " + f"device={pending.device_type.name}", + exc_info=True, + ) + finally: + if DeviceType.CPU in node_to_unlock: assert self.cpu_cache_engine is not None - self.cpu_cache_engine.recycle(buffer_to_free[DeviceType.CPU]) - if DeviceType.SSD in buffer_to_free: + cpu_node = node_to_unlock[DeviceType.CPU][0] + self.cpu_cache_engine.unlock(cpu_node) + ready_length = node_to_unlock[DeviceType.CPU][1] + if ready_length > 0: + self.cpu_cache_engine.set_ready(cpu_node, True, ready_length) + if (is_put and ready_length > 0 + and self.cache_config.enable_p2p_cpu): + self.cpu_cache_engine.local_index.insert_and_publish(cpu_node) + if DeviceType.SSD in node_to_unlock: assert self.ssd_cache_engine is not None - self.ssd_cache_engine.recycle(buffer_to_free[DeviceType.SSD]) - if DeviceType.REMOTE in buffer_to_free: + ssd_node = node_to_unlock[DeviceType.SSD][0] + self.ssd_cache_engine.unlock(ssd_node) + ready_length = node_to_unlock[DeviceType.SSD][1] + if ready_length > 0: + self.ssd_cache_engine.set_ready(ssd_node, True, ready_length) + if (is_put and ready_length > 0 + and self.cache_config.enable_p2p_ssd): + self.ssd_cache_engine.local_index.insert_and_publish(node_to_unlock[DeviceType.SSD][0]) + if DeviceType.REMOTE in node_to_unlock: assert self.remote_cache_engine is not None - self.remote_cache_engine.recycle(buffer_to_free[DeviceType.REMOTE]) - + self.remote_cache_engine.unlock(node_to_unlock[DeviceType.REMOTE][0]) + ready_length = node_to_unlock[DeviceType.REMOTE][1] + if ready_length > 0: + self.remote_cache_engine.set_ready( + node_to_unlock[DeviceType.REMOTE][0], True, ready_length) + if is_put and self.enable_kv_sharing: + self.remote_cache_engine.insert_and_publish(node_to_unlock[DeviceType.REMOTE][0]) + if buffer_to_free is not None: + if DeviceType.CPU in buffer_to_free: + assert self.cpu_cache_engine is not None + self.cpu_cache_engine.recycle(buffer_to_free[DeviceType.CPU]) + if DeviceType.SSD in buffer_to_free: + assert self.ssd_cache_engine is not None + self.ssd_cache_engine.recycle(buffer_to_free[DeviceType.SSD]) + if DeviceType.REMOTE in buffer_to_free: + assert self.remote_cache_engine is not None + self.remote_cache_engine.recycle(buffer_to_free[DeviceType.REMOTE]) + + @_synchronized_cache_tree def _abort_transfer_plan(self, node_to_unlock: Dict[DeviceType, Tuple[RadixNode, int]], buffer_to_free: Optional[Dict[DeviceType, np.ndarray]] = None, + deferred_inserts: Optional[List[DeferredCacheInsert]] = None, swa_reservation: Optional[SWAReadReservation] = None, swa_slots_to_free: Optional[List[Tuple[DeviceType, int]]] = None) -> None: """Roll back a planned get/put whose graph was never launched. @@ -2358,6 +2745,11 @@ def _abort_transfer_plan(self, for device_type, blocks in buffer_to_free.items(): if blocks is not None and len(blocks) > 0: self.cache_engines[device_type].recycle(blocks) + for pending in deferred_inserts or []: + engine = self.cache_engines[pending.device_type] + physical_blocks = np.asarray( + pending.physical_blocks, dtype=np.int64) + self._discard_deferred_insert(engine, pending, physical_blocks) if swa_reservation is not None: self._release_swa_read_reservation(swa_reservation) if swa_slots_to_free: @@ -2365,6 +2757,7 @@ def _abort_transfer_plan(self, if slot >= 0: self.cache_engines[device_type]._free_swa_slot(slot) + @_synchronized_cache_tree def _op_callback(self, device_type: DeviceType, node_to_ready: RadixNode, ready_length: int) -> None: if device_type == DeviceType.CPU: assert self.cpu_cache_engine is not None @@ -2578,6 +2971,7 @@ def _release_swa_read_reservation( _SWA_GPU_PLACEHOLDER = np.array([0], dtype=np.int64) + @_synchronized_cache_tree def _swa_release_load_lock(self, node, staging_slot: int = -1, engine=None) -> None: """SWA H2D completion callback: release the source pin and free any transient CPU staging slot. diff --git a/flexkv/cache/radixtree.py b/flexkv/cache/radixtree.py index 49bf565a56..69e3e16dfb 100644 --- a/flexkv/cache/radixtree.py +++ b/flexkv/cache/radixtree.py @@ -380,6 +380,12 @@ def match_prefix(self, last_ready_node = self.root_node prefix_blocks_num = 0 ready_prefix_blocks_num = 0 + # ``num_ready_matched_blocks`` is a prefix length, not a count of ready + # blocks anywhere on the matched path. A later child may become ready + # before an in-flight parent (for example after a concurrent branch + # split); once that happens, no descendant can extend the readable + # prefix until the parent is ready too. + ready_prefix_open = True last_node_matched_length = 0 physical_blocks = np.array([], dtype=np.int64) # SWA: deepest fully-matched ready node carrying a live SWA slot. @@ -399,7 +405,7 @@ def match_prefix(self, current_node.hit_count += 1 child_hash = sequence.get_hash(prefix_blocks_num + current_node.size()) if child_hash in current_node.children: - if current_node.is_ready: + if ready_prefix_open and current_node.is_ready: last_ready_node = current_node ready_prefix_blocks_num += current_node.size() # current_node is FULLY matched (whole size consumed): if it @@ -408,6 +414,8 @@ def match_prefix(self, if current_node.has_swa(): last_swa_node = current_node swa_hit_blocks = ready_prefix_blocks_num + elif current_node.size() > 0: + ready_prefix_open = False prefix_blocks_num += current_node.size() physical_blocks = np.concatenate([physical_blocks, current_node.physical_blocks]) current_node = current_node.children[child_hash] @@ -426,7 +434,7 @@ def match_prefix(self, physical_blocks = np.concatenate([physical_blocks, current_node.physical_blocks[:matched_length]]) else: matched_length = 0 - if current_node.is_ready: + if ready_prefix_open and current_node.is_ready: last_ready_node = current_node ready_prefix_blocks_num += matched_length # Only a FULL node match (matched_length == size) exposes the @@ -434,6 +442,8 @@ def match_prefix(self, if matched_length == current_node.size() and current_node.has_swa(): last_swa_node = current_node swa_hit_blocks = ready_prefix_blocks_num + elif matched_length > 0: + ready_prefix_open = False last_node_matched_length = matched_length prefix_blocks_num += matched_length break @@ -479,7 +489,6 @@ def insert(self, last_node = match_result.last_node assert last_node is not None last_node_matched_length = match_result.last_node_matched_length - assert last_node_matched_length != 0 or last_node.is_root() assert len(physical_block_ids) == num_insert_blocks - num_matched_blocks, \ f"num_insert_blocks = {num_insert_blocks}, " \ @@ -490,6 +499,13 @@ def insert(self, # not insert any new blocks return None + self._validate_insert_target( + sequence_meta, + last_node, + num_matched_blocks, + last_node_matched_length, + ) + now = time.time() new_node = RadixNode( block_hashes=sequence_meta.block_hashes[num_matched_blocks:num_insert_blocks], @@ -518,6 +534,60 @@ def insert(self, return new_node + def _validate_insert_target( + self, + sequence_meta: SequenceMeta, + last_node: RadixNode, + num_matched_blocks: int, + last_node_matched_length: int, + ) -> None: + """Reject a stale insertion anchor before mutating the tree. + + ``match_prefix`` results may outlive another insertion when this low-level + API is used directly. In that case assigning ``children[head_hash]`` + would silently orphan the existing subtree and its physical blocks. + Raising leaves both the radix tree and ``physical_block_ids`` untouched; + the caller still owns those blocks and may recycle them or rematch/retry. + """ + conflict = "radix insert conflict: stale match result; rematch before retrying" + node_size = last_node.size() + anchor_root = last_node + while anchor_root.parent is not None: + anchor_root = anchor_root.parent + if anchor_root is not self.root_node: + raise RuntimeError(conflict) + + if last_node.is_root(): + if last_node_matched_length != 0 or num_matched_blocks != 0: + raise RuntimeError(conflict) + else: + parent = last_node.parent + if (parent is None + or parent.children.get(last_node.head_hash()) is not last_node + or last_node_matched_length <= 0 + or last_node_matched_length > node_size): + raise RuntimeError(conflict) + + matched_start = num_matched_blocks - last_node_matched_length + if (matched_start < 0 + or not np.array_equal( + last_node.block_hashes[:last_node_matched_length], + sequence_meta.block_hashes[ + matched_start:num_matched_blocks])): + raise RuntimeError(conflict) + + target_hash = sequence_meta.get_hash(num_matched_blocks) + assert target_hash is not None + if last_node_matched_length < node_size: + # A valid partial match diverges at the split point. Equality means + # that the supplied match result no longer describes this node. + if last_node.block_hashes[last_node_matched_length] == target_hash: + raise RuntimeError(conflict) + elif target_hash in last_node.children: + raise RuntimeError( + "radix insert conflict: target child already exists; " + "rematch before retrying") + def evict(self, num_evicted: int) -> Tuple[np.ndarray, np.ndarray]: candidates = [] for node in self.leaf_nodes.values(): diff --git a/flexkv/common/transfer.py b/flexkv/common/transfer.py index c2ae719c33..fd20a634cb 100644 --- a/flexkv/common/transfer.py +++ b/flexkv/common/transfer.py @@ -1,5 +1,5 @@ import threading -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from enum import Enum, IntEnum from typing import ClassVar, List, Set, Dict, Callable, Tuple, Optional @@ -24,6 +24,9 @@ class CompletedOp: transfer_type: Optional[str] = None num_blocks: int = 0 num_bytes: int = 0 + # Per-block completion status for backends that can partially succeed. + # ``None`` preserves the all-or-nothing contract of existing workers. + block_results: Optional[Tuple[bool, ...]] = None # True on the graph-level message that reports the graph terminated because # one of its ops failed. Defaults False so the message stays pickle- and # constructor-compatible with existing producers/consumers. @@ -35,6 +38,21 @@ def is_graph_completed(self) -> bool: def is_graph_failed(self) -> bool: return self.op_id == -1 and self.failed + def slice_blocks(self, offset: int, count: int) -> 'CompletedOp': + """Return this completion restricted to one pre-merge op span.""" + if self.block_results is None: + return self + end = offset + count + if offset < 0 or count < 0 or end > len(self.block_results): + raise ValueError( + f"Invalid completion slice [{offset}:{end}] for " + f"{len(self.block_results)} block results") + return replace( + self, + num_blocks=count, + block_results=self.block_results[offset:end], + ) + def to_tuple(self) -> Tuple[int, int]: return (self.graph_id, self.op_id) @@ -51,6 +69,25 @@ def failed_graph(cls, graph_id: int) -> 'CompletedOp': return cls(graph_id=graph_id, op_id=-1, failed=True) +@dataclass(frozen=True) +class CompletionAwareCallback: + """Opt-in wrapper for callbacks that consume a ``CompletedOp`` result.""" + + callback: Callable[[Optional[CompletedOp]], None] + + def __call__(self, completed_op: Optional[CompletedOp] = None) -> None: + self.callback(completed_op) + + +def invoke_op_callback(callback: Callable, + completed_op: Optional[CompletedOp] = None) -> None: + """Invoke old zero-argument callbacks and result-aware callbacks uniformly.""" + if isinstance(callback, CompletionAwareCallback): + callback(completed_op) + else: + callback() + + class DeviceType(IntEnum): CPU = 0 GPU = 1 @@ -129,6 +166,8 @@ class TransferOp: mooncake_store_block_hashes: Optional[np.ndarray] = None # Tail-hash list for SWA mooncake REMOTE2H/H2REMOTE (one entry per SWA slot). mooncake_store_swa_block_hashes: Optional[List[str]] = None + # Filled by the scheduler as partial-capable worker completions arrive. + block_results: Optional[Tuple[bool, ...]] = field(default=None, init=False) def __post_init__(self) -> None: if self.transfer_type != TransferType.VIRTUAL and \ @@ -497,11 +536,23 @@ def format_blocks(block_ids, max_show=4): print(result) return result -def _make_combined_callback(callbacks: List[Callable]) -> Callable: - def combined_callback(*args, **kwargs): - for cb in callbacks: - cb(*args, **kwargs) - return combined_callback +def _make_combined_callback( + callbacks: List[Callable], + block_spans: Optional[List[Tuple[int, int]]] = None, +) -> Callable: + """Combine callbacks, slicing merged per-block results when requested.""" + if block_spans is not None and len(block_spans) != len(callbacks): + raise ValueError("callbacks and block_spans must have the same length") + + def combined_callback(completed_op: Optional[CompletedOp]) -> None: + for idx, cb in enumerate(callbacks): + child_completion = completed_op + if completed_op is not None and block_spans is not None: + offset, count = block_spans[idx] + child_completion = completed_op.slice_blocks(offset, count) + invoke_op_callback(cb, child_completion) + + return CompletionAwareCallback(combined_callback) def _attach_combined_callback(op: TransferOp, @@ -515,6 +566,34 @@ def _attach_combined_callback(op: TransferOp, op_callback_dict[op.op_id] = _make_combined_callback(callbacks) +def _attach_merged_callbacks( + merged_op: TransferOp, + source_ops: List[TransferOp], + callbacks: List[Tuple[TransferOp, Callable]], + op_callback_dict: Dict[int, Callable], +) -> None: + """Attach callbacks with their per-source spans in the merged op.""" + if not callbacks: + return + + callback_by_op_id = {op.op_id: callback for op, callback in callbacks} + merged_callbacks: List[Callable] = [] + block_spans: List[Tuple[int, int]] = [] + offset = 0 + for op in source_ops: + callback = callback_by_op_id.get(op.op_id) + if callback is not None: + merged_callbacks.append(callback) + block_spans.append((offset, len(op.src_block_ids))) + offset += len(op.src_block_ids) + + if len(source_ops) == 1: + op_callback_dict[merged_op.op_id] = merged_callbacks[0] + else: + op_callback_dict[merged_op.op_id] = _make_combined_callback( + merged_callbacks, block_spans) + + def add_virtual_op_for_multiple_finished_ops( graph: TransferOpGraph, finished_ops_ids: List[int], @@ -540,7 +619,8 @@ def add_virtual_op_for_multiple_finished_ops( def _merge_ops(ops: List[TransferOp], transfer_type: TransferType, - graph: TransferOpGraph, callbacks: List[Callable], + graph: TransferOpGraph, + callbacks: List[Tuple[TransferOp, Callable]], op_callback_dict: Dict[int, Callable]) -> Optional[TransferOp]: """Merge main-KV ops. Concatenates block ids and ``mooncake_store_block_hashes`` (H2REMOTE / REMOTE2H) in the same order. Rejects SWA ops (use ``_merge_swa_ops``) @@ -580,16 +660,14 @@ def _merge_ops(ops: List[TransferOp], transfer_type: TransferType, dp_client_id=ops[0].dp_client_id, mooncake_store_block_hashes=merged_kv_hashes, ) - if callbacks: - if len(callbacks) == 1: - op_callback_dict[merged_op.op_id] = callbacks[0] - else: - op_callback_dict[merged_op.op_id] = _make_combined_callback(callbacks) + _attach_merged_callbacks( + merged_op, ops, callbacks, op_callback_dict) return merged_op def _merge_swa_ops(ops: List[TransferOp], transfer_type: TransferType, - graph: TransferOpGraph, callbacks: List[Callable], + graph: TransferOpGraph, + callbacks: List[Tuple[TransferOp, Callable]], op_callback_dict: Dict[int, Callable]) -> Optional[TransferOp]: """Merge SWA-lane ops. Local lanes carry no mooncake hash. Remote lanes (H2REMOTE / REMOTE2H) concatenate ``mooncake_store_swa_block_hashes``. @@ -634,11 +712,8 @@ def _merge_swa_ops(ops: List[TransferOp], transfer_type: TransferType, is_swa=True, mooncake_store_swa_block_hashes=merged_swa_hashes, ) - if callbacks: - if len(callbacks) == 1: - op_callback_dict[merged_op.op_id] = callbacks[0] - else: - op_callback_dict[merged_op.op_id] = _make_combined_callback(callbacks) + _attach_merged_callbacks( + merged_op, ops, callbacks, op_callback_dict) return merged_op @@ -705,11 +780,6 @@ def merge_to_batch_graph(batch_id: int, task_end_op_ids: List of end op IDs for each task (one per graph) op_callback_dict: Dict mapping old op_id -> callback layerwise_transfer: Whether to merge the graphs into a layerwise transfer op - fuse_swa_into_layerwise: When True (default / production), SWA/state - H2D block ids are carried on the fused LAYERWISE op (uniform SWA - via launch_swa_h2d_layer_, multi-group via launch_swa_mg_h2d_layer_). - When False, SWA/state ops stay as standalone predecessors - (legacy heterogeneous fallback for non-fused debugging). Returns: (merged_graph, batch_end_op_id, new_op_callback_dict) @@ -730,9 +800,9 @@ def merge_to_batch_graph(batch_id: int, merged_graph = TransferOpGraph() ops_by_type: Dict[TransferType, List[TransferOp]] = {} - callbacks_by_type: Dict[TransferType, List[Callable]] = {} + callbacks_by_type: Dict[TransferType, List[Tuple[TransferOp, Callable]]] = {} swa_ops_by_type: Dict[TransferType, List[TransferOp]] = {} - swa_callbacks_by_type: Dict[TransferType, List[Callable]] = {} + swa_callbacks_by_type: Dict[TransferType, List[Tuple[TransferOp, Callable]]] = {} supported_types = {TransferType.DISK2H, TransferType.H2D, TransferType.D2H, TransferType.H2DISK, TransferType.H2REMOTE, TransferType.REMOTE2H} @@ -756,12 +826,12 @@ def merge_to_batch_graph(batch_id: int, swa_ops_by_type[op.transfer_type].append(op) if op.op_id in op_callback_dict: swa_callbacks_by_type[op.transfer_type].append( - op_callback_dict[op.op_id]) + (op, op_callback_dict[op.op_id])) else: ops_by_type[op.transfer_type].append(op) if op.op_id in op_callback_dict: callbacks_by_type[op.transfer_type].append( - op_callback_dict[op.op_id]) + (op, op_callback_dict[op.op_id])) new_op_callback_dict: Dict[int, Callable] = {} # Layerwise folds local DISK2H/H2D into LAYERWISE: merge those ops with a @@ -779,9 +849,8 @@ def merge_to_batch_graph(batch_id: int, if layerwise_transfer: assert has_get, "layerwise batch must contain GET ops" assert not has_put, "layerwise batch must not mix PUT ops" - else: - assert not (has_get and has_put), \ - "batch merge does not support mixed GET/PUT ops in the same batch" + elif has_get and has_put: + raise ValueError("batch merge cannot mix GET and PUT ops") dp_client_id = _pick_dp_client_id( ops_by_type=ops_by_type, swa_ops_by_type=swa_ops_by_type) @@ -856,10 +925,14 @@ def merge_to_batch_graph(batch_id: int, layerwise_transfer_op.op_id, merged_swa_remote2h_op.op_id) layerwise_callbacks: List[Callable] = [] - layerwise_callbacks.extend(callbacks_by_type[TransferType.DISK2H]) - layerwise_callbacks.extend(callbacks_by_type[TransferType.H2D]) - layerwise_callbacks.extend(swa_callbacks_by_type[TransferType.DISK2H]) - layerwise_callbacks.extend(swa_callbacks_by_type[TransferType.H2D]) + layerwise_callbacks.extend( + callback for _, callback in callbacks_by_type[TransferType.DISK2H]) + layerwise_callbacks.extend( + callback for _, callback in callbacks_by_type[TransferType.H2D]) + layerwise_callbacks.extend( + callback for _, callback in swa_callbacks_by_type[TransferType.DISK2H]) + layerwise_callbacks.extend( + callback for _, callback in swa_callbacks_by_type[TransferType.H2D]) _attach_combined_callback( layerwise_transfer_op, layerwise_callbacks, new_op_callback_dict) batch_end_op_id = layerwise_transfer_op.op_id diff --git a/flexkv/kvtask.py b/flexkv/kvtask.py index c0eb1fd1d7..1e3e241131 100644 --- a/flexkv/kvtask.py +++ b/flexkv/kvtask.py @@ -3,7 +3,7 @@ from typing import Dict, Optional, List, Union, Tuple import threading from enum import Enum -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from typing import Callable import multiprocessing as mp import copy @@ -14,7 +14,14 @@ from flexkv.common.config import CacheConfig, ModelConfig, GLOBAL_CONFIG_FROM_ENV from flexkv.common.debug import flexkv_logger from flexkv.common.block import hash_token -from flexkv.common.transfer import TransferOpGraph, merge_to_batch_graph, get_nvtx_default_color, CompletedOp +from flexkv.common.transfer import ( + CompletedOp, + TransferOpGraph, + TransferType, + get_nvtx_default_color, + invoke_op_callback, + merge_to_batch_graph, +) from flexkv.common.tracer import FlexKVTracer from flexkv.cache.cache_engine import ( GlobalCacheEngine, @@ -69,6 +76,7 @@ class KVTask: return_mask: Union[np.ndarray, list[np.ndarray]] callback: Optional[Union[Callable, List[Callable]]] op_callback_dict: Dict[int, Callable] + transfer_failed: bool = False # SWA GPU slot_mapping (SWA-pool token index space), bound LATE at launch — # the SWA counterpart to slot_mapping. None when the request has no SWA ops. @@ -187,6 +195,7 @@ def __init__(self, # (graph_id, op_id) -> completed_count; graph-keyed so a failed # graph's stale per-op counters can be purged. self.uncompleted_ops: Dict[Tuple[int, int], int] = {} + self.uncompleted_op_results: Dict[Tuple[int, int], CompletedOp] = {} # graph_id -> (terminal_count, any_failed) across the N handles. self.uncompleted_graphs: Dict[int, Tuple[int, bool]] = {} self.required_completed_count: int = len(self.transfer_handles) @@ -415,6 +424,29 @@ def _update_tasks(self, timeout: float = 0.001) -> None: if completed_op.is_graph_failed(): self._fail_task(task_id) continue + # A failed pull invalidates the current request and must trigger + # fallback. Mooncake uploads retain the existing asynchronous PUT + # completion contract (the task-end D2H may precede H2REMOTE). + graph_op = getattr(task.graph, "_op_map", {}).get( + completed_op.op_id) + is_remote_load = ( + completed_op.transfer_type == TransferType.REMOTE2H.value + or (graph_op is not None + and graph_op.transfer_type == TransferType.REMOTE2H) + ) + expects_block_results = ( + graph_op is not None + and (graph_op.mooncake_store_block_hashes is not None + or graph_op.mooncake_store_swa_block_hashes is not None) + ) + missing_results = ( + completed_op.block_results is None and expects_block_results) + failed_blocks = ( + completed_op.block_results is not None + and not all(completed_op.block_results) + ) + if is_remote_load and (missing_results or failed_blocks): + task.transfer_failed = True # Record transfer metrics for completed ops (post-completion statistics) # All three counters (ops_total, blocks_total, bytes_total) are updated # here after transfer completion, providing accurate post-transfer metrics. @@ -437,7 +469,17 @@ def _update_tasks(self, timeout: float = 0.001) -> None: self.tasks[task_id].task_end_op_finished = True has_callback = completed_op.op_id in task.op_callback_dict if has_callback: - task.op_callback_dict[completed_op.op_id]() + try: + invoke_op_callback( + task.op_callback_dict[completed_op.op_id], completed_op) + except Exception: + task.transfer_failed = True + flexkv_logger.error( + "Transfer op callback failed: " + f"graph_id={completed_op.graph_id}, " + f"op_id={completed_op.op_id}", + exc_info=True, + ) @staticmethod def _abort_task_plans(task: "KVTask") -> None: @@ -499,6 +541,11 @@ def check_completed(self, task_id: int, completely: bool = False) -> bool: self._process_empty_graph(task_id) if completely: return task.is_completed() + # A partial-capable backend may finish the data-path sink after already + # reporting failed blocks. Wait for graph completion so cleanup runs and + # the caller observes FAILED instead of an early RUNNING-as-success result. + if task.transfer_failed: + return task.is_completed() # For tasks with callback (e.g., PUT tasks that need to call insert_and_publish), # we must wait until _mark_completed is called (i.e., is_completed() returns True) # to ensure the callback is executed before returning success. @@ -573,12 +620,21 @@ def _mark_completed(self, task_id: int) -> None: if task.is_completed(): return if task.callback: - if isinstance(task.callback, list): - for callback in task.callback: + callbacks = ( + task.callback if isinstance(task.callback, list) + else [task.callback] + ) + for callback in callbacks: + try: callback() - else: - task.callback() - task.status = TaskStatus.COMPLETED + except Exception: + task.transfer_failed = True + flexkv_logger.error( + f"Transfer graph callback failed for task_id={task_id}", + exc_info=True, + ) + task.status = ( + TaskStatus.FAILED if task.transfer_failed else TaskStatus.COMPLETED) task.task_end_op_finished = True self._log_task_terminal(task, TaskStatus.COMPLETED) self.graph_to_task.pop(task.graph.graph_id, None) @@ -595,6 +651,10 @@ def _process_empty_graph(self, task_id: int) -> None: def _get_completed_ops(self, timeout: Optional[float] = None) -> List[CompletedOp]: results = [] + # Keep lightweight test/fallback managers created with ``__new__`` + # compatible with the pre-bitmap state shape. + if not hasattr(self, "uncompleted_op_results"): + self.uncompleted_op_results = {} for transfer_handle in self.transfer_handles: completed_ops = transfer_handle.wait(timeout) for completed_op in completed_ops: @@ -623,6 +683,7 @@ def _get_completed_ops(self, timeout: Optional[float] = None) -> List[CompletedO if key[0] == graph_id] for key in stale: self.uncompleted_ops.pop(key, None) + self.uncompleted_op_results.pop(key, None) results.append(CompletedOp.failed_graph(graph_id)) else: results.append(completed_op) @@ -631,13 +692,72 @@ def _get_completed_ops(self, timeout: Optional[float] = None) -> List[CompletedO else: op_key = (completed_op.graph_id, completed_op.op_id) completed_count = self.uncompleted_ops.get(op_key, 0) + 1 + aggregate = self._merge_completed_op( + self.uncompleted_op_results.get(op_key), + completed_op, + ) if completed_count == self.required_completed_count: - results.append(completed_op) + results.append(aggregate) self.uncompleted_ops.pop(op_key, None) + self.uncompleted_op_results.pop(op_key, None) else: self.uncompleted_ops[op_key] = completed_count + self.uncompleted_op_results[op_key] = aggregate return results + @staticmethod + def _merge_completed_op( + current: Optional[CompletedOp], + incoming: CompletedOp, + ) -> CompletedOp: + """Combine multi-handle outcomes; a block succeeds only everywhere.""" + if current is None: + if incoming.block_results is None: + return incoming + expected_blocks = incoming.num_blocks or len(incoming.block_results) + block_results = ( + tuple(bool(result) for result in incoming.block_results) + if len(incoming.block_results) == expected_blocks + else (False,) * expected_blocks + ) + return replace( + incoming, + num_blocks=expected_blocks, + block_results=block_results, + ) + + block_results = None + if (current.block_results is not None + or incoming.block_results is not None): + # Once one handle reports a bitmap, every handle must report the + # same width. Missing or malformed data is a fail-closed result. + expected_blocks = max( + current.num_blocks, + incoming.num_blocks, + len(current.block_results or ()), + len(incoming.block_results or ()), + ) + + def normalize(results: Optional[Tuple[bool, ...]]) -> Tuple[bool, ...]: + if results is None or len(results) != expected_blocks: + return (False,) * expected_blocks + return tuple(bool(result) for result in results) + + left = normalize(current.block_results) + right = normalize(incoming.block_results) + block_results = tuple(a and b for a, b in zip(left, right)) + return replace( + current, + transfer_type=current.transfer_type or incoming.transfer_type, + num_blocks=max( + current.num_blocks, + incoming.num_blocks, + len(block_results or ()), + ), + num_bytes=max(current.num_bytes, incoming.num_bytes), + block_results=block_results, + ) + class KVTaskEngine(KVTaskManager): def __init__(self, model_config: ModelConfig, diff --git a/flexkv/transfer/transfer_engine.py b/flexkv/transfer/transfer_engine.py index 2677020b00..168684113c 100644 --- a/flexkv/transfer/transfer_engine.py +++ b/flexkv/transfer/transfer_engine.py @@ -48,6 +48,7 @@ LayerwiseTransferWorker, build_layerwise_eventfd_socket_path, ) +from flexkv.transfer.worker_op import WorkerTransferResult from flexkv.common.config import ( CacheConfig, LayerGroupSpec, ModelConfig, GLOBAL_CONFIG_FROM_ENV, ) @@ -1165,7 +1166,13 @@ def _scheduler_loop(self) -> None: # Get all available ops in one go to reduce system calls while True: try: - op_id = self.finished_ops_queue.get_nowait() + worker_result = self.finished_ops_queue.get_nowait() + if isinstance(worker_result, WorkerTransferResult): + op_id = worker_result.transfer_op_id + block_results = worker_result.block_results + else: + op_id = worker_result + block_results = None # A bare int means success; a (op_id, False) # tuple is a worker-reported transfer failure. op_succeeded = True @@ -1179,10 +1186,13 @@ def _scheduler_loop(self) -> None: # pending_count and finalize parent when all replicas done. parent_op_id = self._child_to_parent_op_id.pop(op_id) child_op = self._child_id_to_child.pop(op_id) + self._merge_block_results(child_op, block_results) free_op_from_buffer(child_op, self.pin_buffer) if op_id in self.op_id_to_nvtx_range: nvtx.end_range(self.op_id_to_nvtx_range.pop(op_id)) parent_op = self.op_id_to_op[parent_op_id] + self._merge_block_results( + parent_op, child_op.block_results) parent_op.pending_count -= 1 if parent_op.pending_count == 0: self._finalize_or_discard(parent_op, finished_ops) @@ -1191,6 +1201,7 @@ def _scheduler_loop(self) -> None: f"parent op {parent_op_id} pending_count={parent_op.pending_count}") else: op = self.op_id_to_op[op_id] + self._merge_block_results(op, block_results) op.pending_count -= 1 if op.pending_count == 0: self._finalize_or_discard(op, finished_ops) @@ -1362,10 +1373,31 @@ def _finalize_op(self, op: TransferOp, finished_ops: List[TransferOp]) -> None: transfer_type=transfer_type_str, num_blocks=num_blocks, num_bytes=num_bytes, + block_results=op.block_results, )) finished_ops.append(op) del self.op_id_to_op[op.op_id] + @staticmethod + def _merge_block_results( + op: TransferOp, + block_results: Optional[Tuple[bool, ...]], + ) -> None: + """Accumulate per-worker outcomes; every participating worker must win.""" + if block_results is None: + return + normalized = tuple(bool(result) for result in block_results) + if len(normalized) != len(op.src_block_ids): + flexkv_logger.error( + f"Completion result length mismatch for op {op.op_id}: " + f"results={len(normalized)}, blocks={len(op.src_block_ids)}") + normalized = (False,) * len(op.src_block_ids) + if op.block_results is None: + op.block_results = normalized + else: + op.block_results = tuple( + old and new for old, new in zip(op.block_results, normalized)) + @staticmethod def _match_pp_siblings( worker_map: Dict[WorkerKey, WorkerHandle], diff --git a/flexkv/transfer/worker.py b/flexkv/transfer/worker.py index 9509b8236e..e81b219f57 100644 --- a/flexkv/transfer/worker.py +++ b/flexkv/transfer/worker.py @@ -78,7 +78,11 @@ def import_tensor_handles( CompressionStrategy, NullCompressionStrategy, ) -from flexkv.transfer.worker_op import WorkerTransferOp, WorkerLayerwiseTransferOp +from flexkv.transfer.worker_op import ( + WorkerLayerwiseTransferOp, + WorkerTransferOp, + WorkerTransferResult, +) from flexkv.mooncakeEngineWrapper import MoonCakeTransferEngineWrapper from flexkv.external.mooncake_store_keys import PoolKind, build_key @@ -131,7 +135,7 @@ def __init__(self, op_buffer_tensor: torch.Tensor): self.worker_id = worker_id self.transfer_conn = transfer_conn # receive end of pipe - self.finished_ops_queue: MPQueue[int] = finished_ops_queue + self.finished_ops_queue: MPQueue = finished_ops_queue self.op_buffer_tensor = op_buffer_tensor self._op_buffer_pinned = False @@ -472,7 +476,9 @@ def _log_transfer_performance(self, ) @abstractmethod - def launch_transfer(self, transfer_op: WorkerTransferOp) -> bool: + def launch_transfer( + self, transfer_op: WorkerTransferOp + ) -> Union[bool, WorkerTransferResult]: pass def run(self) -> None: @@ -534,7 +540,12 @@ def run(self) -> None: finally: if nvtx_pushed: nvtx.pop_range() - if transfer_status: + if isinstance(transfer_status, WorkerTransferResult): + # Partial-capable backends report completion even when + # zero blocks succeeded, so the graph can clean up and + # the caller can fall back instead of hanging forever. + self.finished_ops_queue.put(transfer_status) + elif transfer_status: self.finished_ops_queue.put(op.transfer_op_id) else: # Report the failure instead of dropping it: a @@ -3724,30 +3735,72 @@ def _block_size_bytes(cpu_kv_layout: "KVCacheLayout", dtype: torch.dtype) -> int return int(cpu_kv_layout.get_block_stride()) return int(cpu_kv_layout.get_elements_per_block() * dtype.itemsize) - def _transfer_impl(self, cpu_ptrs, block_sizes, keys, transfer_type: TransferType) -> None: + def _transfer_impl(self, cpu_ptrs, block_sizes, keys, + transfer_type: TransferType) -> List[bool]: if transfer_type == TransferType.H2REMOTE: put_results = self.mooncake_client.batch_put(keys, cpu_ptrs, block_sizes) if not all(put_results): flexkv_logger.error(f"Mooncake-store batch put partially failed: {put_results}") + return put_results elif transfer_type == TransferType.REMOTE2H: get_results = self.mooncake_client.batch_get(keys, cpu_ptrs, block_sizes) if not all(get_results): flexkv_logger.error(f"Mooncake-store batch get partially failed: {get_results}") + return get_results else: raise ValueError( f"MooncakeStoreTransferWorker only supports H2REMOTE/REMOTE2H, got {transfer_type}") - def launch_transfer(self, transfer_op: WorkerTransferOp) -> bool: - if self.pool_kind == PoolKind.SWA: - cpu_ptrs, block_sizes, keys = self._preprocess_swa(transfer_op) - else: - cpu_ptrs, block_sizes, keys = self._preprocess_kv(transfer_op) + def launch_transfer(self, transfer_op: WorkerTransferOp) -> WorkerTransferResult: + expected_blocks = len(transfer_op.src_block_ids) + block_sizes: List[int] = [] start_time = time.time() - self._transfer_impl(cpu_ptrs, block_sizes, keys, transfer_op.transfer_type) + try: + if self.pool_kind == PoolKind.SWA: + cpu_ptrs, block_sizes, keys = self._preprocess_swa(transfer_op) + else: + cpu_ptrs, block_sizes, keys = self._preprocess_kv(transfer_op) + if len(keys) != expected_blocks: + raise ValueError( + "Mooncake key count does not match transfer block count: " + f"keys={len(keys)}, blocks={expected_blocks}") + block_results = self._transfer_impl( + cpu_ptrs, block_sizes, keys, transfer_op.transfer_type) + if len(block_results) != expected_blocks: + raise ValueError( + "Mooncake result count does not match transfer block count: " + f"results={len(block_results)}, blocks={expected_blocks}") + normalized_results = tuple(bool(result) for result in block_results) + except Exception: + # A completed failure must still reach the scheduler. Otherwise the + # graph never completes and its reserved cache blocks remain leaked. + flexkv_logger.error( + "Mooncake transfer failed; reporting all blocks unsuccessful " + f"for op_id={transfer_op.transfer_op_id}", + exc_info=True, + ) + normalized_results = (False,) * expected_blocks end_time = time.time() - transfer_size = sum(block_sizes) - self._log_transfer_performance(transfer_op, transfer_size, start_time, end_time) - return True + try: + transfer_size = sum(block_sizes) + self._log_transfer_performance( + transfer_op, transfer_size, start_time, end_time) + except Exception: + flexkv_logger.error( + "Mooncake transfer performance logging failed; reporting the " + f"operation unsuccessful for op_id={transfer_op.transfer_op_id}", + exc_info=True, + ) + normalized_results = (False,) * expected_blocks + if not all(normalized_results): + flexkv_logger.warning( + "Mooncake transfer partially failed: " + f"op_id={transfer_op.transfer_op_id}, " + f"successful={sum(normalized_results)}/{len(normalized_results)}") + return WorkerTransferResult( + transfer_op_id=transfer_op.transfer_op_id, + block_results=normalized_results, + ) def _preprocess_kv(self, transfer_op: WorkerTransferOp): cpu_block_ids = ( diff --git a/flexkv/transfer/worker_op.py b/flexkv/transfer/worker_op.py index 8699ca0e48..14f8e090e9 100644 --- a/flexkv/transfer/worker_op.py +++ b/flexkv/transfer/worker_op.py @@ -1,11 +1,19 @@ from dataclasses import dataclass -from typing import Optional +from typing import Optional, Tuple import numpy as np from flexkv.common.transfer import TransferOp, TransferType, LayerwiseTransferOp +@dataclass(frozen=True) +class WorkerTransferResult: + """Worker-to-scheduler completion with optional per-block outcomes.""" + + transfer_op_id: int + block_results: Optional[Tuple[bool, ...]] = None + + @dataclass class WorkerTransferOp: transfer_op_id: int diff --git a/flexkv/transfer_manager.py b/flexkv/transfer_manager.py index 75e938d658..f51709bb00 100644 --- a/flexkv/transfer_manager.py +++ b/flexkv/transfer_manager.py @@ -477,9 +477,12 @@ def _polling_worker(self) -> None: if completed_op.graph_id in self._active_graphs: task_end_op_id = self._active_graphs[completed_op.graph_id] - if task_end_op_id != -1 and completed_op.op_id == task_end_op_id: - end_op = CompletedOp(graph_id=completed_op.graph_id, op_id=task_end_op_id) - self.result_socket.send_pyobj(end_op) + if (completed_op.block_results is not None + or (task_end_op_id != -1 + and completed_op.op_id == task_end_op_id)): + # Preserve backend-specific completion details + # (notably Mooncake per-block results). + self.result_socket.send_pyobj(completed_op) if (completed_op.is_graph_completed() or completed_op.is_graph_failed()): self.result_socket.send_pyobj(completed_op) diff --git a/tests/test_cache_engine.py b/tests/test_cache_engine.py index e8b6cf5d01..75d380766e 100644 --- a/tests/test_cache_engine.py +++ b/tests/test_cache_engine.py @@ -267,6 +267,150 @@ def test_match_and_insert(cache_engine: CacheEngineType, num_insert: int, seq_le assert match_result.num_ready_matched_blocks == insert_sequence_meta.num_blocks +@pytest.mark.parametrize( + "cache_engine", + [{'num_total_blocks': 16, 'tokens_per_block': 1}], + indirect=True, +) +def test_ready_match_is_a_contiguous_prefix(cache_engine: CacheEngineType): + """A ready child below an unready split parent is not a readable prefix.""" + first = SequenceMeta( + token_ids=np.array([1, 2, 3, 4], dtype=np.int64), + tokens_per_block=1, + ) + branch = SequenceMeta( + token_ids=np.array([1, 2, 30, 40], dtype=np.int64), + tokens_per_block=1, + ) + + first_node = cache_engine.insert( + first, cache_engine.take(4), is_ready=False) + branch_match = cache_engine.match(branch) + assert branch_match.num_matched_blocks == 2 + branch_node = cache_engine.insert( + branch, + cache_engine.take(2), + is_ready=False, + match_result=branch_match, + ) + + # This is the dangerous completion order: the new suffix finishes before + # the writer that owns the shared prefix. + cache_engine.set_ready(branch_node, True, 2) + pending_prefix = cache_engine.match(branch) + assert pending_prefix.num_matched_blocks == 4 + assert pending_prefix.num_ready_matched_blocks == 0 + + # The original node pointer is now the split suffix. The saved insertion + # length walks through its parent and makes the whole path readable. + cache_engine.set_ready(first_node, True, 4) + completed = cache_engine.match(branch) + assert completed.num_matched_blocks == 4 + assert completed.num_ready_matched_blocks == 4 + + +@pytest.mark.parametrize( + ("second_tokens", "expected_second_match"), + [ + ([1, 2], 2), # identical stale insert + ([1, 30], 1), # stale insert that should rematch and split + ], +) +@pytest.mark.parametrize( + "cache_engine", + [{'num_total_blocks': 16, 'tokens_per_block': 1}], + indirect=True, +) +def test_stale_insert_rejects_existing_child_without_taking_block_ownership( + cache_engine: CacheEngineType, + second_tokens, + expected_second_match, +): + """A stale match cannot overwrite a child inserted by another caller.""" + first = SequenceMeta( + token_ids=np.array([1, 2], dtype=np.int64), + tokens_per_block=1, + ) + second = SequenceMeta( + token_ids=np.array(second_tokens, dtype=np.int64), + tokens_per_block=1, + ) + first_match = cache_engine.match(first) + stale_second_match = cache_engine.match(second) + first_blocks = cache_engine.take(2) + second_blocks = cache_engine.take(2) + second_blocks_before = second_blocks.copy() + + cache_engine.insert( + first, first_blocks, is_ready=True, match_result=first_match) + with pytest.raises(RuntimeError, match="radix insert conflict"): + cache_engine.insert( + second, + second_blocks, + is_ready=True, + match_result=stale_second_match, + ) + + # Only the first insertion is reachable. The rejected blocks are still + # allocated and unchanged, so the caller can safely recycle or retry them. + assert cache_engine.index.total_cached_blocks() == 2 + assert cache_engine.mempool.num_used_blocks == 4 + np.testing.assert_array_equal(second_blocks, second_blocks_before) + assert cache_engine.match(first).num_matched_blocks == 2 + assert cache_engine.match(second).num_matched_blocks == expected_second_match + + cache_engine.recycle(second_blocks) + assert cache_engine.mempool.num_used_blocks == 2 + + +@pytest.mark.parametrize( + "cache_engine", + [{'num_total_blocks': 16, 'tokens_per_block': 1}], + indirect=True, +) +def test_stale_partial_match_is_rejected_after_concurrent_split( + cache_engine: CacheEngineType, +): + cached = SequenceMeta( + token_ids=np.array([1, 2, 3, 4], dtype=np.int64), + tokens_per_block=1, + ) + stale_sequence = SequenceMeta( + token_ids=np.array([1, 2, 30, 40], dtype=np.int64), + tokens_per_block=1, + ) + concurrent_sequence = SequenceMeta( + token_ids=np.array([1, 2, 30, 50], dtype=np.int64), + tokens_per_block=1, + ) + cache_engine.insert(cached, cache_engine.take(4), is_ready=True) + stale_match = cache_engine.match(stale_sequence) + concurrent_match = cache_engine.match(concurrent_sequence) + assert stale_match.num_matched_blocks == 2 + + cache_engine.insert( + concurrent_sequence, + cache_engine.take(2), + is_ready=True, + match_result=concurrent_match, + ) + rejected_blocks = cache_engine.take(2) + with pytest.raises(RuntimeError, match="radix insert conflict"): + cache_engine.insert( + stale_sequence, + rejected_blocks, + is_ready=True, + match_result=stale_match, + ) + + assert cache_engine.index.total_cached_blocks() == 6 + assert cache_engine.match(cached).num_matched_blocks == 4 + assert cache_engine.match(concurrent_sequence).num_matched_blocks == 4 + assert cache_engine.match(stale_sequence).num_matched_blocks == 3 + cache_engine.recycle(rejected_blocks) + assert cache_engine.mempool.num_used_blocks == 6 + + # --------------------------------------------------------------------------- # Tests – CacheEngine take & recycle # --------------------------------------------------------------------------- diff --git a/tests/test_kvtask_lifecycle.py b/tests/test_kvtask_lifecycle.py index 25365c0ece..1c4ab0a50e 100644 --- a/tests/test_kvtask_lifecycle.py +++ b/tests/test_kvtask_lifecycle.py @@ -1,13 +1,15 @@ # SPDX-FileCopyrightText: Copyright (c) <2025> NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Unit tests: KVTask state machine + SWA load-lock lifecycle at the engine level. +"""Unit tests for KVTask lifecycle, transfer results, and SWA load locks. -Two concerns, both cheap (no TransferManager subprocess, no GPU): +The control-plane cases cover: 1. The ``KVTask`` dataclass state machine: status transitions, is_completed, early-response cleanup, ``swa_slot_mapping``, and shed_heavy_resources. -2. The SWA load-lock lifecycle on ``GlobalCacheEngine``: a GET pins the matched +2. Per-block transfer-result aggregation and partial-load failure handling. + +3. The SWA load-lock lifecycle on ``GlobalCacheEngine``: a GET pins the matched CPU SWA node while building the GET plan, and the SWA H2D completion callback releases it (``_swa_release_load_lock``), leaving the node cached (dec_swa_lock_ref, NOT dec_swa_lock_only). This documents that @@ -15,15 +17,28 @@ in get()/put(), both released via the op/transfer callbacks) — so a fresh GET after a completed GET re-locks cleanly with no residual pin. -Requires flexkv.c_ext for concern 2 (production CacheEngineAccel); concern 1 is -pure Python. +The SWA engine cases require ``flexkv.c_ext``; none starts a TransferManager +subprocess or requires a GPU. """ +from types import SimpleNamespace + import numpy as np import pytest import torch from flexkv.common.request import KVResponseStatus -from flexkv.kvtask import KVTask, KVTaskEngine, TaskType, TaskStatus +from flexkv.common.transfer import ( + CompletedOp, + CompletionAwareCallback, + TransferType, +) +from flexkv.kvtask import ( + KVTask, + KVTaskEngine, + KVTaskManager, + TaskStatus, + TaskType, +) pytestmark = pytest.mark.unit @@ -171,6 +186,129 @@ def test_completed_task_remains_observable_until_first_response(): assert task.task_id not in engine.tasks +class _CompletedOpsHandle: + def __init__(self, completed_ops): + self.completed_ops = completed_ops + + def wait(self, _timeout): + return self.completed_ops + + def shutdown(self): + pass + + +def test_multi_handle_block_results_are_combined_with_and(): + """A block is reusable only when every transfer handle succeeded.""" + graph_id, op_id = 41, 73 + manager = object.__new__(KVTaskManager) + manager.transfer_handles = [ + _CompletedOpsHandle([CompletedOp( + graph_id=graph_id, + op_id=op_id, + num_blocks=3, + block_results=(True, True, False), + )]), + _CompletedOpsHandle([CompletedOp( + graph_id=graph_id, + op_id=op_id, + num_blocks=3, + block_results=(True, False, True), + )]), + ] + manager.required_completed_count = 2 + manager.uncompleted_ops = {} + manager.uncompleted_op_results = {} + manager.uncompleted_graphs = {} + + assert manager._get_completed_ops(timeout=0) == [CompletedOp( + graph_id=graph_id, + op_id=op_id, + num_blocks=3, + block_results=(True, False, False), + )] + assert manager.uncompleted_ops == {} + assert manager.uncompleted_op_results == {} + + +@pytest.mark.parametrize("second_results", [None, (True, True)]) +def test_multi_handle_invalid_block_results_fail_closed(second_results): + graph_id, op_id = 43, 76 + manager = object.__new__(KVTaskManager) + manager.transfer_handles = [ + _CompletedOpsHandle([CompletedOp( + graph_id=graph_id, + op_id=op_id, + num_blocks=3, + block_results=(True, True, True), + )]), + _CompletedOpsHandle([CompletedOp( + graph_id=graph_id, + op_id=op_id, + num_blocks=3, + block_results=second_results, + )]), + ] + manager.required_completed_count = 2 + manager.uncompleted_ops = {} + manager.uncompleted_op_results = {} + manager.uncompleted_graphs = {} + + assert manager._get_completed_ops(timeout=0) == [CompletedOp( + graph_id=graph_id, + op_id=op_id, + num_blocks=3, + block_results=(False, False, False), + )] + + +def test_partial_remote2h_does_not_return_early_success(): + graph_id, op_id, task_end_op_id = 42, 74, 75 + completion = CompletedOp( + graph_id=graph_id, + op_id=op_id, + transfer_type=TransferType.REMOTE2H.value, + num_blocks=3, + block_results=(True, False, False), + ) + seen = [] + task = _make_task( + status=TaskStatus.RUNNING, + task_end_op_id=task_end_op_id, + graph=SimpleNamespace(graph_id=graph_id, num_ops=2, _op_map={}), + op_callback_dict={ + op_id: CompletionAwareCallback(lambda result: seen.append(result)) + }, + ) + manager = object.__new__(KVTaskManager) + manager.graph_to_task = {graph_id: task.task_id} + manager.tasks = {task.task_id: task} + manager._get_completed_ops = lambda _timeout: [completion] + + manager._update_tasks(timeout=0) + + assert task.transfer_failed is True + assert task.task_end_op_finished is False + assert seen == [completion] + assert manager.check_completed(task.task_id) is False + + manager._get_completed_ops = lambda _timeout: [CompletedOp( + graph_id=graph_id, + op_id=task_end_op_id, + transfer_type=TransferType.H2D.value, + )] + manager._update_tasks(timeout=0) + + assert task.task_end_op_finished is True + assert manager.check_completed(task.task_id) is False + + manager._get_completed_ops = lambda _timeout: [ + CompletedOp.completed_graph(graph_id) + ] + manager._update_tasks(timeout=0) + + assert task.status == TaskStatus.FAILED + + # --- M15/M16: SWA-aware get's core arithmetic (pure, mirrors kvtask logic) --- # def _usable_and_swa_mask(full_hit, swa_hit, num_tokens, tpb): diff --git a/tests/test_mooncake_batch_merge.py b/tests/test_mooncake_batch_merge.py index 09d92629dd..0e50c32486 100644 --- a/tests/test_mooncake_batch_merge.py +++ b/tests/test_mooncake_batch_merge.py @@ -19,6 +19,8 @@ import pytest from flexkv.common.transfer import ( + CompletedOp, + CompletionAwareCallback, TransferOp, TransferOpGraph, TransferType, @@ -224,6 +226,54 @@ def test_swa_put_d2h_gates_swa_h2remote(self): (m_h2r,) = _find_ops(merged, transfer_type=TransferType.H2REMOTE, is_swa=True) assert m_d2h.op_id in m_h2r.predecessors + def test_remote2h_completion_is_sliced_by_original_op_span(self): + """A merged Mooncake result must not be broadcast to every request. + + The middle failure belongs only to request A. Request B's later blocks + remain independently successful and must reach B's callback unchanged. + """ + op_a = _mk_op( + TransferType.REMOTE2H, + [0, 1], + [10, 11], + kv_hashes=["a0", "a1"], + ) + op_b = _mk_op( + TransferType.REMOTE2H, + [2, 3, 4], + [12, 13, 14], + kv_hashes=["b0", "b1", "b2"], + ) + seen = {} + + def capture(name): + return CompletionAwareCallback( + lambda completed: seen.__setitem__(name, completed.block_results) + ) + + merged, _, callbacks = merge_to_batch_graph( + 99, + [_one_op_graph(op_a), _one_op_graph(op_b)], + [-1, -1], + { + op_a.op_id: capture("a"), + op_b.op_id: capture("b"), + }, + ) + (merged_remote2h,) = _find_ops( + merged, transfer_type=TransferType.REMOTE2H) + callbacks[merged_remote2h.op_id](CompletedOp( + graph_id=merged.graph_id, + op_id=merged_remote2h.op_id, + num_blocks=5, + block_results=(True, False, True, True, True), + )) + + assert seen == { + "a": (True, False), + "b": (True, True, True), + } + class TestBatchEndOpIdVirtualSink: def test_get_main_only_no_virtual(self): diff --git a/tests/test_mooncake_deferred_insert.py b/tests/test_mooncake_deferred_insert.py new file mode 100644 index 0000000000..ad80e42ed9 --- /dev/null +++ b/tests/test_mooncake_deferred_insert.py @@ -0,0 +1,206 @@ +"""Focused tests for completion-time Mooncake radix publication.""" + +import threading +from types import SimpleNamespace +from unittest.mock import Mock + +import numpy as np +import pytest + + +pytest.importorskip("flexkv.c_ext") + +from flexkv.cache.cache_engine import ( + CacheEngine, + DeferredCacheInsert, + GlobalCacheEngine, + MooncakeLoadResult, +) +from flexkv.common.block import SequenceMeta +from flexkv.common.transfer import DeviceType + + +pytestmark = pytest.mark.unit + + +def _engine(): + return CacheEngine( + device_type=DeviceType.CPU, + num_total_blocks=32, + tokens_per_block=1, + evict_ratio=0.1, + ) + + +def _manager(engine): + manager = object.__new__(GlobalCacheEngine) + manager.cache_engines = {DeviceType.CPU: engine} + return manager + + +def _sequence(tokens): + return SequenceMeta( + token_ids=np.asarray(tokens, dtype=np.int64), + tokens_per_block=1, + ) + + +def _pending( + engine, + sequence, + block_results, + staged_start, + requested_end, + remote_start=None, +): + physical_blocks = engine.take(requested_end - staged_start) + return DeferredCacheInsert( + device_type=DeviceType.CPU, + sequence_meta=sequence, + physical_blocks=physical_blocks, + staged_start_block=staged_start, + remote_start_block=( + staged_start if remote_start is None else remote_start), + requested_end_block=requested_end, + load_result=MooncakeLoadResult(block_results=block_results), + ) + + +def test_deferred_insert_publishes_all_successful_blocks(): + engine = _engine() + sequence = _sequence([1, 2, 3, 4]) + pending = _pending( + engine, sequence, (True, True, True, True), 0, 4) + + _manager(engine)._commit_deferred_insert(pending) + + match = engine.match(sequence) + assert match.num_matched_blocks == 4 + assert match.num_ready_matched_blocks == 4 + assert engine.index.total_unready_blocks() == 0 + assert engine.mempool.num_used_blocks == 4 + + +def test_deferred_insert_stops_at_first_failed_block(): + engine = _engine() + sequence = _sequence([1, 2, 3]) + pending = _pending(engine, sequence, (True, False, True), 0, 3) + + _manager(engine)._commit_deferred_insert(pending) + + match = engine.match(sequence) + assert match.num_matched_blocks == 1 + assert match.num_ready_matched_blocks == 1 + assert engine.index.total_unready_blocks() == 0 + assert engine.mempool.num_used_blocks == 1 + + +@pytest.mark.parametrize(("block_results", "expected_ready"), [ + ((True, False), 3), + ((False, True), 2), +]) +def test_deferred_insert_keeps_local_staging_before_remote_failure( + block_results, + expected_ready, +): + """SSD-loaded staging before the remote span remains publishable.""" + engine = _engine() + sequence = _sequence([1, 2, 3, 4]) + first_block = engine.take(1) + engine.insert( + sequence, first_block, num_insert_blocks=1, is_ready=True) + pending = _pending( + engine, + sequence, + block_results, + staged_start=1, + remote_start=2, + requested_end=4, + ) + + _manager(engine)._commit_deferred_insert(pending) + + match = engine.match(sequence) + assert match.num_ready_matched_blocks == expected_ready + assert engine.index.total_unready_blocks() == 0 + assert engine.mempool.num_used_blocks == expected_ready + + +def test_deferred_insert_rematches_after_concurrent_save_split(): + """A save may split and extend the original match before GET completes.""" + engine = _engine() + manager = _manager(engine) + cached = _sequence([1, 2, 3, 4]) + loading = _sequence([1, 2, 30, 40]) + concurrent_save = _sequence([1, 2, 30, 50]) + + engine.insert(cached, engine.take(4), is_ready=True) + pending = _pending(engine, loading, (True, True), 2, 4) + + # The save splits the old [1,2,3,4] node and adds [30,50]. The GET's fresh + # match is now three blocks, so its reserved block for 30 is redundant. + save_match = engine.match(concurrent_save) + assert save_match.num_matched_blocks == 2 + engine.insert( + concurrent_save, + engine.take(2), + is_ready=True, + match_result=save_match, + ) + + manager._commit_deferred_insert(pending) + + cached_match = engine.match(cached) + loading_match = engine.match(loading) + save_match = engine.match(concurrent_save) + assert cached_match.num_ready_matched_blocks == 4 + assert loading_match.num_ready_matched_blocks == 4 + assert save_match.num_ready_matched_blocks == 4 + assert loading_match.physical_blocks[2] == save_match.physical_blocks[2] + assert loading_match.physical_blocks[3] == pending.physical_blocks[1] + assert engine.index.total_cached_blocks() == 7 + assert engine.index.total_unready_blocks() == 0 + assert engine.mempool.num_used_blocks == 7 + + +def test_deferred_insert_recycles_staging_rejected_by_radix_guard(): + engine = _engine() + sequence = _sequence([1, 2]) + pending = _pending(engine, sequence, (True, True), 0, 2) + staged = pending.physical_blocks.copy() + engine.insert = Mock(side_effect=RuntimeError( + "radix insert conflict: target child already exists; " + "rematch before retrying")) + + _manager(engine)._commit_deferred_insert(pending) + + assert engine.mempool._free_mask[staged].all() + assert engine.mempool.num_used_blocks == 0 + + +def test_callback_cleans_up_old_anchor_when_deferred_commit_fails(): + manager = object.__new__(GlobalCacheEngine) + manager._cache_tree_lock = threading.RLock() + manager._commit_deferred_insert = Mock( + side_effect=RuntimeError("injected commit failure")) + manager.cpu_cache_engine = SimpleNamespace( + unlock=Mock(), + set_ready=Mock(), + recycle=Mock(), + ) + anchor = object() + buffer = np.asarray([7, 8], dtype=np.int64) + pending = SimpleNamespace(device_type=DeviceType.CPU) + + manager._transfer_callback( + node_to_unlock={DeviceType.CPU: (anchor, 2)}, + buffer_to_free={DeviceType.CPU: buffer}, + deferred_inserts=[pending], + ) + + manager._commit_deferred_insert.assert_called_once_with(pending) + manager.cpu_cache_engine.unlock.assert_called_once_with(anchor) + manager.cpu_cache_engine.set_ready.assert_called_once_with(anchor, True, 2) + assert manager.cpu_cache_engine.recycle.call_count == 1 + recycled = manager.cpu_cache_engine.recycle.call_args.args[0] + np.testing.assert_array_equal(recycled, buffer) diff --git a/tests/test_mooncake_deferred_put.py b/tests/test_mooncake_deferred_put.py new file mode 100644 index 0000000000..255399597f --- /dev/null +++ b/tests/test_mooncake_deferred_put.py @@ -0,0 +1,355 @@ +"""Concurrency tests for completion-time Mooncake PUT publication.""" + +import threading +from dataclasses import replace +from types import SimpleNamespace +from unittest.mock import Mock + +import numpy as np +import pytest + + +pytest.importorskip("flexkv.c_ext") + +from flexkv.cache.cache_engine import ( + CacheEngine, + CacheStrategy, + GlobalCacheEngine, +) +from flexkv.cache.radixtree import MatchResult +from flexkv.common.block import SequenceMeta +from flexkv.common.transfer import DeviceType, TransferType +from flexkv.common.type import MatchResultAccel + + +pytestmark = pytest.mark.unit + + +class _MooncakeRemoteStub: + """Key-addressed remote tier: block ids and radix mutation are no-ops.""" + + def __init__(self, kv_hit: int = 0): + self.kv_hit = kv_hit + + def match(self, sequence_meta): + hit = min(self.kv_hit, sequence_meta.num_blocks) + return MatchResultAccel( + num_ready_matched_blocks=hit, + num_matched_blocks=hit, + kv_matched_blocks=hit, + physical_blocks=np.arange(hit, dtype=np.int64), + matched_pos="global", + ) + + def take(self, num_required_blocks, protected_node=None, strict=True): + del protected_node, strict + return np.zeros(num_required_blocks, dtype=np.int64) + + def recycle(self, physical_blocks): + del physical_blocks + + +def _sequence(tokens): + return SequenceMeta( + token_ids=np.asarray(tokens, dtype=np.int64), + tokens_per_block=1, + ) + + +def _manager(remote_hit: int = 0, enable_ssd: bool = False): + manager = object.__new__(GlobalCacheEngine) + manager._cache_tree_lock = threading.RLock() + manager._metrics_collector = None + manager.tokens_per_block = 1 + manager.index_accel = False + manager.use_mooncake_store_backend = True + manager.enable_kv_sharing = False + manager.cache_config = SimpleNamespace( + tokens_per_block=1, + enable_cpu=True, + enable_ssd=enable_ssd, + enable_remote=True, + enable_p2p_cpu=False, + enable_p2p_ssd=False, + ) + manager.cpu_cache_engine = CacheEngine( + device_type=DeviceType.CPU, + num_total_blocks=32, + tokens_per_block=1, + evict_ratio=0.1, + ) + manager.ssd_cache_engine = ( + CacheEngine( + device_type=DeviceType.SSD, + num_total_blocks=32, + tokens_per_block=1, + evict_ratio=0.1, + ) + if enable_ssd else None + ) + manager.remote_cache_engine = _MooncakeRemoteStub(remote_hit) + manager.cache_engines = { + DeviceType.CPU: manager.cpu_cache_engine, + DeviceType.REMOTE: manager.remote_cache_engine, + } + if manager.ssd_cache_engine is not None: + manager.cache_engines[DeviceType.SSD] = manager.ssd_cache_engine + manager.swa_op_constructor = SimpleNamespace(enabled=False) + manager.match_all = lambda sequence_meta, temp_cache_strategy: ( + manager.cpu_cache_engine.match(sequence_meta), + (manager.ssd_cache_engine.match(sequence_meta) + if manager.ssd_cache_engine is not None else MatchResult()), + manager.remote_cache_engine.match(sequence_meta), + ) + manager._empty_put_return = lambda request_id: pytest.fail( + f"unexpected empty PUT plan for request {request_id}") + return manager + + +def _plan(manager, request_id, sequence, gpu_blocks=None): + if gpu_blocks is None: + gpu_blocks = np.arange(sequence.num_blocks, dtype=np.int64) + 100 + return manager._put_impl_global( + request_id=request_id, + sequence_meta=sequence, + block_mask_start=0, + block_mask_end=sequence.num_blocks, + gpu_block_ids=np.asarray(gpu_blocks, dtype=np.int64), + temp_cache_strategy=CacheStrategy(ignore_ssd=True), + dp_client_id=0, + ) + + +def _find_op(plan_or_graph, transfer_type): + graph = getattr(plan_or_graph, "transfer_graph", plan_or_graph) + return next( + op for op in graph._op_map.values() + if op.transfer_type == transfer_type and not op.is_swa + ) + + +def _seed_ready(engine, sequence): + blocks = engine.take(sequence.num_blocks) + node = engine.insert(sequence, blocks, is_ready=True) + assert node is not None + return blocks, node + + +def _start_put(manager, request_id, sequence, gpu_block_start): + token_mask = np.ones(sequence.num_blocks, dtype=np.int64) + slot_mapping = np.arange( + gpu_block_start, + gpu_block_start + sequence.num_blocks, + dtype=np.int64, + ) + graph, _, callback, _, _ = manager.put( + request_id=request_id, + token_ids=sequence.token_ids.copy(), + token_mask=token_mask, + slot_mapping=slot_mapping, + dp_client_id=0, + ) + return graph, callback + + +def test_cpu_ssd_duplicate_put_fresh_rematch_recycles_loser_after_split(): + manager = _manager(enable_ssd=True) + target = _sequence([1, 2, 3, 4]) + existing_branch = _sequence([1, 2, 9]) + cpu_seed, cpu_anchor = _seed_ready( + manager.cpu_cache_engine, existing_branch) + ssd_seed, ssd_anchor = _seed_ready( + manager.ssd_cache_engine, existing_branch) + initial_cpu_locks = cpu_anchor.lock_cnt + initial_ssd_locks = ssd_anchor.lock_cnt + + first_graph, first_callback = _start_put(manager, 1, target, 100) + first_cpu = _find_op(first_graph, TransferType.D2H).dst_block_ids.copy() + first_ssd = _find_op( + first_graph, TransferType.H2DISK).dst_block_ids.copy() + second_graph, second_callback = _start_put(manager, 2, target, 200) + second_cpu = _find_op(second_graph, TransferType.D2H).dst_block_ids.copy() + second_ssd = _find_op( + second_graph, TransferType.H2DISK).dst_block_ids.copy() + + for callback in (first_callback, second_callback): + locked = callback.keywords["node_to_unlock"] + assert locked[DeviceType.CPU][0] is cpu_anchor + assert locked[DeviceType.CPU][1] == 0 + assert locked[DeviceType.SSD][0] is ssd_anchor + assert locked[DeviceType.SSD][1] == 0 + pending_tiers = { + pending.device_type + for pending in callback.keywords["deferred_inserts"] + } + assert pending_tiers == {DeviceType.CPU, DeviceType.SSD} + assert cpu_anchor.lock_cnt == initial_cpu_locks + 2 + assert ssd_anchor.lock_cnt == initial_ssd_locks + 2 + + # The second completion owns both tails and splits the pre-existing branch. + second_callback() + assert cpu_anchor.size() == 1 + assert ssd_anchor.size() == 1 + assert cpu_anchor.lock_cnt == initial_cpu_locks + 1 + assert ssd_anchor.lock_cnt == initial_ssd_locks + 1 + + # The first completion must rematch both trees and recycle both duplicate + # staging allocations without disturbing the winning path. + first_callback() + assert cpu_anchor.lock_cnt == initial_cpu_locks + assert ssd_anchor.lock_cnt == initial_ssd_locks + + expected_by_tier = ( + (manager.cpu_cache_engine, cpu_seed, first_cpu, second_cpu), + (manager.ssd_cache_engine, ssd_seed, first_ssd, second_ssd), + ) + for engine, seed, loser, winner in expected_by_tier: + target_match = engine.match(target) + branch_match = engine.match(existing_branch) + np.testing.assert_array_equal( + target_match.physical_blocks, + np.concatenate((seed[:2], winner)), + ) + np.testing.assert_array_equal(branch_match.physical_blocks, seed) + assert target_match.num_ready_matched_blocks == target.num_blocks + assert branch_match.num_ready_matched_blocks == existing_branch.num_blocks + assert engine.mempool._free_mask[loser].all() + assert not engine.mempool._free_mask[winner].any() + assert engine.mempool.num_used_blocks == engine.index.total_cached_blocks() + assert engine.index.total_unready_blocks() == 0 + + +@pytest.mark.parametrize("finish_second_first", [False, True]) +def test_duplicate_puts_publish_one_owner_and_recycle_the_loser( + finish_second_first, +): + manager = _manager() + sequence = _sequence([1, 2, 3]) + first = _plan(manager, 1, sequence) + second = _plan(manager, 2, sequence) + first_blocks = first.deferred_inserts[0].physical_blocks.copy() + second_blocks = second.deferred_inserts[0].physical_blocks.copy() + + assert manager.cpu_cache_engine.index.total_cached_blocks() == 0 + assert manager.cpu_cache_engine.mempool.num_used_blocks == 6 + + winner, loser = ((second, first) if finish_second_first + else (first, second)) + manager._commit_deferred_insert(winner.deferred_inserts[0]) + manager._commit_deferred_insert(loser.deferred_inserts[0]) + + expected = second_blocks if finish_second_first else first_blocks + duplicate = first_blocks if finish_second_first else second_blocks + match = manager.cpu_cache_engine.match(sequence) + np.testing.assert_array_equal(match.physical_blocks, expected) + assert match.num_ready_matched_blocks == 3 + assert manager.cpu_cache_engine.mempool.num_used_blocks == 3 + assert manager.cpu_cache_engine.mempool._free_mask[duplicate].all() + assert not manager.cpu_cache_engine.mempool._free_mask[expected].any() + + +@pytest.mark.parametrize("finish_second_first", [False, True]) +def test_overlapping_puts_fresh_rematch_and_split_without_leaking( + finish_second_first, +): + manager = _manager() + first_sequence = _sequence([1, 2]) + second_sequence = _sequence([1, 3]) + first = _plan(manager, 1, first_sequence) + second = _plan(manager, 2, second_sequence) + first_blocks = first.deferred_inserts[0].physical_blocks.copy() + second_blocks = second.deferred_inserts[0].physical_blocks.copy() + + ordered = (second, first) if finish_second_first else (first, second) + for plan in ordered: + manager._commit_deferred_insert(plan.deferred_inserts[0]) + + first_match = manager.cpu_cache_engine.match(first_sequence) + second_match = manager.cpu_cache_engine.match(second_sequence) + expected_shared = second_blocks[0] if finish_second_first else first_blocks[0] + assert first_match.physical_blocks.tolist() == [ + expected_shared, first_blocks[1]] + assert second_match.physical_blocks.tolist() == [ + expected_shared, second_blocks[1]] + assert manager.cpu_cache_engine.index.total_cached_blocks() == 3 + assert manager.cpu_cache_engine.index.total_unready_blocks() == 0 + assert manager.cpu_cache_engine.mempool.num_used_blocks == 3 + + +def test_put_never_uses_an_unready_local_prefix_as_mooncake_source(): + manager = _manager() + sequence = _sequence([1, 2, 3]) + unready = manager.cpu_cache_engine.take(1) + manager.cpu_cache_engine.insert( + sequence, unready, num_insert_blocks=1, is_ready=False) + + plan = _plan(manager, 1, sequence) + d2h = _find_op(plan, TransferType.D2H) + h2remote = _find_op(plan, TransferType.H2REMOTE) + + # All three blocks come from this request's GPU data. The pre-existing + # unready block must not appear in the upload source. + assert len(d2h.dst_block_ids) == 3 + np.testing.assert_array_equal(h2remote.src_block_ids, d2h.dst_block_ids) + assert int(unready[0]) not in set(h2remote.src_block_ids.tolist()) + assert manager.cpu_cache_engine.index.total_cached_blocks() == 1 + + +def test_ready_cpu_hit_retries_a_missing_mooncake_upload_without_d2h(): + manager = _manager(remote_hit=0) + sequence = _sequence([1, 2, 3]) + initial = _plan(manager, 1, sequence) + manager._commit_deferred_insert(initial.deferred_inserts[0]) + cached = manager.cpu_cache_engine.match(sequence).physical_blocks.copy() + + retry = _plan(manager, 2, sequence) + op_types = { + op.transfer_type for op in retry.transfer_graph._op_map.values() + } + assert TransferType.D2H not in op_types + assert TransferType.H2REMOTE in op_types + np.testing.assert_array_equal( + _find_op(retry, TransferType.H2REMOTE).src_block_ids, + cached, + ) + assert retry.deferred_inserts == [] + assert retry.node_to_unlock[DeviceType.CPU][1] == 0 + + +def test_duplicate_put_recycles_its_swa_slot_without_replacing_the_winner(): + manager = _manager() + sequence = _sequence([1, 2]) + engine = manager.cpu_cache_engine + freed_slots = [] + engine.swa_pool = SimpleNamespace( + free=lambda slot: freed_slots.append(int(slot))) + + first = _plan(manager, 1, sequence).deferred_inserts[0] + second = _plan(manager, 2, sequence).deferred_inserts[0] + first = replace(first, swa_slot=7) + second = replace(second, swa_slot=8) + + winner = manager._commit_deferred_insert(first) + manager._commit_deferred_insert(second) + + assert winner.has_swa() + assert winner.swa_host_slot == 7 + assert freed_slots == [8] + + +def test_p2p_publication_happens_once_for_the_inserted_ready_node(): + manager = _manager() + sequence = _sequence([1, 2]) + engine = manager.cpu_cache_engine + engine.local_index = SimpleNamespace(insert_and_publish=Mock()) + + first = _plan(manager, 1, sequence).deferred_inserts[0] + second = _plan(manager, 2, sequence).deferred_inserts[0] + first = replace(first, publish_to_peer=True) + second = replace(second, publish_to_peer=True) + + inserted = manager._commit_deferred_insert(first) + manager._commit_deferred_insert(second) + + engine.local_index.insert_and_publish.assert_called_once_with(inserted) + assert engine.match(sequence).num_ready_matched_blocks == 2 diff --git a/tests/test_mooncake_transfer_result.py b/tests/test_mooncake_transfer_result.py new file mode 100644 index 0000000000..e2d3ce888f --- /dev/null +++ b/tests/test_mooncake_transfer_result.py @@ -0,0 +1,103 @@ +"""Focused unit tests for Mooncake per-block transfer completion results.""" + +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +pytest.importorskip("flexkv.c_ext") + +from flexkv.common.transfer import TransferType +from flexkv.external.mooncake_store_keys import PoolKind +from flexkv.transfer.worker import MooncakeStoreTransferWorker +from flexkv.transfer.worker_op import WorkerTransferResult + + +pytestmark = pytest.mark.unit + + +def _worker(block_results=None, error=None): + """Build a worker without CUDA registration or a real Mooncake client.""" + worker = object.__new__(MooncakeStoreTransferWorker) + worker.pool_kind = PoolKind.KV + worker._preprocess_kv = lambda _op: ( + [100, 200, 300], + [16, 16, 16], + ["k0", "k1", "k2"], + ) + worker._log_transfer_performance = lambda *_args: None + + def batch_get(_keys, _ptrs, _sizes): + if error is not None: + raise error + return block_results + + worker.mooncake_client = SimpleNamespace(batch_get=batch_get) + return worker + + +def _remote2h_op(op_id=17): + return SimpleNamespace( + transfer_op_id=op_id, + transfer_type=TransferType.REMOTE2H, + src_block_ids=[0, 1, 2], + ) + + +def test_mooncake_preserves_non_contiguous_block_results(): + worker = _worker([True, False, True]) + + result = worker.launch_transfer(_remote2h_op()) + + assert result == WorkerTransferResult( + transfer_op_id=17, + block_results=(True, False, True), + ) + + +def test_mooncake_exception_still_returns_completed_failure(): + worker = _worker(error=RuntimeError("injected get failure")) + + result = worker.launch_transfer(_remote2h_op(op_id=23)) + + assert result == WorkerTransferResult( + transfer_op_id=23, + block_results=(False, False, False), + ) + + +def test_mooncake_preprocess_exception_still_returns_completed_failure(): + worker = _worker() + + def fail_preprocess(_op): + raise RuntimeError("injected preprocess failure") + + worker._preprocess_kv = fail_preprocess + + result = worker.launch_transfer(_remote2h_op(op_id=29)) + + assert result == WorkerTransferResult( + transfer_op_id=29, + block_results=(False, False, False), + ) + + +@pytest.mark.parametrize("block_results", [None, [True, False]]) +def test_mooncake_invalid_result_count_fails_closed(block_results): + worker = _worker(block_results) + + result = worker.launch_transfer(_remote2h_op(op_id=31)) + + assert result.block_results == (False, False, False) + + +def test_mooncake_perf_logging_exception_still_completes(): + worker = _worker([True, True, True]) + worker._log_transfer_performance = Mock(side_effect=RuntimeError("log failure")) + + result = worker.launch_transfer(_remote2h_op(op_id=37)) + + assert result == WorkerTransferResult( + transfer_op_id=37, + block_results=(False, False, False), + )