Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions csrc/dist/distributed_radix_tree.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -571,6 +571,7 @@ std::shared_ptr<CMatchResult> RefRadixTree::match_prefix(
auto last_ready_node = root;
auto prefix_blocks_num = 0;
auto ready_prefix_blocks_num = 0;
bool ready_prefix_open = true;
auto last_node_matched_length = 0;
auto physical_blocks_tensor = torch::empty({num_blocks}, torch::dtype(torch::kInt64));
auto *pb_out = physical_blocks_tensor.data_ptr<int64_t>();
Expand Down Expand Up @@ -643,9 +644,11 @@ std::shared_ptr<CMatchResult> RefRadixTree::match_prefix(
ni_out[ni_write++] = (*bnis)[i];
}

if (current_node->is_ready()) {
if (ready_prefix_open && current_node->is_ready()) {
last_ready_node = current_node;
ready_prefix_blocks_num += matched;
} else if (matched > 0) {
ready_prefix_open = false;
}

prefix_blocks_num += matched;
Expand Down Expand Up @@ -674,7 +677,7 @@ std::shared_ptr<CMatchResult> RefRadixTree::match_prefix(
auto physical_blocks = physical_blocks_tensor.narrow(0, 0, pb_write);
auto node_ids = node_ids_tensor.narrow(0, 0, ni_write);

return std::make_shared<CMatchResult>(prefix_blocks_num, prefix_blocks_num, last_node_matched_length,
return std::make_shared<CMatchResult>(ready_prefix_blocks_num, prefix_blocks_num, last_node_matched_length,
last_ready_node, current_node, physical_blocks, node_ids);
}

Expand Down Expand Up @@ -1041,4 +1044,3 @@ void attach_and_merge_root_child(CRadixTreeIndex* temp_tree, CRadixNode* root_ch

} // namespace flexkv


8 changes: 7 additions & 1 deletion csrc/dist/local_radix_tree.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -117,13 +117,19 @@ CRadixNode *LocalRadixTree::insert(torch::Tensor &physical_block_ids,
}

assert(last_node != nullptr);
assert(last_node_matched_length != 0 || is_root(last_node));
assert(physical_block_ids.size() == num_insert_blocks - num_matched_blocks);

if (num_matched_blocks >= num_insert_blocks) {
return nullptr;
}

// Keep stale-match handling identical to the base radix tree. This check is
// deliberately before split/LeaseMeta allocation/current_block_count changes;
// a rejected insert leaves both the tree and caller-owned physical blocks
// untouched.
validate_insert_target(last_node, block_hashes, num_blocks,
num_matched_blocks, last_node_matched_length);

// Handle split if the last node only partially matched
if (last_node_matched_length < last_node->size()) {
CRadixNode *new_parent = last_node->split(last_node_matched_length);
Expand Down
77 changes: 73 additions & 4 deletions csrc/radix_tree.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -257,6 +257,62 @@ std::deque<int64_t> *CRadixNode::shrink_simple(int length) {
return shrink_blocks;
}

void CRadixTreeIndex::validate_insert_target(
CRadixNode *last_node, torch::Tensor &block_hashes, int num_blocks,
int num_matched_blocks, int last_node_matched_length) {
static constexpr const char *stale_match_error =
"radix insert conflict: stale match result; rematch before retrying";

if (last_node == nullptr || last_node->get_index() != this ||
num_matched_blocks < 0 ||
num_matched_blocks >= num_blocks) {
throw std::runtime_error(stale_match_error);
}

const int node_size = last_node->size();
auto *block_hashes_ptr = block_hashes.data_ptr<int64_t>();
if (is_root(last_node)) {
if (last_node_matched_length != 0 || num_matched_blocks != 0) {
throw std::runtime_error(stale_match_error);
}
} else {
CRadixNode *parent = last_node->get_parent();
const HashType head_hash = last_node->get_head_hash();
if (parent == nullptr || !parent->lookup_child(head_hash) ||
parent->get_child(head_hash) != last_node ||
last_node_matched_length <= 0 ||
last_node_matched_length > node_size) {
throw std::runtime_error(stale_match_error);
}

const int matched_start =
num_matched_blocks - last_node_matched_length;
if (matched_start < 0) {
throw std::runtime_error(stale_match_error);
}
for (int i = 0; i < last_node_matched_length; ++i) {
if (last_node->get_hash(i) !=
HashType(block_hashes_ptr[matched_start + i])) {
throw std::runtime_error(stale_match_error);
}
}
}

const HashType target_hash =
HashType(block_hashes_ptr[num_matched_blocks]);
if (last_node_matched_length < node_size) {
// A valid partial match diverges at the split point. Equality means the
// supplied match result no longer describes this node.
if (last_node->get_hash(last_node_matched_length) == target_hash) {
throw std::runtime_error(stale_match_error);
}
} else if (last_node->lookup_child(target_hash)) {
throw std::runtime_error(
"radix insert conflict: target child already exists; "
"rematch before retrying");
}
}

CRadixNode *CRadixTreeIndex::insert(torch::Tensor &physical_block_ids,
torch::Tensor &block_hashes, int num_blocks,
int num_insert_blocks, bool ready,
Expand All @@ -278,13 +334,17 @@ CRadixNode *CRadixTreeIndex::insert(torch::Tensor &physical_block_ids,
}

assert(last_node != nullptr);
assert(last_node_matched_length != 0 || is_root(last_node));
assert(physical_block_ids.size() == num_insert_blocks - num_matched_blocks);

if (num_matched_blocks >= num_insert_blocks) {
return nullptr;
}

// This must run before allocating/splitting/removing leaves: on conflict the
// tree is unchanged and the caller retains every supplied physical block.
validate_insert_target(last_node, block_hashes, num_blocks,
num_matched_blocks, last_node_matched_length);

auto new_node = new CRadixNode(this, ready, 0);
auto &new_block_hashes = new_node->get_block_hashes();
auto &new_physical_blocks = new_node->get_physical_blocks();
Expand Down Expand Up @@ -574,6 +634,9 @@ CRadixTreeIndex::match_prefix(torch::Tensor &block_hashes, int num_blocks,
auto last_ready_node = root;
auto prefix_blocks_num = 0;
auto ready_prefix_blocks_num = 0;
// Readiness is a contiguous-prefix property. Once a matched node is unready,
// a ready descendant must not be counted until that ancestor becomes ready.
bool ready_prefix_open = true;
auto last_node_matched_length = 0;
// SWA (node-mount): deepest fully-matched ready node carrying a live SWA
// slot.
Expand Down Expand Up @@ -613,7 +676,7 @@ CRadixTreeIndex::match_prefix(torch::Tensor &block_hashes, int num_blocks,
break;
}
}
if (current_node->is_ready()) {
if (ready_prefix_open && current_node->is_ready()) {
last_ready_node = current_node;
ready_prefix_blocks_num += matched_length;
// SWA hit only when the WHOLE node matched (its trailing page is
Expand All @@ -622,6 +685,8 @@ CRadixTreeIndex::match_prefix(torch::Tensor &block_hashes, int num_blocks,
last_swa_node = current_node;
swa_hit_blocks = ready_prefix_blocks_num;
}
} else if (matched_length > 0) {
ready_prefix_open = false;
}
last_node_matched_length = matched_length;
prefix_blocks_num += matched_length;
Expand All @@ -637,14 +702,16 @@ CRadixTreeIndex::match_prefix(torch::Tensor &block_hashes, int num_blocks,
if (child_hash_opt.has_value() &&
current_node->lookup_child(child_hash_opt.value())) {
child_hash = child_hash_opt.value();
if (current_node->is_ready()) {
if (ready_prefix_open && current_node->is_ready()) {
last_ready_node = current_node;
ready_prefix_blocks_num += current_node->size();
// Whole node matched (we are descending into a child): expose its SWA.
if (current_node->has_swa()) {
last_swa_node = current_node;
swa_hit_blocks = ready_prefix_blocks_num;
}
} else if (current_node->size() > 0) {
ready_prefix_open = false;
}
prefix_blocks_num += current_node->size();
auto &pbs = current_node->get_physical_blocks();
Expand Down Expand Up @@ -683,14 +750,16 @@ CRadixTreeIndex::match_prefix(torch::Tensor &block_hashes, int num_blocks,
matched_length = 0;
}

if (current_node->is_ready()) {
if (ready_prefix_open && current_node->is_ready()) {
last_ready_node = current_node;
ready_prefix_blocks_num += matched_length;
// Only a full node match exposes the trailing page's SWA.
if (matched_length == current_node->size() && current_node->has_swa()) {
last_swa_node = current_node;
swa_hit_blocks = ready_prefix_blocks_num;
}
} else if (matched_length > 0) {
ready_prefix_open = false;
}

last_node_matched_length = matched_length;
Expand Down
9 changes: 9 additions & 0 deletions csrc/radix_tree.h
Original file line number Diff line number Diff line change
Expand Up @@ -340,6 +340,15 @@ class CRadixTreeIndex {
std::vector<int64_t> &out_blocks,
std::vector<int64_t> *out_hashes);

// Validate a caller-supplied match anchor before any split, allocation, or
// child-map mutation. A conflict raises std::runtime_error; ownership of the
// supplied physical blocks remains with the caller so it can rematch/retry or
// recycle them. LocalRadixTree reuses the same guard in its insert override.
void validate_insert_target(CRadixNode *last_node,
torch::Tensor &block_hashes, int num_blocks,
int num_matched_blocks,
int last_node_matched_length);

public:
CRadixTreeIndex(int tokens_per_block, int max_num_blocks = 1000000,
int hit_reward_seconds = 0,
Expand Down
Loading
Loading