Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 7 additions & 4 deletions mooncake-store/include/kv_event/kv_event_publisher.h
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,12 @@ class KvEventPublisher {
// Non-blocking enqueue into a bounded queue; drops oldest when full.
// tenant_id empty defaults to "default" on the wire.
void PublishStored(const std::string& object_key, const std::string& medium,
const std::string& tenant_id = "");
const std::string& tenant_id = "",
const std::string& group_id = "");
void PublishRemoved(const std::string& object_key,
const std::string& medium,
const std::string& tenant_id = "");
const std::string& tenant_id = "",
const std::string& group_id = "");

struct Stats {
uint64_t published_batches{0};
Expand All @@ -58,6 +60,7 @@ class KvEventPublisher {
std::string object_key;
std::string medium;
std::string tenant_id;
std::string group_id;
};

void Enqueue(PendingEvent event);
Expand Down Expand Up @@ -95,9 +98,9 @@ class KvEventPublisher {
bool enabled() const { return false; }

void PublishStored(const std::string&, const std::string&,
const std::string& = "") {}
const std::string& = "", const std::string& = "") {}
void PublishRemoved(const std::string&, const std::string&,
const std::string& = "") {}
const std::string& = "", const std::string& = "") {}

struct Stats {
uint64_t published_batches{0};
Expand Down
3 changes: 2 additions & 1 deletion mooncake-store/include/master_service.h
Original file line number Diff line number Diff line change
Expand Up @@ -2143,7 +2143,8 @@ class MasterService {
const ObjectMetadata& metadata,
const std::string& tenant_id);
void PublishKvRemoved(const std::string& key, const std::string& medium,
const std::string& tenant_id);
const std::string& tenant_id,
const std::string& group_id);
void PublishKvRemovedAfterEvict(const std::string& key,
uint64_t freed_bytes,
const std::string& medium,
Expand Down
18 changes: 12 additions & 6 deletions mooncake-store/src/kv_event/kv_event_publisher.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,8 @@ size_t ComputeEventMapSize(bool is_stored, bool emit_legacy,
bool emit_object_key) {
// Base envelope: event_id, timestamp, event_type, model_name, block_size,
// additional_salt, lora_name, tenant_id, backend_id, medium, dp_rank,
// seq_hashes.
constexpr size_t kBaseFields = 12;
// seq_hashes, group_id.
constexpr size_t kBaseFields = 13;
size_t map_size = kBaseFields;
if (emit_legacy) {
map_size += 2; // type, block_hashes
Expand Down Expand Up @@ -140,20 +140,24 @@ KvEventPublisher::~KvEventPublisher() {

void KvEventPublisher::PublishStored(const std::string& object_key,
const std::string& medium,
const std::string& tenant_id) {
const std::string& tenant_id,
const std::string& group_id) {
if (!config_.enabled) {
return;
}
Enqueue(PendingEvent{EventKind::kStored, object_key, medium, tenant_id});
Enqueue(PendingEvent{EventKind::kStored, object_key, medium, tenant_id,
group_id});
}

void KvEventPublisher::PublishRemoved(const std::string& object_key,
const std::string& medium,
const std::string& tenant_id) {
const std::string& tenant_id,
const std::string& group_id) {
if (!config_.enabled) {
return;
}
Enqueue(PendingEvent{EventKind::kRemoved, object_key, medium, tenant_id});
Enqueue(PendingEvent{EventKind::kRemoved, object_key, medium, tenant_id,
group_id});
}

KvEventPublisher::Stats KvEventPublisher::GetStats() const {
Expand Down Expand Up @@ -288,6 +292,8 @@ void KvEventPublisher::PublishBatch(const std::vector<PendingEvent>& batch) {
packer.pack(tenant_id);
packer.pack("backend_id");
packer.pack(config_.backend_id);
packer.pack("group_id");
PackOptionalString(packer, item.pending.group_id);
packer.pack("medium");
PackOptionalString(packer, item.pending.medium);
packer.pack("dp_rank");
Expand Down
13 changes: 8 additions & 5 deletions mooncake-store/src/master_service.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -7804,7 +7804,7 @@ void MasterService::NoFBatchEvict(double evict_ratio_target,
total_freed_size += metadata.size * erased;
shard_evicted_count++;
PublishKvRemovedAfterEvict(it->first, metadata.size * erased,
"disk", metadata, tenant_it->first);
"disk", metadata, tenant_it->first);
if (!metadata.IsValid()) {
it = EraseMetadata(tenant_state, it, tenant_it->first,
QuotaEraseMode::kFull, &shard);
Expand Down Expand Up @@ -9589,22 +9589,25 @@ void MasterService::PublishKvStored(const std::string& key,
if (replica_type == ReplicaType::ALL) {
medium = MediumForMetadata(metadata);
}
kv_event_publisher_->PublishStored(key, medium, tenant_id);
kv_event_publisher_->PublishStored(key, medium, tenant_id,
metadata.group_id);
}

void MasterService::PublishKvRemoved(const std::string& key,
const std::string& medium,
const std::string& tenant_id) {
const std::string& tenant_id,
const std::string& group_id) {
if (!kv_event_publisher_ || !kv_event_publisher_->enabled()) {
return;
}
kv_event_publisher_->PublishRemoved(key, medium, tenant_id);
kv_event_publisher_->PublishRemoved(key, medium, tenant_id, group_id);
}

void MasterService::PublishKvRemoved(const std::string& key,
const ObjectMetadata& metadata,
const std::string& tenant_id) {
PublishKvRemoved(key, MediumForMetadata(metadata), tenant_id);
PublishKvRemoved(key, MediumForMetadata(metadata), tenant_id,
metadata.group_id);
}

void MasterService::PublishKvRemovedAfterEvict(const std::string& key,
Expand Down
11 changes: 10 additions & 1 deletion mooncake-store/tests/kv_event_publisher_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,9 @@ TEST(KvEventPublisherTest, PublishesSglangObjectKeyOverZmq) {
const std::string endpoint = MakeIpcEndpoint();
const std::string object_key =
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855_0_k";
const std::string group_id =
"sglang-hicache:"
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";

KvEventConfig config;
config.enabled = true;
Expand All @@ -100,7 +103,7 @@ TEST(KvEventPublisherTest, PublishesSglangObjectKeyOverZmq) {
// Allow SUB connect before first publish propagates.
std::this_thread::sleep_for(std::chrono::milliseconds(100));

publisher.PublishStored(object_key, "cpu", "tenant-a");
publisher.PublishStored(object_key, "cpu", "tenant-a", group_id);

std::vector<std::string> frames;
ASSERT_TRUE(ReceiveZmqMultipart(sub, frames)) << zmq_strerror(zmq_errno());
Expand All @@ -122,6 +125,7 @@ TEST(KvEventPublisherTest, PublishesSglangObjectKeyOverZmq) {
ASSERT_EQ(event.type, msgpack::type::MAP);

bool has_object_key = false;
bool has_group_id = false;
bool has_empty_seq_hashes = false;
std::string event_type;
std::string backend_id;
Expand All @@ -136,6 +140,10 @@ TEST(KvEventPublisherTest, PublishesSglangObjectKeyOverZmq) {
EXPECT_EQ(std::string(val.via.str.ptr, val.via.str.size),
object_key);
has_object_key = true;
} else if (field == "group_id") {
ASSERT_EQ(val.type, msgpack::type::STR);
EXPECT_EQ(std::string(val.via.str.ptr, val.via.str.size), group_id);
has_group_id = true;
} else if (field == "seq_hashes") {
ASSERT_EQ(val.type, msgpack::type::ARRAY);
EXPECT_EQ(val.via.array.size, 0u);
Expand All @@ -156,6 +164,7 @@ TEST(KvEventPublisherTest, PublishesSglangObjectKeyOverZmq) {
EXPECT_EQ(backend_id, "mooncake-test");
EXPECT_EQ(tenant_id, "tenant-a");
EXPECT_TRUE(has_object_key);
EXPECT_TRUE(has_group_id);
EXPECT_TRUE(has_empty_seq_hashes);

// Wait for async worker to finish publishing.
Expand Down
Loading