Repository navigation
feat(store): add optional RFC #1527 KV events publisher on master #2214
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 18 commits
4d73577
0278598
ac17fbc
012bf0f
a25f852
642eef8
d5c7a05
6e9a1d8
12d913c
6a75ae2
1caac0d
1216fb6
c462efc
e4dbbc4
7abfdca
58046c3
02b494f
eb9f1fe
6e13367
e9fcf14
6d670b5
2f9df5b
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,36 @@ | ||
| #pragma once | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Spec vs implementation review (RFC #1527 master KV publisher) Overall the implementation matches the spec in #1527 quite closely:
Below are concrete line-level checks and a few small nits:
Conclusion
Otherwise this aligns well with issue #1527. ✅ 🤖 Generated by Qoder • Fix in Qoder |
||
|
|
||
| #include <cstdint> | ||
| #include <optional> | ||
| #include <string> | ||
|
|
||
| namespace mooncake { | ||
|
|
||
| // Parses object keys encoded as decimal or 0x-prefixed hex u64 hashes. | ||
| inline std::optional<uint64_t> ParseSeqHashFromObjectKey( | ||
| const std::string& object_key) { | ||
| if (object_key.empty()) { | ||
| return std::nullopt; | ||
| } | ||
| try { | ||
| size_t idx = 0; | ||
| if (object_key.size() >= 2 && | ||
| (object_key[0] == '0' && | ||
| (object_key[1] == 'x' || object_key[1] == 'X'))) { | ||
| uint64_t value = std::stoull(object_key, &idx, 16); | ||
| if (idx == object_key.size()) { | ||
| return value; | ||
| } | ||
| return std::nullopt; | ||
| } | ||
| uint64_t value = std::stoull(object_key, &idx, 10); | ||
| if (idx == object_key.size()) { | ||
| return value; | ||
| } | ||
| } catch (const std::exception&) { | ||
| return std::nullopt; | ||
| } | ||
| return std::nullopt; | ||
| } | ||
|
|
||
| } // namespace mooncake | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| #pragma once | ||
|
|
||
| #include <cstdint> | ||
| #include <string> | ||
|
|
||
| namespace mooncake { | ||
|
|
||
| // Configuration for the optional KV Events publisher (RFC #1527). | ||
| struct KvEventConfig { | ||
| bool enabled{false}; | ||
| // ZMQ PUB bind address, e.g. "tcp://0.0.0.0:5557". | ||
| std::string bind_endpoint; | ||
| std::string model_name; | ||
| std::string backend_id; | ||
| std::string tenant_id{"default"}; | ||
| std::string additional_salt; | ||
| std::string lora_name; | ||
| uint32_t block_size{0}; | ||
| uint32_t dp_rank{0}; | ||
| // Emit legacy vLLM/SGLang field names alongside RFC #1527 fields. | ||
| bool emit_legacy_compat_fields{true}; | ||
| // Emit Mooncake object_key for consumers that match on store key format. | ||
| bool emit_object_key{true}; | ||
| }; | ||
|
|
||
| } // namespace mooncake |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,121 @@ | ||
| #pragma once | ||
|
|
||
| #include <atomic> | ||
| #include <condition_variable> | ||
| #include <cstdint> | ||
| #include <deque> | ||
| #include <memory> | ||
| #include <mutex> | ||
| #include <optional> | ||
| #include <string> | ||
| #include <thread> | ||
| #include <vector> | ||
|
|
||
| #include "kv_event/kv_event_config.h" | ||
| #include "kv_event/key_util.h" | ||
|
|
||
| namespace mooncake { | ||
|
|
||
| #if defined(MOONCAKE_ENABLE_KV_EVENTS) && MOONCAKE_ENABLE_KV_EVENTS | ||
|
|
||
| // Publishes standardized KV cache events (RFC #1527) over ZMQ for indexers. | ||
| class KvEventPublisher { | ||
| public: | ||
| explicit KvEventPublisher(KvEventConfig config); | ||
| ~KvEventPublisher(); | ||
|
|
||
| KvEventPublisher(const KvEventPublisher&) = delete; | ||
| KvEventPublisher& operator=(const KvEventPublisher&) = delete; | ||
|
|
||
| bool enabled() const { return config_.enabled; } | ||
|
|
||
| // Non-blocking enqueue; never drops events (unbounded queue). | ||
| // tenant_id empty uses config_.tenant_id (RFC envelope default). | ||
| void PublishStored(const std::string& object_key, const std::string& medium, | ||
| const std::string& tenant_id = ""); | ||
| void PublishRemoved(const std::string& object_key, | ||
| const std::string& medium, | ||
| const std::string& tenant_id = ""); | ||
|
|
||
| struct Stats { | ||
| uint64_t published_batches{0}; | ||
| uint64_t published_events{0}; | ||
| uint64_t dropped_events{0}; | ||
| uint64_t skipped_unparsed_keys{0}; | ||
| }; | ||
| Stats GetStats() const; | ||
|
|
||
| static std::optional<uint64_t> ParseSeqHashFromObjectKey( | ||
| const std::string& object_key) { | ||
| return mooncake::ParseSeqHashFromObjectKey(object_key); | ||
| } | ||
|
|
||
| private: | ||
| enum class EventKind { kStored, kRemoved }; | ||
|
|
||
| struct PendingEvent { | ||
| EventKind kind; | ||
| std::string object_key; | ||
| std::string medium; | ||
| std::string tenant_id; | ||
| }; | ||
|
|
||
| void Enqueue(PendingEvent event); | ||
| void WorkerLoop(); | ||
| void PublishBatch(const std::vector<PendingEvent>& batch); | ||
| void DrainRemainingQueue(std::vector<PendingEvent>& batch); | ||
|
|
||
| KvEventConfig config_; | ||
| void* zmq_context_{nullptr}; | ||
| void* zmq_socket_{nullptr}; | ||
|
|
||
| mutable std::mutex queue_mutex_; | ||
| std::deque<PendingEvent> queue_; | ||
| std::condition_variable queue_cv_; | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. missing
Collaborator
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. No require |
||
| std::thread worker_; | ||
| std::atomic<bool> stop_{false}; | ||
|
|
||
| std::atomic<uint64_t> next_event_id_{1}; | ||
| std::atomic<uint64_t> next_zmq_sequence_{1}; | ||
|
|
||
| std::atomic<uint64_t> published_batches_{0}; | ||
| std::atomic<uint64_t> published_events_{0}; | ||
| std::atomic<uint64_t> dropped_events_{0}; | ||
| std::atomic<uint64_t> skipped_unparsed_keys_{0}; | ||
| }; | ||
|
|
||
| #else | ||
|
|
||
| // Stub when mooncake_store is built without libzmq (ENABLE_KV_EVENTS=OFF). | ||
| class KvEventPublisher { | ||
| public: | ||
| explicit KvEventPublisher(KvEventConfig config) | ||
| : config_(std::move(config)) {} | ||
|
|
||
| bool enabled() const { return false; } | ||
|
|
||
| void PublishStored(const std::string&, const std::string&, | ||
| const std::string& = "") {} | ||
| void PublishRemoved(const std::string&, const std::string&, | ||
| const std::string& = "") {} | ||
|
|
||
| struct Stats { | ||
| uint64_t published_batches{0}; | ||
| uint64_t published_events{0}; | ||
| uint64_t dropped_events{0}; | ||
| uint64_t skipped_unparsed_keys{0}; | ||
| }; | ||
| Stats GetStats() const { return {}; } | ||
|
|
||
| static std::optional<uint64_t> ParseSeqHashFromObjectKey( | ||
| const std::string& object_key) { | ||
| return mooncake::ParseSeqHashFromObjectKey(object_key); | ||
| } | ||
|
|
||
| private: | ||
| KvEventConfig config_; | ||
| }; | ||
|
|
||
| #endif | ||
|
|
||
| } // namespace mooncake | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Why we need these configurations.