Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
248 changes: 225 additions & 23 deletions mooncake-integration/store/store_py.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -315,6 +315,64 @@ int DistributedObjectStore::allocateSlices(
return 0;
}

int DistributedObjectStore::allocateBatchedSlices(
std::vector<std::string> &keys,
const std::unordered_map<std::string, std::span<const char>> &batches,
std::unordered_map<std::string, std::vector<mooncake::Slice>>
&batched_slices) {
if (batches.empty()) return -1;
for (const auto &item : batches) {
uint64_t offset = 0;
auto key = item.first;
auto value = item.second;
std::vector<Slice> slices;
while (offset < value.size()) {
auto chunk_size = std::min(value.size() - offset, kMaxSliceSize);
auto ptr = client_buffer_allocator_->allocate(chunk_size);
if (!ptr) {
return 1;
}
memcpy(ptr, value.data() + offset, chunk_size);
slices.emplace_back(Slice{ptr, chunk_size});
offset += chunk_size;
}
keys.emplace_back(key);
batched_slices.emplace(key, std::move(slices));
}
return 0;
}

int DistributedObjectStore::allocateBatchedSlices(
const std::vector<std::string> &keys,
std::unordered_map<std::string, std::vector<mooncake::Slice>>
&batched_slices,
const mooncake::Client::BatchObjectInfo &batched_object_info,
std::unordered_map<std::string, uint64_t> &str_length_map) {
if (batched_object_info.batch_replica_list.empty()) return -1;
for (const auto &key : keys) {
auto object_info_it = batched_object_info.batch_replica_list.find(key);
if (object_info_it == batched_object_info.batch_replica_list.end()) {
LOG(ERROR) << "Key not found: " << key;
return 1;
}
// Get first replica
auto &replica = object_info_it->second[0];
uint64_t length = 0;
for (auto &handle : replica.buffer_descriptors) {
auto chunk_size = handle.size_;
assert(chunk_size <= kMaxSliceSize);
auto ptr = client_buffer_allocator_->allocate(chunk_size);
if (!ptr) {
return 1;
}
batched_slices[key].emplace_back(Slice{ptr, chunk_size});
length += chunk_size;
}
str_length_map.emplace(key, length);
}
return 0;
}

char *DistributedObjectStore::exportSlices(
const std::vector<mooncake::Slice> &slices, uint64_t length) {
char *buf = new char[length + 1];
Expand Down Expand Up @@ -377,6 +435,36 @@ int DistributedObjectStore::put(const std::string &key,
return 0;
}

int DistributedObjectStore::put_batch(
const std::unordered_map<std::string, std::span<const char>> &batches) {
if (!client_) {
LOG(ERROR) << "Client is not initialized";
return 1;
}
std::vector<std::string> keys;
std::unordered_map<std::string, std::vector<mooncake::Slice>>
batched_slices;
int ret = allocateBatchedSlices(keys, batches, batched_slices);
if (ret) {
LOG(ERROR) << "Failed to allocate slices for put_batch operation";
return ret;
}

ReplicateConfig config;
config.replica_num = 1;
ErrorCode error_code = client_->BatchPut(keys, batched_slices, config);
if (error_code != ErrorCode::OK) {
LOG(ERROR) << "BatchPut operation failed with error: "
<< toString(error_code);
return toInt(error_code);
}

for (auto &slice : batched_slices) {
freeSlices(slice.second);
}
return 0;
}

int DistributedObjectStore::put_parts(
const std::string &key, std::vector<std::span<const char>> values) {
if (!client_) {
Expand Down Expand Up @@ -466,6 +554,76 @@ pybind11::bytes DistributedObjectStore::get(const std::string &key) {
return result;
}

std::vector<pybind11::bytes> DistributedObjectStore::get_batch(
const std::vector<std::string> &keys) {
const auto kNullString = pybind11::bytes("\0", 0);
if (!client_) {
LOG(ERROR) << "Client is not initialized";
return {kNullString};
}
std::unordered_set<std::string> seen;
for (const auto &key : keys) {
if (!seen.insert(key).second) {
LOG(ERROR) << "Duplicate key not supported for Batch API, key: "
<< key;
return {kNullString};
}
}

std::vector<pybind11::bytes> results;
mooncake::Client::BatchObjectInfo batched_object_info;
std::unordered_map<std::string, std::vector<mooncake::Slice>>
batched_slices;
std::unordered_map<std::string, uint64_t> str_length_map;
{
py::gil_scoped_release release_gil;
ErrorCode error_code = client_->BatchQuery(keys, batched_object_info);
if (error_code != ErrorCode::OK) {
py::gil_scoped_acquire acquire_gil;
return {kNullString};
} else {
int ret = allocateBatchedSlices(
keys, batched_slices, batched_object_info, str_length_map);
if (ret) {
py::gil_scoped_acquire acquire_gil;
return {kNullString};
}
error_code =
client_->BatchGet(keys, batched_object_info, batched_slices);
if (error_code != ErrorCode::OK) {
py::gil_scoped_acquire acquire_gil;
return {kNullString};
}
}
for (const auto &key : keys) {
if (batched_slices[key].size() == 1 &&
batched_slices[key][0].size == str_length_map[key]) {
results.push_back(pybind11::bytes(
static_cast<char *>(batched_slices[key][0].ptr),
str_length_map[key]));
} else {
char *exported_str_ptr =
exportSlices(batched_slices[key], str_length_map[key]);
if (!exported_str_ptr) {
return {kNullString};
} else {
results.push_back(
pybind11::bytes(exported_str_ptr, str_length_map[key]));
delete[] exported_str_ptr;
}
}
}
if (results.size() != keys.size()) {
LOG(ERROR) << "Results size does not match keys size";
return {kNullString};
}
for (auto &slice : batched_slices) {
freeSlices(slice.second);
}
return results;
}
}

int DistributedObjectStore::remove(const std::string &key) {
if (!client_) {
LOG(ERROR) << "Client is not initialized";
Expand Down Expand Up @@ -523,6 +681,25 @@ int64_t DistributedObjectStore::getSize(const std::string &key) {
return total_size;
}

std::unordered_map<std::string, bool> DistributedObjectStore::BatchIsExist(
const std::vector<std::string> &keys) {
std::unordered_map<std::string, bool> result_map;
if (!client_) {
LOG(ERROR) << "Client is not initialized";
return result_map;
}
const auto existence_map = client_->BatchIsExist(keys);
if (existence_map.size() != keys.size()) {
LOG(ERROR) << "Internal error: existence_map.size() is not equal to "
"keys.size()";
return result_map;
}
for (const auto &[key, err] : existence_map) {
result_map[key] = (err == ErrorCode::OK);
}
return result_map;
}

// SliceBuffer implementation
SliceBuffer::SliceBuffer(DistributedObjectStore &store, void *buffer,
uint64_t size, bool use_allocator_free)
Expand Down Expand Up @@ -648,6 +825,7 @@ PYBIND11_MODULE(store, m) {
.def("setup", &DistributedObjectStore::setup)
.def("init_all", &DistributedObjectStore::initAll)
.def("get", &DistributedObjectStore::get)
.def("get_batch", &DistributedObjectStore::get_batch)
.def("get_buffer", &DistributedObjectStore::get_buffer,
py::call_guard<py::gil_scoped_release>(),
py::return_value_policy::take_ownership)
Expand All @@ -657,6 +835,8 @@ PYBIND11_MODULE(store, m) {
py::call_guard<py::gil_scoped_release>())
.def("is_exist", &DistributedObjectStore::isExist,
py::call_guard<py::gil_scoped_release>())
.def("is_batch_exist", &DistributedObjectStore::BatchIsExist,
py::call_guard<py::gil_scoped_release>())
.def("close", &DistributedObjectStore::tearDownAll)
.def("get_size", &DistributedObjectStore::getSize,
py::call_guard<py::gil_scoped_release>())
Expand All @@ -669,27 +849,49 @@ PYBIND11_MODULE(store, m) {
static_cast<char *>(info.ptr),
static_cast<size_t>(info.size)));
})
.def("put_parts", [](DistributedObjectStore &self,
const std::string &key, py::args parts) {
// 1) Python buffer → span
std::vector<py::buffer_info> infos;
std::vector<std::span<const char>> spans;
infos.reserve(parts.size());
spans.reserve(parts.size());

for (auto &obj : parts) {
py::buffer buf = py::reinterpret_borrow<py::buffer>(obj);
infos.emplace_back(buf.request(false));
const auto &info = infos.back();
if (info.ndim != 1 || info.itemsize != 1)
throw std::runtime_error("parts must be 1-D bytes-like");

spans.emplace_back(static_cast<const char *>(info.ptr),
static_cast<size_t>(info.size));
}

// 2) Call C++ function
py::gil_scoped_release unlock;
return self.put_parts(key, spans);
});
.def("put_parts",
[](DistributedObjectStore &self, const std::string &key,
py::args parts) {
// 1) Python buffer → span
std::vector<py::buffer_info> infos;
std::vector<std::span<const char>> spans;
infos.reserve(parts.size());
spans.reserve(parts.size());

for (auto &obj : parts) {
py::buffer buf = py::reinterpret_borrow<py::buffer>(obj);
infos.emplace_back(buf.request(false));
const auto &info = infos.back();
if (info.ndim != 1 || info.itemsize != 1)
throw std::runtime_error(
"parts must be 1-D bytes-like");

spans.emplace_back(static_cast<const char *>(info.ptr),
static_cast<size_t>(info.size));
}

// 2) Call C++ function
py::gil_scoped_release unlock;
return self.put_parts(key, spans);
})
.def(
"put_batch",
[](DistributedObjectStore &self,
const std::unordered_map<std::string, py::bytes> &batches) {
std::unordered_map<std::string, std::span<const char>>
batches_c;
std::vector<std::vector<char>> temp_buffers;
for (auto item : batches) {
const std::string &key = item.first;
const std::string &value = item.second;
temp_buffers.emplace_back(value.begin(), value.end());

batches_c.emplace(
key, std::span<const char>(temp_buffers.back().data(),
temp_buffers.back().size()));
}

py::gil_scoped_release release;
return self.put_batch(batches_c);
});
}
26 changes: 26 additions & 0 deletions mooncake-integration/store/store_py.h
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
#pragma once

#include <pybind11/pybind11.h>
#include <pybind11/stl.h>

#include <csignal>
#include <mutex>
Expand Down Expand Up @@ -112,11 +113,15 @@ class DistributedObjectStore {

int put(const std::string &key, std::span<const char> value);

int put_batch(const std::unordered_map<std::string, std::span<const char>> &batches);

int put_parts(const std::string &key,
std::vector<std::span<const char>> values);

pybind11::bytes get(const std::string &key);

std::vector<pybind11::bytes> get_batch(
const std::vector<std::string> &keys);
/**
* @brief Get a buffer containing the data for a key
* @param key Key to get data for
Expand All @@ -138,6 +143,14 @@ class DistributedObjectStore {
*/
int isExist(const std::string &key);

/**
* @brief Check if a batch of objects exists
* @param keys Key to check
* @return Map of keys to booleans
*/
std::unordered_map<std::string, bool> BatchIsExist(

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I’d prefer to return `vector that matches up one-to-one with the keys in the input parameters.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

According to SGLang integration, is it better to return <key, is_exist> pair so that the existed key can be easily appended instead of addressing them by index again.

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was looking through the sglang code and noticed this part:

mooncake_exist_keys = self.mooncake_l3_kv_pool.is_batch_exist(
    fragment_keys
)
non_exist_keys = []
non_exist_value = []
for i in range(len(fragment_keys)):
    if not mooncake_exist_keys[fragment_keys[i]]:

I think we could simplify it to:

mooncake_exist_keys = self.mooncake_l3_kv_pool.is_batch_exist(
    fragment_keys
)
non_exist_keys = []
non_exist_value = []
for i in range(len(fragment_keys)):
    if mooncake_exist_keys[i] == 1:

This way we're using direct indexing instead of querying the hash map, which should be slightly more efficient.

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

LMCache needs same interface, so I’ve submitted a separate PR for batch exist: #542.

Appreciate it if you could take a look. Thanks!

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

great work! Since array is more efficient, let's use array as the return value of batch exist interface.

const std::vector<std::string> &keys);

/**
* @brief Get the size of an object
* @param key Key of the object
Expand All @@ -160,6 +173,19 @@ class DistributedObjectStore {
int allocateSlicesPacked(std::vector<mooncake::Slice> &slices,
const std::vector<std::span<const char>> &parts);

int allocateBatchedSlices(
const std::vector<std::string> &keys,
std::unordered_map<std::string, std::vector<mooncake::Slice>>
&batched_slices,
const mooncake::Client::BatchObjectInfo &batched_object_info,
std::unordered_map<std::string, uint64_t> &str_length_map);

int allocateBatchedSlices(
std::vector<std::string> &keys,
const std::unordered_map<std::string, std::span<const char>> &batches,
std::unordered_map<std::string, std::vector<mooncake::Slice>>
&batched_slices);

char *exportSlices(const std::vector<mooncake::Slice> &slices,
uint64_t length);

Expand Down
8 changes: 8 additions & 0 deletions mooncake-store/include/client.h
Original file line number Diff line number Diff line change
Expand Up @@ -191,6 +191,14 @@ class Client {
*/
ErrorCode IsExist(const std::string& key);

/**
* @brief Checks if a batch of objects exist
* @param keys Keys to check
* @return existence map, true if exists, false if not
*/
std::unordered_map<std::string, ErrorCode> BatchIsExist(

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

same above

const std::vector<std::string>& keys);

private:
/**
* @brief Private constructor to enforce creation through Create() method
Expand Down
8 changes: 8 additions & 0 deletions mooncake-store/include/master_client.h
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,14 @@ class MasterClient {
[[nodiscard]] ExistKeyResponse ExistKey(
const std::string& object_key);

/**
* @brief Checks if a batch of objects exist
* @param object_keys Keys to query
* @return ErrorCode indicating success/failure
*/
[[nodiscard]] BatchExistKeyResponse BatchExistKey(
const std::vector<std::string>& object_keys);

/**
* @brief Gets object metadata without transferring data
* @param object_key Key to query
Expand Down
7 changes: 7 additions & 0 deletions mooncake-store/include/master_service.h
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,13 @@ class MasterService {
*/
ErrorCode ExistKey(const std::string& key);

/**
* @brief Check if an batch of objects exist
* @return existence map, key is the object name, value is the error code
*/
std::unordered_map<std::string, ErrorCode> BatchExistKey(
const std::vector<std::string>& key);

/**
* @brief Fetch all keys
* @return ErrorCode::OK if exists
Expand Down
Loading