Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ jobs:
cd build
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib
ldconfig -v || echo "always continue"
MC_METADATA_SERVER=http://127.0.0.1:8080/metadata make test -j ARGS="-V"
MC_METADATA_SERVER=http://127.0.0.1:8080/metadata MC_STORE_MEMCPY=false make test -j ARGS="-V"
shell: bash

- name: Stop Mooncake Master Service
Expand Down Expand Up @@ -172,7 +172,7 @@ jobs:
- name: Run tests
run: |
source test_env/bin/activate
./scripts/run_tests.sh
MC_STORE_MEMCPY=false ./scripts/run_tests.sh
deactivate
shell: bash

Expand Down
152 changes: 152 additions & 0 deletions mooncake-integration/store/store_py.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -603,6 +603,125 @@ std::shared_ptr<SliceBuffer> DistributedObjectStore::get_buffer(
return result;
}

int DistributedObjectStore::register_buffer(void *buffer, size_t size) {
if (!client_) {
LOG(ERROR) << "Client is not initialized";
return 1;
}
ErrorCode error_code =
client_->RegisterLocalMemory(buffer, size, kWildcardLocation);
if (error_code != ErrorCode::OK) {
LOG(ERROR) << "Register buffer failed with error: "
<< toString(error_code);
return toInt(error_code);
}
return 0;
}

int DistributedObjectStore::get_into(const std::string &key, void *buffer,
size_t size) {
// NOTE: The buffer address must be previously registered with
// register_buffer() for zero-copy RDMA operations to work correctly
if (!client_) {
LOG(ERROR) << "Client is not initialized";
return -1;
}

mooncake::Client::ObjectInfo object_info;
ErrorCode error_code;

// Step 1: Get object info
error_code = client_->Query(key, object_info);
if (error_code == ErrorCode::OBJECT_NOT_FOUND) {
VLOG(1) << "Object not found for key: " << key;
return -toInt(error_code);
}
if (error_code != ErrorCode::OK) {
LOG(ERROR) << "Query failed for key: " << key
<< " with error: " << toString(error_code);
return -toInt(error_code);
}

// Calculate total size from object info
uint64_t total_size = 0;
if (object_info.replica_list.empty()) {
LOG(ERROR) << "Internal error: object_info.replica_list is empty";
return -1;
}

auto &replica = object_info.replica_list[0];
for (auto &handle : replica.buffer_descriptors) {
total_size += handle.size_;
}

// Check if user buffer is large enough
if (size < total_size) {
LOG(ERROR) << "User buffer too small. Required: " << total_size
<< ", provided: " << size;
return -1;
}

// Step 2: Split user buffer according to object info and create slices
std::vector<mooncake::Slice> slices;
uint64_t offset = 0;

for (auto &handle : replica.buffer_descriptors) {
auto chunk_size = handle.size_;
void *chunk_ptr = static_cast<char *>(buffer) + offset;
slices.emplace_back(Slice{chunk_ptr, chunk_size});
offset += chunk_size;
}

// Step 3: Read data directly into user buffer
error_code = client_->Get(key, object_info, slices);
if (error_code != ErrorCode::OK) {
LOG(ERROR) << "Get failed for key: " << key
<< " with error: " << toString(error_code);
return -toInt(error_code);
}

return static_cast<int>(total_size);
}

int DistributedObjectStore::put_from(const std::string &key, void *buffer,
size_t size) {
// NOTE: The buffer address must be previously registered with
// register_buffer() for zero-copy RDMA operations to work correctly
if (!client_) {
LOG(ERROR) << "Client is not initialized";
return -1;
}

if (size == 0) {
LOG(WARNING) << "Attempting to put empty data for key: " << key;
return 0;
}

// Create slices directly from the user buffer
std::vector<mooncake::Slice> slices;
uint64_t offset = 0;

while (offset < size) {
auto chunk_size = std::min(size - offset, kMaxSliceSize);
void *chunk_ptr = static_cast<char *>(buffer) + offset;
slices.emplace_back(Slice{chunk_ptr, chunk_size});
offset += chunk_size;
}

ReplicateConfig config;
config.replica_num = 1; // Make configurable
config.preferred_segment = this->local_hostname;

ErrorCode error_code = client_->Put(key, slices, config);
if (error_code != ErrorCode::OK) {
LOG(ERROR) << "Put operation failed with error: "
<< toString(error_code);
return -toInt(error_code);
}

return 0;
}

PYBIND11_MODULE(store, m) {
// Define the SliceBuffer class
py::class_<SliceBuffer, std::shared_ptr<SliceBuffer>>(m, "SliceBuffer",
Expand Down Expand Up @@ -660,6 +779,39 @@ PYBIND11_MODULE(store, m) {
.def("close", &DistributedObjectStore::tearDownAll)
.def("get_size", &DistributedObjectStore::getSize,
py::call_guard<py::gil_scoped_release>())
.def(
"register_buffer",
[](DistributedObjectStore &self, uintptr_t buffer_ptr,
size_t size) {
// Register memory buffer for RDMA operations
void *buffer = reinterpret_cast<void *>(buffer_ptr);
py::gil_scoped_release release;
return self.register_buffer(buffer, size);
},
py::arg("buffer_ptr"), py::arg("size"),
"Register a memory buffer for direct access operations")
.def(
"get_into",
[](DistributedObjectStore &self, const std::string &key,
uintptr_t buffer_ptr, size_t size) {
// Get data directly into user-provided buffer
void *buffer = reinterpret_cast<void *>(buffer_ptr);
py::gil_scoped_release release;
return self.get_into(key, buffer, size);
},
py::arg("key"), py::arg("buffer_ptr"), py::arg("size"),
"Get object data directly into a pre-allocated buffer")
.def(
"put_from",
[](DistributedObjectStore &self, const std::string &key,
uintptr_t buffer_ptr, size_t size) {
// Put data directly from user-provided buffer
void *buffer = reinterpret_cast<void *>(buffer_ptr);
py::gil_scoped_release release;
return self.put_from(key, buffer, size);
},
py::arg("key"), py::arg("buffer_ptr"), py::arg("size"),
"Put object data directly from a pre-allocated buffer")
.def("put",
[](DistributedObjectStore &self, const std::string &key,
py::buffer buf) {
Expand Down
26 changes: 26 additions & 0 deletions mooncake-integration/store/store_py.h
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,32 @@ class DistributedObjectStore {

int put(const std::string &key, std::span<const char> value);

int register_buffer(void *buffer, size_t size);

/**
* @brief Get object data directly into a pre-allocated buffer
* @param key Key of the object to get
* @param buffer Pointer to the pre-allocated buffer (must be registered
* with register_buffer)
* @param size Size of the buffer
* @return Number of bytes read on success, negative value on error
* @note The buffer address must be previously registered with
* register_buffer() for zero-copy operations
*/
int get_into(const std::string &key, void *buffer, size_t size);

/**
* @brief Put object data directly from a pre-allocated buffer
* @param key Key of the object to put
* @param buffer Pointer to the buffer containing data (must be registered
* with register_buffer)
* @param size Size of the data to put
* @return 0 on success, negative value on error
* @note The buffer address must be previously registered with
* register_buffer() for zero-copy operations
*/
int put_from(const std::string &key, void *buffer, size_t size);

int put_parts(const std::string &key,
std::vector<std::span<const char>> values);

Expand Down
1 change: 1 addition & 0 deletions mooncake-store/include/transfer_task.h
Original file line number Diff line number Diff line change
Expand Up @@ -292,6 +292,7 @@ class TransferSubmitter {
TransferEngine& engine_;
const std::string local_hostname_;
std::unique_ptr<MemcpyWorkerPool> memcpy_pool_;
bool memcpy_enabled_;

/**
* @brief Select the optimal transfer strategy
Expand Down
2 changes: 1 addition & 1 deletion mooncake-store/src/allocator.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ AllocatedBuffer::~AllocatedBuffer() {
<< " size=" << size_;
} else {
MasterMetricManager::instance().dec_allocated_size(size_);
LOG(WARNING) << "allocator=expired_or_null in buf_handle_destructor";
VLOG(1) << "allocator=expired_or_null in buf_handle_destructor";
}
}

Expand Down
45 changes: 38 additions & 7 deletions mooncake-store/src/transfer_task.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
#include <glog/logging.h>

#include <algorithm>
#include <cstdlib>

#include "utils.h"

Expand Down Expand Up @@ -150,8 +151,8 @@ void TransferEngineOperationState::check_task_status() {
}

if (has_failure) {
LOG(INFO) << "Setting batch " << batch_id_
<< " result to TRANSFER_FAIL due to task failures";
VLOG(1) << "Setting batch " << batch_id_
<< " result to TRANSFER_FAIL due to task failures";
set_result_internal(ErrorCode::TRANSFER_FAIL);
return;
}
Expand All @@ -174,8 +175,8 @@ void TransferEngineOperationState::set_result_internal(ErrorCode error_code) {
return; // Don't crash, just return early
}

LOG(INFO) << "Setting transfer result for batch " << batch_id_ << " to "
<< static_cast<int>(error_code);
VLOG(1) << "Setting transfer result for batch " << batch_id_ << " to "
<< static_cast<int>(error_code);
result_.emplace(error_code);

cv_.notify_all();
Expand Down Expand Up @@ -249,6 +250,31 @@ TransferSubmitter::TransferSubmitter(TransferEngine& engine,
local_hostname_(local_hostname),
memcpy_pool_(std::make_unique<MemcpyWorkerPool>()) {
CHECK(!local_hostname_.empty()) << "Local hostname cannot be empty";

// Read MC_STORE_MEMCPY environment variable, default to true (enabled)
const char* env_value = std::getenv("MC_STORE_MEMCPY");
if (env_value == nullptr) {
memcpy_enabled_ = true; // Default: enabled
} else {
std::string env_str(env_value);
// Convert to lowercase for case-insensitive comparison
std::transform(env_str.begin(), env_str.end(), env_str.begin(),
::tolower);
if (env_str == "false" || env_str == "0" || env_str == "no" ||
env_str == "off") {
memcpy_enabled_ = false;
} else if (env_str == "true" || env_str == "1" || env_str == "yes" ||
env_str == "on") {
memcpy_enabled_ = true;
} else {
LOG(WARNING) << "Invalid value for MC_STORE_MEMCPY: " << env_str
<< ", defaulting to enabled";
memcpy_enabled_ = true;
}
}

VLOG(1) << "TransferSubmitter initialized with memcpy_enabled="
<< memcpy_enabled_;
}

std::optional<TransferFuture> TransferSubmitter::submit(
Expand Down Expand Up @@ -371,9 +397,14 @@ std::optional<TransferFuture> TransferSubmitter::submitTransferEngineOperation(
TransferStrategy TransferSubmitter::selectStrategy(
const std::vector<AllocatedBuffer::Descriptor>& handles,
const std::vector<Slice>& slices) const {
// Check conditions for local memcpy optimization (currently disabled as
// RDMA is faster)
// TODO: add a config to disable this optimization
// Check if memcpy operations are enabled via environment variable
if (!memcpy_enabled_) {
VLOG(2) << "Memcpy operations disabled via MC_STORE_MEMCPY environment "
"variable";
return TransferStrategy::TRANSFER_ENGINE;
}

// Check conditions for local memcpy optimization
if (isLocalTransfer(handles)) {
return TransferStrategy::LOCAL_MEMCPY;
}
Expand Down
Loading