From fc3f2f71581066f18d6ee83d9ecc64585ebc9f07 Mon Sep 17 00:00:00 2001 From: Jinyang Su <751080330@qq.com> Date: Fri, 20 Jun 2025 17:15:28 +0800 Subject: [PATCH 1/4] feat(store): add zero-copy operations for python binding --- .github/workflows/ci.yml | 4 +- mooncake-integration/store/store_py.cpp | 152 ++++++++++++++++++ mooncake-integration/store/store_py.h | 26 +++ mooncake-store/include/transfer_task.h | 1 + mooncake-store/src/allocator.cpp | 2 +- mooncake-store/src/transfer_task.cpp | 45 +++++- .../tests/test_distributed_object_store.py | 62 ++++++- 7 files changed, 280 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 90678488300..f00dde7ffb3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -102,7 +102,7 @@ jobs: cd build export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib ldconfig -v || echo "always continue" - MC_METADATA_SERVER=http://127.0.0.1:8080/metadata make test -j ARGS="-V" + MC_METADATA_SERVER=http://127.0.0.1:8080/metadata MC_STORE_MEMCPY=false make test -j ARGS="-V" shell: bash - name: Stop Mooncake Master Service @@ -172,7 +172,7 @@ jobs: - name: Run tests run: | source test_env/bin/activate - ./scripts/run_tests.sh + MC_STORE_MEMCPY=false ./scripts/run_tests.sh deactivate shell: bash diff --git a/mooncake-integration/store/store_py.cpp b/mooncake-integration/store/store_py.cpp index f6339068c78..fe86ed2a09e 100644 --- a/mooncake-integration/store/store_py.cpp +++ b/mooncake-integration/store/store_py.cpp @@ -603,6 +603,125 @@ std::shared_ptr DistributedObjectStore::get_buffer( return result; } +int DistributedObjectStore::register_buffer(void *buffer, size_t size) { + if (!client_) { + LOG(ERROR) << "Client is not initialized"; + return 1; + } + ErrorCode error_code = + client_->RegisterLocalMemory(buffer, size, kWildcardLocation); + if (error_code != ErrorCode::OK) { + LOG(ERROR) << "Register buffer failed with error: " + << toString(error_code); + return toInt(error_code); + } + return 0; +} + +int DistributedObjectStore::get_into(const std::string &key, void *buffer, + size_t size) { + // NOTE: The buffer address must be previously registered with + // register_buffer() for zero-copy RDMA operations to work correctly + if (!client_) { + LOG(ERROR) << "Client is not initialized"; + return -1; + } + + mooncake::Client::ObjectInfo object_info; + ErrorCode error_code; + + // Step 1: Get object info + error_code = client_->Query(key, object_info); + if (error_code == ErrorCode::OBJECT_NOT_FOUND) { + VLOG(1) << "Object not found for key: " << key; + return -toInt(error_code); + } + if (error_code != ErrorCode::OK) { + LOG(ERROR) << "Query failed for key: " << key + << " with error: " << toString(error_code); + return -toInt(error_code); + } + + // Calculate total size from object info + uint64_t total_size = 0; + if (object_info.replica_list.empty()) { + LOG(ERROR) << "Internal error: object_info.replica_list is empty"; + return -1; + } + + auto &replica = object_info.replica_list[0]; + for (auto &handle : replica.buffer_descriptors) { + total_size += handle.size_; + } + + // Check if user buffer is large enough + if (size < total_size) { + LOG(ERROR) << "User buffer too small. Required: " << total_size + << ", provided: " << size; + return -1; + } + + // Step 2: Split user buffer according to object info and create slices + std::vector slices; + uint64_t offset = 0; + + for (auto &handle : replica.buffer_descriptors) { + auto chunk_size = handle.size_; + void *chunk_ptr = static_cast(buffer) + offset; + slices.emplace_back(Slice{chunk_ptr, chunk_size}); + offset += chunk_size; + } + + // Step 3: Read data directly into user buffer + error_code = client_->Get(key, object_info, slices); + if (error_code != ErrorCode::OK) { + LOG(ERROR) << "Get failed for key: " << key + << " with error: " << toString(error_code); + return -toInt(error_code); + } + + return static_cast(total_size); +} + +int DistributedObjectStore::put_from(const std::string &key, void *buffer, + size_t size) { + // NOTE: The buffer address must be previously registered with + // register_buffer() for zero-copy RDMA operations to work correctly + if (!client_) { + LOG(ERROR) << "Client is not initialized"; + return -1; + } + + if (size == 0) { + LOG(WARNING) << "Attempting to put empty data for key: " << key; + return 0; + } + + // Create slices directly from the user buffer + std::vector slices; + uint64_t offset = 0; + + while (offset < size) { + auto chunk_size = std::min(size - offset, kMaxSliceSize); + void *chunk_ptr = static_cast(buffer) + offset; + slices.emplace_back(Slice{chunk_ptr, chunk_size}); + offset += chunk_size; + } + + ReplicateConfig config; + config.replica_num = 1; // Make configurable + config.preferred_segment = this->local_hostname; + + ErrorCode error_code = client_->Put(key, slices, config); + if (error_code != ErrorCode::OK) { + LOG(ERROR) << "Put operation failed with error: " + << toString(error_code); + return -toInt(error_code); + } + + return 0; +} + PYBIND11_MODULE(store, m) { // Define the SliceBuffer class py::class_>(m, "SliceBuffer", @@ -660,6 +779,39 @@ PYBIND11_MODULE(store, m) { .def("close", &DistributedObjectStore::tearDownAll) .def("get_size", &DistributedObjectStore::getSize, py::call_guard()) + .def( + "register_buffer", + [](DistributedObjectStore &self, uintptr_t buffer_ptr, + size_t size) { + // Register memory buffer for RDMA operations + void *buffer = reinterpret_cast(buffer_ptr); + py::gil_scoped_release release; + return self.register_buffer(buffer, size); + }, + py::arg("buffer_ptr"), py::arg("size"), + "Register a memory buffer for direct access operations") + .def( + "get_into", + [](DistributedObjectStore &self, const std::string &key, + uintptr_t buffer_ptr, size_t size) { + // Get data directly into user-provided buffer + void *buffer = reinterpret_cast(buffer_ptr); + py::gil_scoped_release release; + return self.get_into(key, buffer, size); + }, + py::arg("key"), py::arg("buffer_ptr"), py::arg("size"), + "Get object data directly into a pre-allocated buffer") + .def( + "put_from", + [](DistributedObjectStore &self, const std::string &key, + uintptr_t buffer_ptr, size_t size) { + // Put data directly from user-provided buffer + void *buffer = reinterpret_cast(buffer_ptr); + py::gil_scoped_release release; + return self.put_from(key, buffer, size); + }, + py::arg("key"), py::arg("buffer_ptr"), py::arg("size"), + "Put object data directly from a pre-allocated buffer") .def("put", [](DistributedObjectStore &self, const std::string &key, py::buffer buf) { diff --git a/mooncake-integration/store/store_py.h b/mooncake-integration/store/store_py.h index c30a5c7c3e3..8ff5f78e06f 100644 --- a/mooncake-integration/store/store_py.h +++ b/mooncake-integration/store/store_py.h @@ -112,6 +112,32 @@ class DistributedObjectStore { int put(const std::string &key, std::span value); + int register_buffer(void *buffer, size_t size); + + /** + * @brief Get object data directly into a pre-allocated buffer + * @param key Key of the object to get + * @param buffer Pointer to the pre-allocated buffer (must be registered + * with register_buffer) + * @param size Size of the buffer + * @return Number of bytes read on success, negative value on error + * @note The buffer address must be previously registered with + * register_buffer() for zero-copy operations + */ + int get_into(const std::string &key, void *buffer, size_t size); + + /** + * @brief Put object data directly from a pre-allocated buffer + * @param key Key of the object to put + * @param buffer Pointer to the buffer containing data (must be registered + * with register_buffer) + * @param size Size of the data to put + * @return 0 on success, negative value on error + * @note The buffer address must be previously registered with + * register_buffer() for zero-copy operations + */ + int put_from(const std::string &key, void *buffer, size_t size); + int put_parts(const std::string &key, std::vector> values); diff --git a/mooncake-store/include/transfer_task.h b/mooncake-store/include/transfer_task.h index 931473e2f96..196bef4e933 100644 --- a/mooncake-store/include/transfer_task.h +++ b/mooncake-store/include/transfer_task.h @@ -292,6 +292,7 @@ class TransferSubmitter { TransferEngine& engine_; const std::string local_hostname_; std::unique_ptr memcpy_pool_; + bool memcpy_enabled_; /** * @brief Select the optimal transfer strategy diff --git a/mooncake-store/src/allocator.cpp b/mooncake-store/src/allocator.cpp index 1f2c10be961..e5a44c365e1 100644 --- a/mooncake-store/src/allocator.cpp +++ b/mooncake-store/src/allocator.cpp @@ -17,7 +17,7 @@ AllocatedBuffer::~AllocatedBuffer() { << " size=" << size_; } else { MasterMetricManager::instance().dec_allocated_size(size_); - LOG(WARNING) << "allocator=expired_or_null in buf_handle_destructor"; + VLOG(1) << "allocator=expired_or_null in buf_handle_destructor"; } } diff --git a/mooncake-store/src/transfer_task.cpp b/mooncake-store/src/transfer_task.cpp index c9f619ce8a5..7106421af99 100644 --- a/mooncake-store/src/transfer_task.cpp +++ b/mooncake-store/src/transfer_task.cpp @@ -3,6 +3,7 @@ #include #include +#include #include "utils.h" @@ -150,8 +151,8 @@ void TransferEngineOperationState::check_task_status() { } if (has_failure) { - LOG(INFO) << "Setting batch " << batch_id_ - << " result to TRANSFER_FAIL due to task failures"; + VLOG(1) << "Setting batch " << batch_id_ + << " result to TRANSFER_FAIL due to task failures"; set_result_internal(ErrorCode::TRANSFER_FAIL); return; } @@ -174,8 +175,8 @@ void TransferEngineOperationState::set_result_internal(ErrorCode error_code) { return; // Don't crash, just return early } - LOG(INFO) << "Setting transfer result for batch " << batch_id_ << " to " - << static_cast(error_code); + VLOG(1) << "Setting transfer result for batch " << batch_id_ << " to " + << static_cast(error_code); result_.emplace(error_code); cv_.notify_all(); @@ -249,6 +250,31 @@ TransferSubmitter::TransferSubmitter(TransferEngine& engine, local_hostname_(local_hostname), memcpy_pool_(std::make_unique()) { CHECK(!local_hostname_.empty()) << "Local hostname cannot be empty"; + + // Read MC_STORE_MEMCPY environment variable, default to true (enabled) + const char* env_value = std::getenv("MC_STORE_MEMCPY"); + if (env_value == nullptr) { + memcpy_enabled_ = true; // Default: enabled + } else { + std::string env_str(env_value); + // Convert to lowercase for case-insensitive comparison + std::transform(env_str.begin(), env_str.end(), env_str.begin(), + ::tolower); + if (env_str == "false" || env_str == "0" || env_str == "no" || + env_str == "off") { + memcpy_enabled_ = false; + } else if (env_str == "true" || env_str == "1" || env_str == "yes" || + env_str == "on") { + memcpy_enabled_ = true; + } else { + LOG(WARNING) << "Invalid value for MC_STORE_MEMCPY: " << env_str + << ", defaulting to enabled"; + memcpy_enabled_ = true; + } + } + + VLOG(1) << "TransferSubmitter initialized with memcpy_enabled=" + << memcpy_enabled_; } std::optional TransferSubmitter::submit( @@ -371,9 +397,14 @@ std::optional TransferSubmitter::submitTransferEngineOperation( TransferStrategy TransferSubmitter::selectStrategy( const std::vector& handles, const std::vector& slices) const { - // Check conditions for local memcpy optimization (currently disabled as - // RDMA is faster) - // TODO: add a config to disable this optimization + // Check if memcpy operations are enabled via environment variable + if (!memcpy_enabled_) { + VLOG(2) << "Memcpy operations disabled via MC_STORE_MEMCPY environment " + "variable"; + return TransferStrategy::TRANSFER_ENGINE; + } + + // Check conditions for local memcpy optimization if (isLocalTransfer(handles)) { return TransferStrategy::LOCAL_MEMCPY; } diff --git a/mooncake-wheel/tests/test_distributed_object_store.py b/mooncake-wheel/tests/test_distributed_object_store.py index 22f63d9baa9..ee2b0eac202 100644 --- a/mooncake-wheel/tests/test_distributed_object_store.py +++ b/mooncake-wheel/tests/test_distributed_object_store.py @@ -14,7 +14,7 @@ def get_client(store): protocol = os.getenv("PROTOCOL", "tcp") device_name = os.getenv("DEVICE_NAME", "ibp6s0") local_hostname = os.getenv("LOCAL_HOSTNAME", "localhost") - metadata_server = os.getenv("MC_METADATA_SERVER", "127.0.0.1:2379") + metadata_server = os.getenv("MC_METADATA_SERVER", "http://127.0.0.1:8080/metadata") global_segment_size = 3200 * 1024 * 1024 # 3200 MB local_buffer_size = 512 * 1024 * 1024 # 512 MB master_server_address = os.getenv("MASTER_SERVER", "127.0.0.1:50051") @@ -105,6 +105,64 @@ def test_basic_put_get_exist_operations(self): self.assertEqual(self.store.remove(key_2), 0) self.assertLess(self.store.get_size(key_2), 0) self.assertEqual(self.store.is_exist(key_2), 0) + + def test_zero_copy_operations(self): + """Test zero-copy get_into and put_from operations.""" + import ctypes + + # Test data + test_data = b"Hello, Zero-Copy World! " * 1000 # ~24KB test data + key = "test_zero_copy_key" + + # Allocate a buffer and register it + buffer_size = len(test_data) + 1024 # Extra space for safety + buffer = (ctypes.c_ubyte * buffer_size)() + buffer_ptr = ctypes.addressof(buffer) + + # Register the buffer for zero-copy operations + result = self.store.register_buffer(buffer_ptr, buffer_size) + self.assertEqual(result, 0, "Buffer registration should succeed") + + # Copy test data to buffer + ctypes.memmove(buffer, test_data, len(test_data)) + + # Test put_from (zero-copy write) + result = self.store.put_from(key, buffer_ptr, len(test_data)) + self.assertEqual(result, 0, "put_from should succeed") + + # Verify data was stored correctly using regular get + retrieved_data = self.store.get(key) + self.assertEqual(retrieved_data, test_data, "Data should match after put_from") + + # Clear buffer for get_into test + ctypes.memset(buffer, 0, buffer_size) + + # Test get_into (zero-copy read) + bytes_read = self.store.get_into(key, buffer_ptr, buffer_size) + self.assertEqual(bytes_read, len(test_data), "get_into should return correct byte count") + + # Verify data was read correctly + read_data = bytes(buffer[:bytes_read]) + self.assertEqual(read_data, test_data, "Data should match after get_into") + + # Test error cases + # Test get_into with buffer too small + small_buffer_size = len(test_data) // 2 + small_buffer = (ctypes.c_ubyte * small_buffer_size)() + small_buffer_ptr = ctypes.addressof(small_buffer) + + # Register small buffer + result = self.store.register_buffer(small_buffer_ptr, small_buffer_size) + self.assertEqual(result, 0, "Small buffer registration should succeed") + + # get_into should fail with buffer too small + bytes_read = self.store.get_into(key, small_buffer_ptr, small_buffer_size) + self.assertLess(bytes_read, 0, "get_into should fail with small buffer") + + # Cleanup + time.sleep(DEFAULT_KV_LEASE_TTL / 1000) + self.assertEqual(self.store.remove(key), 0) + def test_concurrent_stress_with_barrier(self): """Test concurrent Put/Get operations with multiple threads using barrier.""" @@ -276,6 +334,6 @@ def test_dict_fuzz_e2e(self): time.sleep(DEFAULT_KV_LEASE_TTL / 1000) for key in list(reference.keys()): self.store.remove(key) - + if __name__ == '__main__': unittest.main() From 80361d22d878dfb3fb1e4ee9a23129f774c85ee9 Mon Sep 17 00:00:00 2001 From: Jinyang Su <751080330@qq.com> Date: Fri, 20 Jun 2025 17:23:50 +0800 Subject: [PATCH 2/4] test: rename dict fuzz e2e test to run last --- mooncake-wheel/tests/test_distributed_object_store.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mooncake-wheel/tests/test_distributed_object_store.py b/mooncake-wheel/tests/test_distributed_object_store.py index ee2b0eac202..e51e27adc25 100644 --- a/mooncake-wheel/tests/test_distributed_object_store.py +++ b/mooncake-wheel/tests/test_distributed_object_store.py @@ -272,7 +272,8 @@ def worker(thread_id): print(f"System Put bandwidth: {total_data_size_gb/put_duration:.2f} GB/sec") print(f"System Get bandwidth: {total_data_size_gb/get_duration:.2f} GB/sec") - def test_dict_fuzz_e2e(self): + # Mark this test as zzz_ so that it is the last test to run + def zzz_test_dict_fuzz_e2e(self): """End-to-end fuzz test comparing distributed store behavior with dict. Performs ~1000 random operations (put, get, remove) with random value sizes between 1KB and 64MB. After testing, all keys are removed. From a03cc2f16ef14bda64cc3601bb3f6d4d978d122f Mon Sep 17 00:00:00 2001 From: Jinyang Su <751080330@qq.com> Date: Fri, 20 Jun 2025 17:29:38 +0800 Subject: [PATCH 3/4] test: remove obsolete test_multicards.py from repository --- mooncake-wheel/tests/test_multicards.py | 51 ------------------------- 1 file changed, 51 deletions(-) delete mode 100644 mooncake-wheel/tests/test_multicards.py diff --git a/mooncake-wheel/tests/test_multicards.py b/mooncake-wheel/tests/test_multicards.py deleted file mode 100644 index 1f5467b3aff..00000000000 --- a/mooncake-wheel/tests/test_multicards.py +++ /dev/null @@ -1,51 +0,0 @@ -import os -import unittest - -from mooncake.store import MooncakeDistributedStore - -class TestMultiCards(unittest.TestCase): - @classmethod - def setUpClass(cls): - """Initialize the store once for all tests.""" - cls.store = MooncakeDistributedStore() - protocol = os.getenv("PROTOCOL", "rdma") - # get device from auto discovery - device_name = os.getenv("DEVICE_NAME", "") - # Changed to the local hostname (retrieved by `ip addr`) accordingly - local_hostname = os.getenv("LOCAL_HOSTNAME", "127.0.0.1") - metadata_server = os.getenv("MC_METADATA_SERVER", "127.0.0.1:2379") - global_segment_size = 3200 * 1024 * 1024 - local_buffer_size = 512 * 1024 * 1024 - master_server_address = os.getenv("MASTER_SERVER", "127.0.0.1:50051") - retcode = cls.store.setup(local_hostname, - metadata_server, - global_segment_size, - local_buffer_size, - protocol, - device_name, - master_server_address) - - if retcode: - raise RuntimeError(f"Failed to setup store client. Return code: {retcode}") - - def test_two_cards_send_to_each_other(self): - """To verify if one card can work correctly as both receiver and sender at same time - inside same process with multicards configured via auto discovery. - """ - index = 0 - # make value length just equal to 2 * (slice size) - # so that two cards send data to each other at almost same time. - value_length = 65536 * 2 - max_requests = 10 - value = os.urandom(value_length) - while index < max_requests: - key = "k_" + str(index) - value = bytes([index % 256] * value_length) - retcode = self.store.put(key, value) - if retcode: - raise RuntimeError("put failed, key", key) - index += 1 - print("completed", index, "entries") - -if __name__ == '__main__': - unittest.main() \ No newline at end of file From 1ed1ebe821d8cbc851b69f578837f5287b64ee90 Mon Sep 17 00:00:00 2001 From: Jinyang Su <751080330@qq.com> Date: Fri, 20 Jun 2025 17:34:15 +0800 Subject: [PATCH 4/4] chore(tests): remove multicards test execution from script --- scripts/run_tests.sh | 7 ------- 1 file changed, 7 deletions(-) diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index b724fafd70c..c3622cc0fde 100755 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -28,13 +28,6 @@ sleep 1 MC_METADATA_SERVER=http://127.0.0.1:8080/metadata python test_distributed_object_store.py kill $MASTER_PID || true -echo "Running with multicards tests..." -mooncake_master & -MASTER_PID=$! -sleep 1 -MC_METADATA_SERVER=http://127.0.0.1:8080/metadata python test_multicards.py -kill $MASTER_PID || true - echo "Running CLI entry point tests..." python test_cli.py