From bf17c03377fccc2dc33d4f0966544217915cdf62 Mon Sep 17 00:00:00 2001 From: Teng Ma Date: Mon, 2 Mar 2026 21:47:43 +0800 Subject: [PATCH 1/3] [TransferEngine] Fix RDMA GID auto-discovery for IPv6 and reduce spurious errors Fixes #1593 and #1592 Changes: 1. Accept all RoCE v2 GIDs (both IPv4-mapped and pure IPv6) instead of only IPv4-mapped GIDs. This allows Transfer Engine to work in IPv6-only environments. 2. Stop querying GID indices on first failure instead of iterating through all 256 possible indices. This eliminates spurious 'Failed to query GID' error logs when devices have fewer GIDs. 3. Allow user-specified GIDs without network devices. When MC_GID_INDEX is explicitly set, log a warning but continue initialization instead of failing. Users setting this variable know their configuration. --- .../transport/rdma_transport/rdma_context.cpp | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/mooncake-transfer-engine/src/transport/rdma_transport/rdma_context.cpp b/mooncake-transfer-engine/src/transport/rdma_transport/rdma_context.cpp index ec1c225164..d698ead384 100644 --- a/mooncake-transfer-engine/src/transport/rdma_transport/rdma_context.cpp +++ b/mooncake-transfer-engine/src/transport/rdma_transport/rdma_context.cpp @@ -447,13 +447,11 @@ GidNetworkState RdmaContext::findBestGidIndex(const std::string &device_name, for (i = 0; i < port_attr.gid_tbl_len; i++) { if (ibv_query_gid_ex(context, port, i, &gid_entry, 0)) { - PLOG(ERROR) << "Failed to query GID " << i << " on " << device_name - << "/" << port; - continue; // if gid is invalid ibv_query_gid_ex() will return !0 + // Reached end of valid GID indices + break; } - if ((ipv6_addr_v4mapped((struct in6_addr *)gid_entry.gid.raw) && - gid_entry.gid_type == IBV_GID_TYPE_ROCE_V2) || + if (gid_entry.gid_type == IBV_GID_TYPE_ROCE_V2 || gid_entry.gid_type == IBV_GID_TYPE_IB) { // Check if this GID has an associated network device if (hasNetworkDevice(device_name, port, i)) { @@ -584,16 +582,17 @@ int RdmaContext::openRdmaDevice(const std::string &device_name, uint8_t port, } } else { // Also check network state for user-specified GID - if (!hasNetworkDevice(device_name, port, gid_index)) { + bool has_ndev = hasNetworkDevice(device_name, port, gid_index); + if (!has_ndev) { LOG(WARNING) << "User-specified GID index " << gid_index << " on " << device_name << "/" << port << " has no associated network device, " << "may not be optimal for RDMA operations"; - goto cleanup_context_and_devices; } LOG(INFO) << "Using user-specified GID index: " << gid_index - << " on " << device_name << "/" << port - << " (with network device)"; + << " on " << device_name << "/" << port << " (" + << (has_ndev ? "with" : "without") + << " network device)"; } // Continue with GID validation From de5404c311c326f8629d47c76c19b4c9e5d38708 Mon Sep 17 00:00:00 2001 From: Teng Ma Date: Mon, 9 Mar 2026 19:56:53 +0800 Subject: [PATCH 2/3] update --- .../src/transport/rdma_transport/rdma_context.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mooncake-transfer-engine/src/transport/rdma_transport/rdma_context.cpp b/mooncake-transfer-engine/src/transport/rdma_transport/rdma_context.cpp index d698ead384..b4da63b52f 100644 --- a/mooncake-transfer-engine/src/transport/rdma_transport/rdma_context.cpp +++ b/mooncake-transfer-engine/src/transport/rdma_transport/rdma_context.cpp @@ -591,8 +591,7 @@ int RdmaContext::openRdmaDevice(const std::string &device_name, uint8_t port, } LOG(INFO) << "Using user-specified GID index: " << gid_index << " on " << device_name << "/" << port << " (" - << (has_ndev ? "with" : "without") - << " network device)"; + << (has_ndev ? "with" : "without") << " network device)"; } // Continue with GID validation From 1be319d9710cbff1a56cdd602a8a2d1cb75052d1 Mon Sep 17 00:00:00 2001 From: Teng Ma Date: Mon, 9 Mar 2026 20:30:00 +0800 Subject: [PATCH 3/3] update --- .../transport/rdma_transport/rdma_context.cpp | 54 +++++++++++++++---- 1 file changed, 43 insertions(+), 11 deletions(-) diff --git a/mooncake-transfer-engine/src/transport/rdma_transport/rdma_context.cpp b/mooncake-transfer-engine/src/transport/rdma_transport/rdma_context.cpp index b4da63b52f..816cb89046 100644 --- a/mooncake-transfer-engine/src/transport/rdma_transport/rdma_context.cpp +++ b/mooncake-transfer-engine/src/transport/rdma_transport/rdma_context.cpp @@ -442,7 +442,9 @@ GidNetworkState RdmaContext::findBestGidIndex(const std::string &device_name, gid_index = -1; int i; struct ibv_gid_entry gid_entry; - bool fallback_found = false; + int fallback_ipv4_gid_without_network = -1; + int fallback_ipv6_gid_with_network = -1; + int fallback_ipv6_gid_without_network = -1; GidNetworkState state = GidNetworkState::GID_NOT_FOUND; for (i = 0; i < port_attr.gid_tbl_len; i++) { @@ -451,23 +453,53 @@ GidNetworkState RdmaContext::findBestGidIndex(const std::string &device_name, break; } - if (gid_entry.gid_type == IBV_GID_TYPE_ROCE_V2 || - gid_entry.gid_type == IBV_GID_TYPE_IB) { - // Check if this GID has an associated network device - if (hasNetworkDevice(device_name, port, i)) { - // Found a GID with network device, this is the best choice + if (gid_entry.gid_type != IBV_GID_TYPE_ROCE_V2 && + gid_entry.gid_type != IBV_GID_TYPE_IB) { + continue; + } + + const bool is_ipv4_gid = + gid_entry.gid_type == IBV_GID_TYPE_ROCE_V2 && + ipv6_addr_v4mapped((struct in6_addr *)gid_entry.gid.raw); + const bool has_network_device = hasNetworkDevice(device_name, port, i); + + if (is_ipv4_gid) { + if (has_network_device) { gid_index = i; - state = GidNetworkState::GID_WITH_NETWORK; - break; + return GidNetworkState::GID_WITH_NETWORK; } - // No network device, keep the first one as fallback candidate - if (!fallback_found) { + if (fallback_ipv4_gid_without_network < 0) { gid_index = i; - fallback_found = true; + fallback_ipv4_gid_without_network = i; state = GidNetworkState::GID_WITHOUT_NETWORK; } + continue; + } + + if (has_network_device && fallback_ipv6_gid_with_network < 0) { + fallback_ipv6_gid_with_network = i; + } + + if (!has_network_device && fallback_ipv6_gid_without_network < 0) { + fallback_ipv6_gid_without_network = i; } } + + if (fallback_ipv4_gid_without_network >= 0) { + gid_index = fallback_ipv4_gid_without_network; + return GidNetworkState::GID_WITHOUT_NETWORK; + } + + if (fallback_ipv6_gid_with_network >= 0) { + gid_index = fallback_ipv6_gid_with_network; + return GidNetworkState::GID_WITH_NETWORK; + } + + if (fallback_ipv6_gid_without_network >= 0) { + gid_index = fallback_ipv6_gid_without_network; + return GidNetworkState::GID_WITHOUT_NETWORK; + } + return state; }