From e12ef7bf2e525d117ff8bcf0d929cdab47f6e914 Mon Sep 17 00:00:00 2001 From: HanChao Date: Fri, 9 Oct 2026 07:12:41 +0000 Subject: [PATCH 1/2] [xccl] WA: preload libccl.so.1 so the oneCCL legacy plugin can load libccl.so.2 dlopen()s its legacy plugin libccl_legacy.so, which lists libccl.so.1 in DT_NEEDED but ships without RPATH/RUNPATH. A dlopen'd object does not inherit the search path of whoever pulled it in, so in a flat PyPI/venv layout libccl.so.1 is looked up only in /etc/ld.so.cache, /lib and /usr/lib. It is not found there and oneCCL fails with "oneCCL could not be initialized! Could not load any plugin.", taking down every XCCL process group creation unless the user sets LD_LIBRARY_PATH by hand. dlopen("libccl.so.1", RTLD_NOW | RTLD_GLOBAL) from this library resolves it through our own RPATH -- the same one that already locates libccl.so.2 -- and leaves it in the global scope, so the plugin binds against the already-loaded object instead of searching the filesystem. Reproduced outside torch with a stub mirroring the real layout (NEEDED libccl.so.2 + RUNPATH to the oneCCL dir, oneCCL not on LD_LIBRARY_PATH): without preload: dlopen libccl_legacy.so -> libccl.so.1: cannot open shared object file with preload : dlopen libccl.so.1 -> OK dlopen libccl_legacy.so -> OK --- cmake/XCCL.cmake | 4 +++- src/xccl/xccl.cpp | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/cmake/XCCL.cmake b/cmake/XCCL.cmake index 2fe9efd1b40..15a2c59fea9 100644 --- a/cmake/XCCL.cmake +++ b/cmake/XCCL.cmake @@ -24,7 +24,9 @@ if(NOT __XCCL_INCLUDED) ${XCCL_INCLUDE_DIR}) # Wrap with --no-as-needed,...,--as-needed so it remains in NEEDED. # Once the weak attribute is removed upstream, this wrapper can be dropped. + # CMAKE_DL_LIBS is for the libccl.so.1 preload in src/xccl/xccl.cpp; it can go + # away together with that workaround. set_property( TARGET torch::xccl PROPERTY INTERFACE_LINK_LIBRARIES - "-Wl,--no-as-needed,${XCCL_LIBRARY},--as-needed") + "-Wl,--no-as-needed,${XCCL_LIBRARY},--as-needed" ${CMAKE_DL_LIBS}) endif() diff --git a/src/xccl/xccl.cpp b/src/xccl/xccl.cpp index 8e08741934f..3cab633fb39 100644 --- a/src/xccl/xccl.cpp +++ b/src/xccl/xccl.cpp @@ -10,6 +10,38 @@ #include +#ifndef _WIN32 +#include +#endif + +namespace { + +// WA. revert it once DLE 2026.2 releases. +// +// libccl.so.2 dlopen()s its legacy plugin libccl_legacy.so, which lists +// libccl.so.1 in DT_NEEDED but ships without RPATH/RUNPATH. A dlopen'd object +// does not inherit the search path of whoever pulled it in, so in the flat +// PyPI/venv layout libccl.so.1 is looked up only in /etc/ld.so.cache, /lib and +// /usr/lib; it is not found and oneCCL fails with "Could not load any plugin", +// taking down every process group creation. Loading it here resolves it +// through this library's RPATH -- the same one that already locates +// libccl.so.2 -- and leaves it in the global scope for the plugin to bind +// against. +// +// A failure here is expected on a full oneAPI/DLE layout, where the plugin +// resolves its own dependencies, so it is deliberately not reported. +// +// oneCCL 2022.2 adds $ORIGIN search paths to its wheels; drop this once that +// is the minimum supported version. +[[maybe_unused]] const bool preloaded_oneccl_legacy_dep = []() { +#ifndef _WIN32 + (void)dlopen("libccl.so.1", RTLD_NOW | RTLD_GLOBAL); +#endif + return true; +}(); + +} // namespace + namespace c10d { namespace xccl { From 7d3e518b6436f2f6c04f90d882b2e314364f60e6 Mon Sep 17 00:00:00 2001 From: HanChao Date: Sat, 10 Oct 2026 00:53:04 +0000 Subject: [PATCH 2/2] also remove ci wa --- .github/actions/linux-uttest/action.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/actions/linux-uttest/action.yml b/.github/actions/linux-uttest/action.yml index 96b0acd4efc..b8b1b7879d6 100644 --- a/.github/actions/linux-uttest/action.yml +++ b/.github/actions/linux-uttest/action.yml @@ -171,7 +171,6 @@ runs: run: | xpu-smi topology -m mkdir -p ut_log/xpu_distributed - export PATH="${VENV_ROOT}/bin/libfabric:${PATH}" XCCL_ENABLE=$(python -c "import torch;print(torch.distributed.is_xccl_available())") if [[ "${XCCL_ENABLE,,}" == 'false' ]] || [[ "${XCCL_ENABLE}" == '0' ]]; then echo -e "[ERROR] XCCL is not enabled"